contentful 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4ba77ca80d0071ad28546271729538c6a701616d
-  data.tar.gz: 65a9ee25b3709bb641deed8345862914480068e7
+  metadata.gz: 78c2a767a88bc7cf15bde4fcc9173611d0c2a116
+  data.tar.gz: d99bee10915e4128e254b9f41fb4ad8472ec2f83
 SHA512:
-  metadata.gz: 51c966da267b2dc62c189a57dd69200ec9a83f81cba5bccd4695c5ef756babdc6aeccbb53a7c98bde97325fbfff86c68eac08912871192d035d36188828de389
-  data.tar.gz: d5c14ff5244ef3463bfec9cac959be1de527f45ca0f48e8885101ea473dc5f53f1b8d471a2479b668504cc8ee003ccb23ea4be549a3e5aee1135929f03039c91
+  metadata.gz: e0118fccbdb2a308c14b89748a5f9fdc42f32bf1593e6917897e2edb0613ea98843fdb47dda3642a722a475dad7f75df4297c9500638e185e14109b52a829532
+  data.tar.gz: e6c7aff051696aa83e5c9fa0d52c021bffec3535c04e45650f0b00d4dcf4364f0d18089010af3fd5b11153851b25e64c59e9e359eb75da7034d51fce7f40e394

data/CHANGELOG.md

@@ -1,6 +1,11 @@
 # Change Log
 ## Unreleased
 
+## 0.12.0
+
+### Added
+* Added Rate Limit automatic handling
+
 ## 0.11.0
 ### Fixed
 * Fixed Locale handling [#98](https://github.com/contentful/contentful.rb/issues/98)

data/README.md

@@ -238,6 +238,17 @@ Specify the port number that is used by the proxy server for client connections.
 
 To use the proxy with authentication, you need to specify ```port_username``` and ```port_password```.
 
+### :max_rate_limit_retries
+
+To increase or decrease the retry attempts after a 429 Rate Limit error. Default value is 1. Using 0 will disable retry behaviour.
+Each retry will be attempted after the value (in seconds) of the `X-Contentful-RateLimit-Reset` header, which contains the amount of seconds until the next
+non rate limited request is available, has passed. This is blocking per execution thread.
+
+### :max_rate_limit_wait
+
+Maximum time to wait for next available request (in seconds). Default value is 60 seconds. Keep in mind that if you hit the houly rate limit maximum, you
+can have up to 60 minutes of blocked requests. It is set to a default of 60 seconds in order to avoid blocking processes for too long, as rate limit retry behaviour
+is blocking per execution thread.
 
 
 ### Proxy example

data/lib/contentful/client.rb

@@ -30,8 +30,12 @@ module Contentful
       proxy_host: nil,
       proxy_username: nil,
       proxy_password: nil,
-      proxy_port: nil
+      proxy_port: nil,
+      max_rate_limit_retries: 1,
+      max_rate_limit_wait: 60
     }
+    # Rate Limit Reset Header Key
+    RATE_LIMIT_RESET_HEADER_KEY = 'x-contentful-ratelimit-reset'
 
     attr_reader :configuration, :dynamic_entry_cache, :logger, :proxy
 
@@ -56,6 +60,8 @@ module Contentful
     # @option given_configuration [String] :proxy_username
     # @option given_configuration [String] :proxy_password
     # @option given_configuration [Number] :proxy_port
+    # @option given_configuration [Number] :max_rate_limit_retries
+    # @option given_configuration [Number] :max_rate_limit_wait
     # @option given_configuration [Boolean] :gzip_encoded
     # @option given_configuration [Boolean] :raw_mode
     # @option given_configuration [false, ::Logger] :logger
@@ -196,9 +202,44 @@ module Contentful
     #
     # @private
     def get(request, build_resource = true)
+      retries_left = configuration[:max_rate_limit_retries]
+      begin
+        response = run_request(request)
+
+        return response if !build_resource || configuration[:raw_mode]
+
+        result = do_build_resource(response)
+
+        fail result if result.is_a?(Error) && configuration[:raise_errors]
+      rescue Contentful::RateLimitExceeded => rate_limit_error
+        reset_time = rate_limit_error.response.raw[RATE_LIMIT_RESET_HEADER_KEY].to_i
+        if should_retry(retries_left, reset_time, configuration[:max_rate_limit_wait])
+          retries_left -= 1
+          retry_message = 'Contentful API Rate Limit Hit! '
+          retry_message += "Retrying - Retries left: #{retries_left}"
+          retry_message += "- Time until reset (seconds): #{reset_time}"
+          logger.info(retry_message) if logger
+          sleep(reset_time * Random.new.rand(1.0..1.2))
+          retry
+        end
+
+        raise
+      end
+
+      result
+    end
+
+    # @private
+    def should_retry(retries_left, reset_time, max_wait)
+      retries_left > 0 && max_wait > reset_time
+    end
+
+    # Runs request and parses Response
+    # @private
+    def run_request(request)
       url = request.absolute? ? request.url : base_url + request.url
       logger.info(request: { url: url, query: request.query, header: request_headers }) if logger
-      response = Response.new(
+      Response.new(
         self.class.get_http(
           url,
           request_query(request.query),
@@ -206,19 +247,19 @@ module Contentful
           proxy_params
         ), request
       )
+    end
 
-      return response if !build_resource || configuration[:raw_mode]
+    # Runs Resource Builder
+    # @private
+    def do_build_resource(response)
       logger.debug(response: response) if logger
-      result = configuration[:resource_builder].new(
+      configuration[:resource_builder].new(
         self,
         response,
         configuration[:resource_mapping],
         configuration[:entry_mapping],
         configuration[:default_locale]
       ).run
-
-      fail result if result.is_a?(Error) && configuration[:raise_errors]
-      result
     end
 
     # Use this method together with the client's :dynamic_entries configuration.
@@ -226,12 +267,12 @@ module Contentful
     # @private
     def update_dynamic_entry_cache!
       @dynamic_entry_cache = Hash[
-                             content_types(limit: 1000).map do |ct|
-                               [
-                                 ct.id.to_sym,
-                                 DynamicEntry.create(ct)
-                               ]
-                             end
+        content_types(limit: 1000).map do |ct|
+          [
+            ct.id.to_sym,
+            DynamicEntry.create(ct)
+          ]
+        end
       ]
     end
 

data/lib/contentful/version.rb

@@ -1,5 +1,5 @@
 # Contentful Namespace
 module Contentful
   # Gem Version
-  VERSION = '0.11.0'
+  VERSION = '0.12.0'
 end

data/spec/error_requests_spec.rb

@@ -1,5 +1,27 @@
 require 'spec_helper'
 
+class NonCachingClient < Contentful::Client
+  def request_headers
+    headers = super
+    headers['Cf-No-Cache'] = 'foobar'
+    headers
+  end
+end
+
+class RetryLoggerMock < Logger
+  attr_reader :retry_attempts
+
+  def initialize(*)
+    super
+    @retry_attempts = 0
+  end
+
+  def info(message)
+    super
+    @retry_attempts += 1 if message.include?('Contentful API Rate Limit Hit! Retrying')
+  end
+end
+
 describe 'Error Requests' do
   it 'will return 404 (Unauthorized) if resource not found' do
     expect_vcr('not found')do
@@ -29,13 +51,32 @@ describe 'Error Requests' do
     skip
   end
 
-  it 'will return a 429 if the ratelimit is reached' do
-    client = Contentful::Client.new(space: 'wrong', access_token: 'credentials')
+  it 'will return a 429 if the ratelimit is reached and is not set to retry' do
+    client = Contentful::Client.new(space: 'wrong', access_token: 'credentials', max_rate_limit_retries: 0)
     expect_vcr('ratelimit') {
       client.entry('nyancat')
     }.to raise_error(Contentful::RateLimitExceeded)
   end
 
+  it 'will retry on 429 by default' do
+    logger = RetryLoggerMock.new(STDOUT)
+    client = NonCachingClient.new(
+      api_url: 'cdnorigin.flinkly.com',
+      space: '164vhtp008kz',
+      access_token: '7699b6c6f6cee9b6abaa216c71fbcb3eee56cb6f082f57b5e21b2b50f86bdea0',
+      raise_errors: true,
+      logger: logger
+    )
+
+    vcr('ratelimit_retry') {
+      3.times {
+        client.assets
+      }
+    }
+
+    expect(logger.retry_attempts).to eq 1
+  end
+
   it 'will return 503 (ServiceUnavailable) when the service is unavailable' do
     client = Contentful::Client.new(space: 'wrong', access_token: 'credentials')
 

data/spec/fixtures/vcr_cassettes/ratelimit_retry.yml

@@ -0,0 +1,294 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://cdnorigin.flinkly.com/spaces/164vhtp008kz/assets
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - RubyContentfulGem/0.10.0
+      Authorization:
+      - Bearer 7699b6c6f6cee9b6abaa216c71fbcb3eee56cb6f082f57b5e21b2b50f86bdea0
+      Content-Type:
+      - application/vnd.contentful.delivery.v1+json
+      Accept-Encoding:
+      - gzip
+      Cf-No-Cache:
+      - foobar
+      Connection:
+      - close
+      Host:
+      - cdnorigin.flinkly.com
+  response:
+    status:
+      code: 429
+      message: Too Many Requests
+    headers:
+      Accept-Ranges:
+      - bytes
+      Access-Control-Allow-Headers:
+      - Accept,Accept-Language,Authorization,Cache-Control,Content-Length,Content-Range,Content-Type,DNT,Destination,Expires,If-Match,If-Modified-Since,If-None-Match,Keep-Alive,Last-Modified,Origin,Pragma,Range,User-Agent,X-Http-Method-Override,X-Mx-ReqToken,X-Requested-With,X-Contentful-Version,X-Contentful-Content-Type,X-Contentful-Organization,X-Contentful-Skip-Transformation,X-Contentful-User-Agent
+      Access-Control-Allow-Methods:
+      - GET,HEAD,OPTIONS
+      Access-Control-Allow-Origin:
+      - "*"
+      Access-Control-Max-Age:
+      - '86400'
+      Cache-Control:
+      - max-age=0
+      Content-Type:
+      - application/vnd.contentful.management.v1+json
+      Date:
+      - Tue, 09 Aug 2016 08:46:00 GMT
+      Server:
+      - nginx
+      X-Contentful-Ratelimit-Reset:
+      - '1'
+      Content-Length:
+      - '304'
+      Connection:
+      - Close
+    body:
+      encoding: UTF-8
+      string: |
+        {
+          "sys": {
+            "type": "Error",
+            "id": "RateLimitExceeded"
+          },
+          "message": "You have exceeded the rate limit of the Organization this Space belongs to by making too many API requests within a short timespan. Please wait a moment before trying the request again.",
+          "requestId": "395-766168967"
+        }
+    http_version: 
+  recorded_at: Tue, 09 Aug 2016 08:46:00 GMT
+- request:
+    method: get
+    uri: https://cdnorigin.flinkly.com/spaces/164vhtp008kz/assets
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - RubyContentfulGem/0.10.0
+      Authorization:
+      - Bearer 7699b6c6f6cee9b6abaa216c71fbcb3eee56cb6f082f57b5e21b2b50f86bdea0
+      Content-Type:
+      - application/vnd.contentful.delivery.v1+json
+      Accept-Encoding:
+      - gzip
+      Cf-No-Cache:
+      - foobar
+      Connection:
+      - close
+      Host:
+      - cdnorigin.flinkly.com
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Accept-Ranges:
+      - bytes
+      Access-Control-Allow-Headers:
+      - Accept,Accept-Language,Authorization,Cache-Control,Content-Length,Content-Range,Content-Type,DNT,Destination,Expires,If-Match,If-Modified-Since,If-None-Match,Keep-Alive,Last-Modified,Origin,Pragma,Range,User-Agent,X-Http-Method-Override,X-Mx-ReqToken,X-Requested-With,X-Contentful-Version,X-Contentful-Content-Type,X-Contentful-Organization,X-Contentful-Skip-Transformation,X-Contentful-User-Agent
+      Access-Control-Allow-Methods:
+      - GET,HEAD,OPTIONS
+      Access-Control-Allow-Origin:
+      - "*"
+      Access-Control-Expose-Headers:
+      - Etag
+      Access-Control-Max-Age:
+      - '86400'
+      Cache-Control:
+      - max-age=0
+      Content-Type:
+      - application/vnd.contentful.delivery.v1+json
+      Date:
+      - Tue, 09 Aug 2016 08:46:00 GMT
+      Etag:
+      - '"29f2c21be26360c424f617d8592cf6f9"'
+      Server:
+      - nginx
+      Surrogate-Control:
+      - max-age=21600
+      Surrogate-Key:
+      - space-164vhtp008kz organization-7Kp22ygMRaA3DCjACF64PF
+      Vary:
+      - Accept-Encoding
+      X-Content-Type-Options:
+      - nosniff
+      X-Contentful-Request-Id:
+      - 395-766168968
+      Content-Length:
+      - '97'
+      Connection:
+      - Close
+    body:
+      encoding: UTF-8
+      string: |
+        {
+          "sys": {
+            "type": "Array"
+          },
+          "total": 0,
+          "skip": 0,
+          "limit": 100,
+          "items": []
+        }
+    http_version: 
+  recorded_at: Tue, 09 Aug 2016 08:46:00 GMT
+- request:
+    method: get
+    uri: https://cdnorigin.flinkly.com/spaces/164vhtp008kz/assets
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - RubyContentfulGem/0.10.0
+      Authorization:
+      - Bearer 7699b6c6f6cee9b6abaa216c71fbcb3eee56cb6f082f57b5e21b2b50f86bdea0
+      Content-Type:
+      - application/vnd.contentful.delivery.v1+json
+      Accept-Encoding:
+      - gzip
+      Cf-No-Cache:
+      - foobar
+      Connection:
+      - close
+      Host:
+      - cdnorigin.flinkly.com
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Accept-Ranges:
+      - bytes
+      Access-Control-Allow-Headers:
+      - Accept,Accept-Language,Authorization,Cache-Control,Content-Length,Content-Range,Content-Type,DNT,Destination,Expires,If-Match,If-Modified-Since,If-None-Match,Keep-Alive,Last-Modified,Origin,Pragma,Range,User-Agent,X-Http-Method-Override,X-Mx-ReqToken,X-Requested-With,X-Contentful-Version,X-Contentful-Content-Type,X-Contentful-Organization,X-Contentful-Skip-Transformation,X-Contentful-User-Agent
+      Access-Control-Allow-Methods:
+      - GET,HEAD,OPTIONS
+      Access-Control-Allow-Origin:
+      - "*"
+      Access-Control-Expose-Headers:
+      - Etag
+      Access-Control-Max-Age:
+      - '86400'
+      Cache-Control:
+      - max-age=0
+      Content-Type:
+      - application/vnd.contentful.delivery.v1+json
+      Date:
+      - Tue, 09 Aug 2016 08:46:00 GMT
+      Etag:
+      - '"29f2c21be26360c424f617d8592cf6f9"'
+      Server:
+      - nginx
+      Surrogate-Control:
+      - max-age=21600
+      Surrogate-Key:
+      - space-164vhtp008kz organization-7Kp22ygMRaA3DCjACF64PF
+      Vary:
+      - Accept-Encoding
+      X-Content-Type-Options:
+      - nosniff
+      X-Contentful-Request-Id:
+      - 395-766168969
+      Content-Length:
+      - '97'
+      Connection:
+      - Close
+    body:
+      encoding: UTF-8
+      string: |
+        {
+          "sys": {
+            "type": "Array"
+          },
+          "total": 0,
+          "skip": 0,
+          "limit": 100,
+          "items": []
+        }
+    http_version: 
+  recorded_at: Tue, 09 Aug 2016 08:46:00 GMT
+- request:
+    method: get
+    uri: https://cdnorigin.flinkly.com/spaces/164vhtp008kz/assets
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - RubyContentfulGem/0.10.0
+      Authorization:
+      - Bearer 7699b6c6f6cee9b6abaa216c71fbcb3eee56cb6f082f57b5e21b2b50f86bdea0
+      Content-Type:
+      - application/vnd.contentful.delivery.v1+json
+      Accept-Encoding:
+      - gzip
+      Cf-No-Cache:
+      - foobar
+      Connection:
+      - close
+      Host:
+      - cdnorigin.flinkly.com
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Accept-Ranges:
+      - bytes
+      Access-Control-Allow-Headers:
+      - Accept,Accept-Language,Authorization,Cache-Control,Content-Length,Content-Range,Content-Type,DNT,Destination,Expires,If-Match,If-Modified-Since,If-None-Match,Keep-Alive,Last-Modified,Origin,Pragma,Range,User-Agent,X-Http-Method-Override,X-Mx-ReqToken,X-Requested-With,X-Contentful-Version,X-Contentful-Content-Type,X-Contentful-Organization,X-Contentful-Skip-Transformation,X-Contentful-User-Agent
+      Access-Control-Allow-Methods:
+      - GET,HEAD,OPTIONS
+      Access-Control-Allow-Origin:
+      - "*"
+      Access-Control-Expose-Headers:
+      - Etag
+      Access-Control-Max-Age:
+      - '86400'
+      Cache-Control:
+      - max-age=0
+      Content-Type:
+      - application/vnd.contentful.delivery.v1+json
+      Date:
+      - Tue, 09 Aug 2016 08:46:02 GMT
+      Etag:
+      - '"29f2c21be26360c424f617d8592cf6f9"'
+      Server:
+      - nginx
+      Surrogate-Control:
+      - max-age=21600
+      Surrogate-Key:
+      - space-164vhtp008kz organization-7Kp22ygMRaA3DCjACF64PF
+      Vary:
+      - Accept-Encoding
+      X-Content-Type-Options:
+      - nosniff
+      X-Contentful-Request-Id:
+      - 395-766168970
+      Content-Length:
+      - '97'
+      Connection:
+      - Close
+    body:
+      encoding: UTF-8
+      string: |
+        {
+          "sys": {
+            "type": "Array"
+          },
+          "total": 0,
+          "skip": 0,
+          "limit": 100,
+          "items": []
+        }
+    http_version: 
+  recorded_at: Tue, 09 Aug 2016 08:46:02 GMT
+recorded_with: VCR 3.0.3

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: contentful
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Contentful GmbH (Jan Lelis)
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-09 00:00:00.000000000 Z
+date: 2016-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -400,6 +400,7 @@ files:
 - spec/fixtures/vcr_cassettes/not_found.yml
 - spec/fixtures/vcr_cassettes/nyancat.yml
 - spec/fixtures/vcr_cassettes/ratelimit.yml
+- spec/fixtures/vcr_cassettes/ratelimit_retry.yml
 - spec/fixtures/vcr_cassettes/reloaded_entry.yml
 - spec/fixtures/vcr_cassettes/space.yml
 - spec/fixtures/vcr_cassettes/sync_deleted_asset.yml
@@ -493,6 +494,7 @@ test_files:
 - spec/fixtures/vcr_cassettes/not_found.yml
 - spec/fixtures/vcr_cassettes/nyancat.yml
 - spec/fixtures/vcr_cassettes/ratelimit.yml
+- spec/fixtures/vcr_cassettes/ratelimit_retry.yml
 - spec/fixtures/vcr_cassettes/reloaded_entry.yml
 - spec/fixtures/vcr_cassettes/space.yml
 - spec/fixtures/vcr_cassettes/sync_deleted_asset.yml