itrp-client 1.1.4 → 1.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3191ba404199a9c372100cfbaf4d098b4bf421ea
-  data.tar.gz: 6af16dbe18fe503f3186cb8ed1110bf8e00b518c
+  metadata.gz: 8703ef24c263256f975b6bcc841ea70b2b9dd9fb
+  data.tar.gz: b7d384aab10d84b2b1cab2e33fff2975fbf69a40
 SHA512:
-  metadata.gz: 722a14bab6be83a23c483d2dece405bf680901dbdff3f7389211e7dcea9f6af2028cdc0f07510e091f045ce8909fa7fbb432e486436307e10481ad96150a583e
-  data.tar.gz: fb8f9090c5ce0c3da2c81e4d8431521a132f448c94a493ebed8ff459f527eecf0b5173341ad5e36a949f2f5fe3438d170d5f5790c0c7d048905cbbbb64bd029a
+  metadata.gz: 66205dde73494d6e6a95a45a316bf80522f5a61106735b315ad8d9636a8bd3d2fd6a375228476f42ad1b154c40933d3a6e24538afd20a1fea98e6ba46e2df40e
+  data.tar.gz: 213c2468c8be27d03b53f318a3b6aa2cf7e872a05be9042128b88db7bd1f90327a8d91170ac0c23227720a7fe99e51b35668cae647d924872dd32439a7c8d085

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    itrp-client (1.1.1)
+    itrp-client (1.1.5)
       activesupport
       gem_config
       mime-types
@@ -9,26 +9,27 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.2.7.1)
-      i18n (~> 0.7)
-      json (~> 1.7, >= 1.7.7)
+    activesupport (5.2.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
-      thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
     addressable (2.5.0)
       public_suffix (~> 2.0, >= 2.0.2)
+    concurrent-ruby (1.1.4)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     diff-lcs (1.2.5)
     docile (1.1.5)
     gem_config (0.3.1)
     hashdiff (0.3.1)
-    i18n (0.7.0)
+    i18n (1.5.1)
+      concurrent-ruby (~> 1.0)
     json (1.8.3)
-    mime-types (3.1)
+    mime-types (3.2.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2016.0521)
-    minitest (5.9.1)
+    mime-types-data (3.2018.0812)
+    minitest (5.11.3)
     public_suffix (2.0.4)
     rake (11.3.0)
     rspec (3.3.0)
@@ -50,8 +51,8 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
-    thread_safe (0.3.5)
-    tzinfo (1.2.2)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
     webmock (2.1.0)
       addressable (>= 2.3.6)

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2017 ITRP Inc.
+Copyright (c) 2019 ITRP Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -40,9 +40,10 @@ All options available:
 * _max_retry_time_: maximum nr of seconds to wait for server to respond (default = 5400 = 1.5 hours)<br/>
   The sleep time between retries starts at 2 seconds and doubles after each retry, i.e.
   2, 6, 18, 54, 162, 486, 1458, 4374, 13122, ... seconds.<br/>
-  One retry will always be performed unless you set the value to -1.
+  Set to 0 to prevent retries.
 * _read_timeout_:   [HTTP read timeout](http://ruby-doc.org/stdlib-2.0.0/libdoc/net/http/rdoc/Net/HTTP.html#method-i-read_timeout-3D) in seconds (default = 25)
-* _block_at_rate_limit_: Set to `true` to block the request until the [rate limit](http://developer.itrp.com/v1/#rate-limiting) is lifted, default: `false`
+* _block_at_rate_limit_: Set to `true` to block the request until the [rate limit](http://developer.itrp.com/v1/#rate-limiting) is lifted, default: `false`<br/>
+  The `Retry-After` header is used to compute when the retry should be performed. If that moment is later than the _max_retry_time_ the request will not be blocked and the throttled response is returned.
 * _proxy_host_:     Define in case HTTP traffic needs to go through a proxy
 * _proxy_port_:     Port of the proxy, defaults to 8080
 * _proxy_user_:     Proxy user

data/lib/itrp/client.rb

@@ -298,14 +298,25 @@ module Itrp
   end
 
   module SendWithRateLimitBlock
-    # Wraps the _send method with retries when the server does not responsd, see +initialize+ option +:rate_limit_block+
+    # Wraps the _send method with retries when the server does not respond, see +initialize+ option +:rate_limit_block+
     def _send(request, domain = @domain, port = @port, ssl = @ssl)
       return super(request, domain, port, ssl) unless option(:block_at_rate_limit)
       now = Time.now
+      timed_out = false
+      # respect the max_retry_time with fallback to max 1 hour and 1 minute wait time
+      max_retry_time = option(:max_retry_time) > 0 ? option(:max_retry_time) : 3660
       begin
         _response = super(request, domain, port, ssl)
-        @logger.warn { "Request throttled, trying again in 5 minutes: #{_response.message}" } and sleep(300) if _response.throttled?
-      end while _response.throttled? && (Time.now - now) < 3660 # max 1 hour and 1 minute
+        if _response.throttled?
+          retry_after = _response.retry_after == 0 ? 300 : [_response.retry_after, 2].max
+          if (Time.now - now + retry_after) < max_retry_time
+            @logger.warn { "Request throttled, trying again in #{retry_after} seconds: #{_response.message}" }
+            sleep(retry_after)
+          else
+            timed_out = true
+          end
+        end
+      end while _response.throttled? && !timed_out
       _response
     end
   end
@@ -314,15 +325,24 @@ module Itrp
   module SendWithRetries
     # Wraps the _send method with retries when the server does not respond, see +initialize+ option +:retries+
     def _send(request, domain = @domain, port = @port, ssl = @ssl)
+      return super(request, domain, port, ssl) unless option(:max_retry_time) > 0
       retries = 0
-      sleep_time = 2
-      total_retry_time = 0
+      sleep_time = 1
+      now = Time.now
+      timed_out = false
       begin
         _response = super(request, domain, port, ssl)
-        @logger.warn { "Request failed, retry ##{retries += 1} in #{sleep_time} seconds: #{_response.message}" } and sleep(sleep_time) if (_response.raw.code.to_s != '204' && _response.empty?) && option(:max_retry_time) > 0
-        total_retry_time += sleep_time
-        sleep_time *= 2
-      end while (_response.raw.code.to_s != '204' && _response.empty?) && total_retry_time < option(:max_retry_time)
+        # throttling is handled separately
+        if !_response.success? && !_response.throttled?
+          sleep_time *= 2
+          if (Time.now - now + sleep_time) < option(:max_retry_time)
+            @logger.warn { "Request failed, retry ##{retries += 1} in #{sleep_time} seconds: #{_response.message}" }
+            sleep(sleep_time)
+          else
+            timed_out = true
+          end
+        end
+      end while !_response.success? && !_response.throttled? && !timed_out
       _response
     end
   end

data/lib/itrp/client/response.rb

@@ -111,6 +111,10 @@ module Itrp
       !!(@response.code.to_s == '429' || (message && message =~ /Too Many Requests/))
     end
 
+    def retry_after
+      @current_page ||= @response.header['Retry-After'].to_i
+    end
+
     def to_s
       valid? ? json.to_s : message
     end

data/lib/itrp/client/version.rb

@@ -1,5 +1,5 @@
 module Itrp
   class Client
-    VERSION = '1.1.4'
+    VERSION = '1.1.5'
   end
 end

data/spec/lib/itrp/client_spec.rb

@@ -281,8 +281,9 @@ describe Itrp::Client do
 
   context 'import' do
     before(:each) do
+      csv_mime_type = ['text/csv', 'text/comma-separated-values'].detect{|t| MIME::Types[t].any?} # which mime type is used depends on version of mime-types gem
       @client = Itrp::Client.new(api_token: 'secret', max_retry_time: -1)
-      @multi_part_body = "--0123456789ABLEWASIEREISAWELBA9876543210\r\nContent-Disposition: form-data; name=\"type\"\r\n\r\npeople\r\n--0123456789ABLEWASIEREISAWELBA9876543210\r\nContent-Disposition: form-data; name=\"file\"; filename=\"#{@fixture_dir}/people.csv\"\r\nContent-Type: text/comma-separated-values\r\n\r\nPrimary Email,Name\nchess.cole@example.com,Chess Cole\ned.turner@example.com,Ed Turner\r\n--0123456789ABLEWASIEREISAWELBA9876543210--"
+      @multi_part_body = "--0123456789ABLEWASIEREISAWELBA9876543210\r\nContent-Disposition: form-data; name=\"type\"\r\n\r\npeople\r\n--0123456789ABLEWASIEREISAWELBA9876543210\r\nContent-Disposition: form-data; name=\"file\"; filename=\"#{@fixture_dir}/people.csv\"\r\nContent-Type: #{csv_mime_type}\r\n\r\nPrimary Email,Name\nchess.cole@example.com,Chess Cole\ned.turner@example.com,Ed Turner\r\n--0123456789ABLEWASIEREISAWELBA9876543210--"
       @multi_part_headers = {'Accept'=>'*/*', 'Content-Type'=>'multipart/form-data; boundary=0123456789ABLEWASIEREISAWELBA9876543210', 'User-Agent'=>'Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/523.10.6 (KHTML, like Gecko) Version/3.0.4 Safari/523.10.6'}
 
       @import_queued_response = {body: {state: 'queued'}.to_json}
@@ -476,10 +477,11 @@ describe Itrp::Client do
 
     it 'should not retry 4 times when max_retry_time = 16' do
       stub = stub_request(:get, 'https://api.itrp.com/v1/me').with(basic_auth: ['secret', 'x']).to_raise(StandardError.new('network error'))
-      [2,4,8,16].each_with_index do |secs, i|
+      [2,4,8].each_with_index do |secs, i|
         expect_log('Sending GET request to api.itrp.com:443/v1/me', :debug )
         expect_log("Request failed, retry ##{i+1} in #{secs} seconds: 500: No Response from Server - network error for 'api.itrp.com:443/v1/me'", :warn)
       end
+      expect_log('Sending GET request to api.itrp.com:443/v1/me', :debug )
 
       client = Itrp::Client.new(api_token: 'secret', max_retry_time: 16)
       allow(client).to receive(:sleep)
@@ -518,10 +520,40 @@ describe Itrp::Client do
       expect(response.message).to eq('429: Too Many Requests')
     end
 
-    it 'should block on rate limit when block_at_rate_limit is true' do
+    it 'should block on rate limit (max 300 seconds) when block_at_rate_limit is true' do
       stub = stub_request(:get, 'https://api.itrp.com/v1/me').with(basic_auth: ['secret', 'x']).to_return(status: 429, body: {message: 'Too Many Requests'}.to_json).then.to_return(body: {name: 'my name'}.to_json)
       expect_log('Sending GET request to api.itrp.com:443/v1/me', :debug )
-      expect_log('Request throttled, trying again in 5 minutes: 429: Too Many Requests', :warn)
+      expect_log('Request throttled, trying again in 300 seconds: 429: Too Many Requests', :warn)
+      expect_log('Sending GET request to api.itrp.com:443/v1/me', :debug )
+      expect_log(%(Response:\n{\n  "name": "my name"\n}), :debug )
+
+      client = Itrp::Client.new(api_token: 'secret', block_at_rate_limit: true)
+      allow(client).to receive(:sleep)
+      response = client.get('me')
+      expect(stub).to have_been_requested.times(2)
+      expect(response.valid?).to be_truthy
+      expect(response[:name]).to eq('my name')
+    end
+
+    it 'should block on rate limit using Retry-After when block_at_rate_limit is true' do
+      stub = stub_request(:get, 'https://api.itrp.com/v1/me').with(basic_auth: ['secret', 'x']).to_return(status: 429, body: {message: 'Too Many Requests'}.to_json, headers: {'Retry-After' => '20'}).then.to_return(body: {name: 'my name'}.to_json)
+      expect_log('Sending GET request to api.itrp.com:443/v1/me', :debug )
+      expect_log('Request throttled, trying again in 20 seconds: 429: Too Many Requests', :warn)
+      expect_log('Sending GET request to api.itrp.com:443/v1/me', :debug )
+      expect_log(%(Response:\n{\n  "name": "my name"\n}), :debug )
+
+      client = Itrp::Client.new(api_token: 'secret', block_at_rate_limit: true)
+      allow(client).to receive(:sleep)
+      response = client.get('me')
+      expect(stub).to have_been_requested.times(2)
+      expect(response.valid?).to be_truthy
+      expect(response[:name]).to eq('my name')
+    end
+
+    it 'should block on rate limit using Retry-After with minimum of 2 seconds when block_at_rate_limit is true' do
+      stub = stub_request(:get, 'https://api.itrp.com/v1/me').with(basic_auth: ['secret', 'x']).to_return(status: 429, body: {message: 'Too Many Requests'}.to_json, headers: {'Retry-After' => '1'}).then.to_return(body: {name: 'my name'}.to_json)
+      expect_log('Sending GET request to api.itrp.com:443/v1/me', :debug )
+      expect_log('Request throttled, trying again in 2 seconds: 429: Too Many Requests', :warn)
       expect_log('Sending GET request to api.itrp.com:443/v1/me', :debug )
       expect_log(%(Response:\n{\n  "name": "my name"\n}), :debug )
 

data/spec/lib/itrp/response_spec.rb

@@ -93,10 +93,10 @@ describe Itrp::Response do
       stub_request(:get, 'https://api.itrp.com/v1/organizations').with(basic_auth: ['secret', 'x']).to_return(body: '==$$!invalid')
       response = @client.get('organizations')
 
-      message = "Invalid JSON - 757: unexpected token at '==$$!invalid' for:\n#{response.body}"
-      expect(response.json[:message]).to eq(message)
-      expect(response.json['message']).to eq(message)
-      expect(response.message).to eq(message)
+      message = "unexpected token at '==$$!invalid' for:\n#{response.body}"
+      expect(response.json[:message]).to include(message)
+      expect(response.json['message']).to include(message)
+      expect(response.message).to include(message)
     end
 
     it 'should have a blank message when single record is succesfully retrieved' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: itrp-client
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.1.5
 platform: ruby
 authors:
 - ITRP
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-21 00:00:00.000000000 Z
+date: 2019-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gem_config
@@ -171,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Client for accessing the ITRP REST API