influxdb-client 1.7.0.pre.1015 → 1.7.0.pre.1121

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8f7d3acebd47909037578d03e9c41a563328fc8a112019ef705417f4ce0b5dc7
-  data.tar.gz: f5fd4db9b000fb952eedd3ef7cbdc93fec3b5806101d819fd5c9febc33b95a68
+  metadata.gz: f97e3e086f7c291167f6aa7e1433294a603cd65fc1e673e9f75daa7459c2d0a2
+  data.tar.gz: 1f8be80e71b870bb4d2b7ef5839b9434665a53b76b8706b6a8d480eca9f56185
 SHA512:
-  metadata.gz: 1da2d03a086ad92b8c7d82a29cdff3e228dd9140808df9b3ae1ed961670535ad8125d3345202b5dc6dad0f7d370057f24e8607f9bfc6479d756bbea61e04cebd
-  data.tar.gz: 1fe5947810dff9c7d97ec8eefc4d2d54d8e67db663c42e2ae606747f444bd96b9083beb1439e27fb2d2fd1c1fc6bae055d3ffd9816f0819b85039eb47b37f7a8
+  metadata.gz: 646f4d4553df39f808f5ec8f7cbc23e014bf893faf32c86f1ce9e0631c3744bafb996413ab317ca6835c481d907271b3fb297a73e0866ace6c8acd569fbe9ab3
+  data.tar.gz: 26e1400b766d2f4a13bc145513f10d6f16f043b993b80d58f3bbd2d294d61d1857385213b0397f14648f68e6a1fab6d8de4c4f8fc9b2077c39bfeb9a70bd4c82

data/.rubocop.yml

@@ -39,3 +39,5 @@ Metrics/CyclomaticComplexity:
   Max: 15
 Metrics/PerceivedComplexity:
   Max: 15
+Metrics/ParameterLists:
+  Max: 10

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 ## 1.7.0 [unreleased]
 
+### Features
+1. [#47](https://github.com/influxdata/influxdb-client-ruby/pull/47): Added max_retries, max_retry_delay and exponential_base to WriteApi
+
 ## 1.6.0 [2020-07-17]
 
 ### Bug Fixes

data/README.md

@@ -123,12 +123,16 @@ The writes are processed in batches which are configurable by `WriteOptions`:
 | --- | --- | --- |
 | batchSize | the number of data point to collect in batch | 1000 |
 | flush_interval | the number of milliseconds before the batch is written | 1000 |
-| retry_interval | the number of milliseconds to retry unsuccessful write. The retry interval is used when the InfluxDB server does not specify "Retry-After" header. | 1000 |
+| retry_interval | the number of milliseconds to retry unsuccessful write. The retry interval is used when the InfluxDB server does not specify "Retry-After" header. | 5000 |
 | jitter_interval | the number of milliseconds to increase the batch flush interval by a random amount | 0 |
-
+| max_retries | the number of max retries when write fails | 5 |
+| max_retry_delay | maximum delay when retrying write in milliseconds | 180000 |
+| exponential_base | the base for the exponential retry delay, the next delay is computed as `retry_interval * exponential_base^(attempts - 1) + random(jitter_interval)` | 5 |
 ```ruby
 write_options = InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING,
-                                            batch_size: 10, flush_interval: 5_000)
+                                            batch_size: 10, flush_interval: 5_000, 
+                                            max_retries: 3, max_retry_delay: 15_000,
+                                            exponential_base: 2)
 client = InfluxDB2::Client.new('http://localhost:9999',
                                'my-token',
                                bucket: 'my-bucket',

data/lib/influxdb2/client/default_api.rb

@@ -83,6 +83,8 @@ module InfluxDB2
         else
           raise InfluxError.from_response(response)
         end
+      rescue *InfluxError::HTTP_ERRORS => error
+        raise InfluxError.from_error(error)
       ensure
         http.finish if http.started?
       end

data/lib/influxdb2/client/influx_error.rb

@@ -1,19 +1,33 @@
 module InfluxDB2
   # InfluxError that is raised during HTTP communication.
   class InfluxError < StandardError
+    HTTP_ERRORS = [
+      EOFError,
+      Errno::ECONNREFUSED,
+      Errno::ECONNRESET,
+      Errno::EINVAL,
+      Net::HTTPBadResponse,
+      Net::HTTPHeaderSyntaxError,
+      Net::ProtocolError,
+      Timeout::Error
+    ].freeze
+
     # HTTP status code
     attr_reader :code
     # Reference code unique to the error type
     attr_reader :reference
     # The Retry-After header describes when to try the request again.
     attr_reader :retry_after
+    # original error
+    attr_reader :original
 
-    def initialize(message:, code:, reference:, retry_after:)
+    def initialize(original = nil, message:, code:, reference:, retry_after:)
       super(message)
 
       @code = code
       @reference = reference
       @retry_after = retry_after
+      @original = original
     end
 
     def self.from_response(response)
@@ -27,5 +41,10 @@ module InfluxDB2
       obj = new(message: message, code: '', reference: '', retry_after: '')
       obj
     end
+
+    def self.from_error(error)
+      obj = new(error, message: error.message, code: '', reference: '', retry_after: '')
+      obj
+    end
   end
 end

data/lib/influxdb2/client/worker.rb

@@ -31,6 +31,8 @@ module InfluxDB2
 
       @queue_event.push(true)
 
+      Thread.abort_on_exception = true
+
       @thread_flush = Thread.new do
         until api_client.closed
           sleep @write_options.flush_interval.to_f / 1_000
@@ -94,22 +96,32 @@ module InfluxDB2
 
     def _write(data)
       data.each do |key, points|
-        _write_raw(key, points)
+        _write_raw(key, points, 1, @write_options.retry_interval)
       end
     end
 
-    def _write_raw(key, points)
+    def _write_raw(key, points, attempts, retry_interval)
       if @write_options.jitter_interval > 0
         jitter_delay = (@write_options.jitter_interval.to_f / 1_000) * rand
         sleep jitter_delay
       end
       @api_client.write_raw(points.join("\n"), precision: key.precision, bucket: key.bucket, org: key.org)
     rescue InfluxError => e
-      raise e if e.code.nil? || !(%w[429 503].include? e.code)
+      raise e if attempts > @write_options.max_retries
+      raise e if (e.code.nil? || e.code.to_i < 429) && !_connection_error(e.original)
+
+      timeout = if e.retry_after.empty?
+                  [retry_interval.to_f, @write_options.max_retry_delay.to_f].min / 1_000
+                else
+                  e.retry_after.to_f
+                end
 
-      timeout = e.retry_after.empty? ? @write_options.retry_interval.to_f / 1_000 : e.retry_after.to_f
       sleep timeout
-      _write_raw(key, points)
+      _write_raw(key, points, attempts + 1, retry_interval * @write_options.exponential_base)
+    end
+
+    def _connection_error(error)
+      InfluxError::HTTP_ERRORS.any? { |c| error.instance_of? c }
     end
   end
 end

data/lib/influxdb2/client/write_api.rb

@@ -34,21 +34,32 @@ module InfluxDB2
     # @param [Integer] retry_interval: number of milliseconds to retry unsuccessful write.
     #   The retry interval is used when the InfluxDB server does not specify "Retry-After" header.
     # @param [Integer] jitter_interval: the number of milliseconds to increase the batch flush interval
+    # @param [Integer] max_retries: max number of retries when write fails
+    # @param [Integer] max_retry_delay: maximum delay when retrying write in milliseconds
     #   by a random amount
-    def initialize(write_type: WriteType::SYNCHRONOUS, batch_size: 1_000, flush_interval: 1_000, retry_interval: 1_000,
-                   jitter_interval: 0)
+    # @param [Integer] exponential_base: base for the exponential retry delay, the next delay is computed as
+    #   "exponential_base^(attempts-1) + random(jitter_interval)"
+    def initialize(write_type: WriteType::SYNCHRONOUS, batch_size: 1_000, flush_interval: 1_000, retry_interval: 5_000,
+                   jitter_interval: 0, max_retries: 5, max_retry_delay: 180_000, exponential_base: 5)
       _check_not_negative('batch_size', batch_size)
       _check_not_negative('flush_interval', flush_interval)
       _check_not_negative('retry_interval', retry_interval)
       _check_positive('jitter_interval', jitter_interval)
+      _check_positive('max_retries', jitter_interval)
+      _check_positive('max_retry_delay', jitter_interval)
+      _check_positive('exponential_base', exponential_base)
       @write_type = write_type
       @batch_size = batch_size
       @flush_interval = flush_interval
       @retry_interval = retry_interval
       @jitter_interval = jitter_interval
+      @max_retries = max_retries
+      @max_retry_delay = max_retry_delay
+      @exponential_base = exponential_base
     end
 
-    attr_reader :write_type, :batch_size, :flush_interval, :retry_interval, :jitter_interval
+    attr_reader :write_type, :batch_size, :flush_interval, :retry_interval, :jitter_interval,
+                :max_retries, :max_retry_delay, :exponential_base
 
     def _check_not_negative(key, value)
       raise ArgumentError, "The '#{key}' should be positive or zero, but is: #{value}" if value <= 0

data/test/influxdb/write_api_batching_test.rb

@@ -187,6 +187,65 @@ class WriteApiBatchingTest < MiniTest::Test
                      'h2o_feet,location=coyote_creek level\\ water_level=3.0 3')
   end
 
+  def test_jitter_interval
+    @client.close!
+
+    @client = InfluxDB2::Client.new('http://localhost:9999',
+                                    'my-token',
+                                    bucket: 'my-bucket',
+                                    org: 'my-org',
+                                    precision: InfluxDB2::WritePrecision::NANOSECOND,
+                                    use_ssl: false)
+
+    @write_options = InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING,
+                                                 batch_size: 2, flush_interval: 5_000, jitter_interval: 2_000)
+    @write_client = @client.create_write_api(write_options: @write_options)
+
+    stub_request(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 204)
+
+    request = "h2o_feet,location=coyote_creek water_level=1.0 1\n" \
+               'h2o_feet,location=coyote_creek water_level=2.0 2'
+
+    @write_client.write(data: ['h2o_feet,location=coyote_creek water_level=1.0 1',
+                               'h2o_feet,location=coyote_creek water_level=2.0 2'])
+
+    sleep(0.05)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 0, body: request)
+
+    sleep(2)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: request)
+  end
+end
+
+class WriteApiRetryStrategyTest < MiniTest::Test
+  def setup
+    WebMock.disable_net_connect!
+
+    @write_options = InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING,
+                                                 batch_size: 2, flush_interval: 5_000, retry_interval: 2_000)
+    @client = InfluxDB2::Client.new('http://localhost:9999',
+                                    'my-token',
+                                    bucket: 'my-bucket',
+                                    org: 'my-org',
+                                    precision: InfluxDB2::WritePrecision::NANOSECOND,
+                                    use_ssl: false)
+
+    @write_client = @client.create_write_api(write_options: @write_options)
+  end
+
+  def teardown
+    @client.close!
+
+    assert_equal true, @write_client.closed
+
+    WebMock.reset!
+  end
+
   def test_retry_interval_by_config
     error_body = '{"code":"temporarily unavailable","message":"Token is temporarily over quota. '\
                  'The Retry-After header describes when to try the write again."}'
@@ -211,7 +270,7 @@ class WriteApiBatchingTest < MiniTest::Test
     assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
                      times: 1, body: request)
 
-    sleep(1)
+    sleep(5)
 
     assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
                      times: 2, body: request)
@@ -256,37 +315,234 @@ class WriteApiBatchingTest < MiniTest::Test
                      times: 2, body: request)
   end
 
-  def test_jitter_interval
-    @client.close!
+  def test_max_retries
+    error_body = '{"code":"temporarily unavailable","message":"Server is temporarily unavailable to accept writes. '\
+                 'The Retry-After header describes when to try the write again."}'
 
-    @client = InfluxDB2::Client.new('http://localhost:9999',
-                                    'my-token',
-                                    bucket: 'my-bucket',
-                                    org: 'my-org',
-                                    precision: InfluxDB2::WritePrecision::NANOSECOND,
-                                    use_ssl: false)
+    headers = { 'X-Platform-Error-Code' => 'temporarily unavailable' }
 
-    @write_options = InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING,
-                                                 batch_size: 2, flush_interval: 5_000, jitter_interval: 2_000)
-    @write_client = @client.create_write_api(write_options: @write_options)
+    stub_request(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 429, headers: headers, body: error_body).then # retry
+      .to_return(status: 429, headers: headers, body: error_body).then # retry
+      .to_return(status: 429, headers: headers, body: error_body).then # retry
+      .to_return(status: 429, headers: headers, body: error_body).then # retry
+      .to_return(status: 429, headers: headers, body: error_body) # not called
+
+    point = InfluxDB2::Point.new(name: 'h2o')
+                            .add_tag('location', 'europe')
+                            .add_field('level', 2.0)
+
+    request = 'h2o,location=europe level=2.0'
+
+    write_options = InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING,
+                                                batch_size: 1, retry_interval: 2_000, max_retries: 3,
+                                                max_retry_delay: 5_000, exponential_base: 2)
+
+    error = assert_raises InfluxDB2::InfluxError do
+      @client.create_write_api(write_options: write_options).write(data: point)
+
+      sleep(0.5)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 1, body: request)
+
+      sleep(2)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 2, body: request)
+
+      sleep(4)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 3, body: request)
+
+      sleep(5)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 4, body: request)
+
+      sleep(5)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 4, body: request)
+
+      sleep(5)
+    end
+
+    assert_equal('429', error.code)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 4, body: request)
+  end
+
+  def test_influx_exception
+    error_body = '{"code":"invalid","message":"unable to parse '\
+                 '\'h2o_feet, location=coyote_creek water_level=1.0 1\': missing tag key"}'
+
+    stub_request(:any, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 400, headers: { 'X-Platform-Error-Code' => 'invalid' }, body: error_body)
+
+    write_options = InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING,
+                                                batch_size: 1, retry_interval: 2_000, max_retries: 3,
+                                                max_retry_delay: 5_000, exponential_base: 2)
+
+    error = assert_raises InfluxDB2::InfluxError do
+      @client.create_write_api(write_options: write_options).write(data: 'h2o,location=west value=33i 15')
+
+      sleep(1)
+    end
+
+    assert_equal '400', error.code
+    assert_equal 'invalid', error.reference
+    assert_equal "unable to parse 'h2o_feet, location=coyote_creek water_level=1.0 1': missing tag key", error.message
+  end
+
+  def test_max_retries_by_header
+    error_body = '{"code":"temporarily unavailable","message":"Server is temporarily unavailable to accept writes. '\
+                 'The Retry-After header describes when to try the write again."}'
+
+    headers = { 'X-Platform-Error-Code' => 'temporarily unavailable', 'Retry-After' => '3' }
+
+    stub_request(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 429, headers: headers, body: error_body).then # retry
+      .to_return(status: 429, headers: headers, body: error_body).then # retry
+      .to_return(status: 429, headers: headers, body: error_body).then # retry
+      .to_return(status: 429, headers: headers, body: error_body).then # retry
+      .to_return(status: 429, headers: headers, body: error_body) # not called
+
+    point = InfluxDB2::Point.new(name: 'h2o')
+                            .add_tag('location', 'europe')
+                            .add_field('level', 2.0)
+
+    request = 'h2o,location=europe level=2.0'
+
+    write_options = InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING,
+                                                batch_size: 1, retry_interval: 2_000, max_retries: 3,
+                                                max_retry_delay: 5_000, exponential_base: 2)
+
+    error = assert_raises InfluxDB2::InfluxError do
+      @client.create_write_api(write_options: write_options).write(data: point)
+
+      sleep(0.5)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 1, body: request)
+
+      sleep(3)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 2, body: request)
+
+      sleep(3)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 3, body: request)
+
+      sleep(3)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 4, body: request)
+
+      sleep(3)
+    end
+
+    assert_equal('429', error.code)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 4, body: request)
+  end
+
+  def test_connection_error
+    error_message = 'Failed to open TCP connection to localhost:9999' \
+        '(Connection refused - connect(2) for "localhost" port 9999)'
 
     stub_request(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_raise(Errno::ECONNREFUSED.new(error_message))
+      .to_raise(Errno::ECONNREFUSED.new(error_message))
+      .to_raise(Errno::ECONNREFUSED.new(error_message))
+      .to_raise(Errno::ECONNREFUSED.new(error_message))
+      .to_raise(Errno::ECONNREFUSED.new(error_message))
+
+    point = InfluxDB2::Point.new(name: 'h2o')
+                            .add_tag('location', 'europe')
+                            .add_field('level', 2.0)
+
+    request = 'h2o,location=europe level=2.0'
+
+    write_options = InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING,
+                                                batch_size: 1, retry_interval: 2_000, max_retries: 3,
+                                                max_retry_delay: 5_000, exponential_base: 2)
+
+    error = assert_raises InfluxDB2::InfluxError do
+      @client.create_write_api(write_options: write_options).write(data: point)
+
+      sleep(0.5)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 1, body: request)
+
+      sleep(2)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 2, body: request)
+
+      sleep(4)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 3, body: request)
+
+      sleep(5)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 4, body: request)
+
+      sleep(5)
+
+      assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                       times: 4, body: request)
+
+      sleep(5)
+    end
+
+    assert_equal('Connection refused - ' + error_message, error.message)
+  end
+
+  def test_write_connection_error
+    stub_request(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_raise(Errno::ECONNREFUSED.new(''))
+      .to_raise(Errno::ECONNREFUSED.new(''))
       .to_return(status: 204)
 
-    request = "h2o_feet,location=coyote_creek water_level=1.0 1\n" \
-               'h2o_feet,location=coyote_creek water_level=2.0 2'
+    point = InfluxDB2::Point.new(name: 'h2o')
+                            .add_tag('location', 'europe')
+                            .add_field('level', 2.0)
 
-    @write_client.write(data: ['h2o_feet,location=coyote_creek water_level=1.0 1',
-                               'h2o_feet,location=coyote_creek water_level=2.0 2'])
+    request = 'h2o,location=europe level=2.0'
 
-    sleep(0.05)
+    write_options = InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING,
+                                                batch_size: 1, retry_interval: 2_000, max_retries: 3,
+                                                max_retry_delay: 5_000, exponential_base: 2)
+
+    @client.create_write_api(write_options: write_options).write(data: point)
+
+    sleep(0.5)
 
     assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
-                     times: 0, body: request)
+                     times: 1, body: request)
 
     sleep(2)
 
     assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
-                     times: 1, body: request)
+                     times: 2, body: request)
+
+    sleep(4)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 3, body: request)
+
+    sleep(5)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 3, body: request)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: influxdb-client
 version: !ruby/object:Gem::Version
-  version: 1.7.0.pre.1015
+  version: 1.7.0.pre.1121
 platform: ruby
 authors:
 - Jakub Bednar
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-20 00:00:00.000000000 Z
+date: 2020-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler