influxdb-client 1.2.0.pre.535 → 1.2.0.pre.562

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76180dd26673089eef78a0e5a0ee439de3fc78ff7287e9fda338ea537f867be4
-  data.tar.gz: 9fc1d0ccac566b186ed8fd4b6909325b399d6596aea2ce3d31902ac48d3a7cb4
+  metadata.gz: a5a1a523648c0252791755455bff70a6d2fddb7e6de8de793aa6e92871e9a76d
+  data.tar.gz: 804b8543ec36402c6fb62d361b83eec7a03c5b4d46e23d225171ba702aad07e2
 SHA512:
-  metadata.gz: 1caa69a1606fd833f732bac74d40362f97c420f3c529b6eda7a0ec9a865fd7de6aee4ab50a26a07d4379754070e25a71e69789580cc2f3cdd7431518c3468823
-  data.tar.gz: 98ade13f63f89d09aaab9d323d5677116374618ed817d955b46fbc77a314e3993f5f82d48b0d1efd2367b22592cd1c95ecc094274cd18837b005ab85131b965c
+  metadata.gz: 060ec40dd8735238025978a3c404ab38f0ecaa90e894a06f22390cc83ff976820c42963e3a155fff9fc14b5b24d9c942ec2b2647d47e1a6aa15daf16ab4898c1
+  data.tar.gz: e6336c095161b4750603d2714dd67383d7c3340442be492d758f32a9ddb073e098f83879943e09fb164c82ac3734bde641750a1fa3281a62987179c5e9626a6d

data/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Features
 1. [#23](https://github.com/influxdata/influxdb-client-ruby/issues/23): Added DeleteApi to delete time series data from InfluxDB.
+1. [#24](https://github.com/influxdata/influxdb-client-ruby/issues/24): Added jitter_interval and retry_interval to WriteApi
 
 ### Bugs
 1. [#22](https://github.com/influxdata/influxdb-client-ruby/pull/22): Fixed batch write

data/README.md

@@ -123,6 +123,8 @@ The writes are processed in batches which are configurable by `WriteOptions`:
 | --- | --- | --- |
 | **batchSize** | the number of data point to collect in batch | 1000 |
 | **flushInterval** | the number of milliseconds before the batch is written | 1000 |
+| **retry_interval** | the number of milliseconds to retry unsuccessful write. The retry interval is used when the InfluxDB server does not specify "Retry-After" header. | 1000 |
+| **jitter_interval** | the number of milliseconds to increase the batch flush interval by a random amount | 0 |
 
 ```ruby
 write_options = InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING,

data/lib/influxdb2/client/influx_error.rb

@@ -5,22 +5,26 @@ module InfluxDB2
     attr_reader :code
     # Reference code unique to the error type
     attr_reader :reference
+    # The Retry-After header describes when to try the request again.
+    attr_reader :retry_after
 
-    def initialize(message:, code:, reference:)
+    def initialize(message:, code:, reference:, retry_after:)
       super(message)
 
       @code = code
       @reference = reference
+      @retry_after = retry_after
     end
 
     def self.from_response(response)
       json = JSON.parse(response.body)
-      obj = new(message: json['message'] || '', code: response.code, reference: json['code'] || '')
+      obj = new(message: json['message'] || '', code: response.code, reference: json['code'] || '',
+                retry_after: response['Retry-After'] || '')
       obj
     end
 
     def self.from_message(message)
-      obj = new(message: message, code: '', reference: '')
+      obj = new(message: message, code: '', reference: '', retry_after: '')
       obj
     end
   end

data/lib/influxdb2/client/worker.rb

@@ -33,24 +33,36 @@ module InfluxDB2
 
       @thread_flush = Thread.new do
         until api_client.closed
-          sleep @write_options.flush_interval / 1_000
-          check_background_queue
+          sleep @write_options.flush_interval.to_f / 1_000
+          _check_background_queue
         end
       end
 
       @thread_size = Thread.new do
         until api_client.closed
-          check_background_queue(size: true) if @queue.length >= @write_options.batch_size
+          _check_background_queue(size: true) if @queue.length >= @write_options.batch_size
           sleep 0.01
         end
       end
     end
 
     def push(payload)
-      @queue.push(payload)
+      if payload.respond_to? :each
+        payload.each do |item|
+          push(item)
+        end
+      else
+        @queue.push(payload)
+      end
+    end
+
+    def flush_all
+      _check_background_queue until @queue.empty?
     end
 
-    def check_background_queue(size: false)
+    private
+
+    def _check_background_queue(size: false)
       @queue_event.pop
       data = {}
       points = 0
@@ -74,20 +86,30 @@ module InfluxDB2
       end
 
       begin
-        write(data) unless data.values.flatten.empty?
+        _write(data) unless data.values.flatten.empty?
       ensure
         @queue_event.push(true)
       end
     end
 
-    def flush_all
-      check_background_queue until @queue.empty?
+    def _write(data)
+      data.each do |key, points|
+        _write_raw(key, points)
+      end
     end
 
-    def write(data)
-      data.each do |key, points|
-        @api_client.write_raw(points.join("\n"), precision: key.precision, bucket: key.bucket, org: key.org)
+    def _write_raw(key, points)
+      if @write_options.jitter_interval.positive?
+        jitter_delay = (@write_options.jitter_interval.to_f / 1_000) * rand
+        sleep jitter_delay
       end
+      @api_client.write_raw(points.join("\n"), precision: key.precision, bucket: key.bucket, org: key.org)
+    rescue InfluxError => e
+      raise e if e.code.nil? || !(%w[429 503].include? e.code)
+
+      timeout = e.retry_after.empty? ? @write_options.retry_interval.to_f / 1_000 : e.retry_after.to_f
+      sleep timeout
+      _write_raw(key, points)
     end
   end
 end

data/lib/influxdb2/client/write_api.rb

@@ -27,17 +27,36 @@ module InfluxDB2
 
   # Creates write api configuration.
   #
-  # @param write_type: methods of write (batching, asynchronous, synchronous)
-  # @param batch_size: the number of data point to collect in batch
-  # @param flush_interval: flush data at least in this interval
   class WriteOptions
-    def initialize(write_type: WriteType::SYNCHRONOUS, batch_size: 1_000, flush_interval: 1_000)
+    # @param [WriteType] write_type: methods of write (batching, synchronous)
+    # @param [Integer] batch_size: the number of data point to collect in batch
+    # @param [Integer] flush_interval: flush data at least in this interval
+    # @param [Integer] retry_interval: number of milliseconds to retry unsuccessful write.
+    #   The retry interval is used when the InfluxDB server does not specify "Retry-After" header.
+    # @param [Integer] jitter_interval: the number of milliseconds to increase the batch flush interval
+    #   by a random amount
+    def initialize(write_type: WriteType::SYNCHRONOUS, batch_size: 1_000, flush_interval: 1_000, retry_interval: 1_000,
+                   jitter_interval: 0)
+      _check_not_negative('batch_size', batch_size)
+      _check_not_negative('flush_interval', flush_interval)
+      _check_not_negative('retry_interval', retry_interval)
+      _check_positive('jitter_interval', jitter_interval)
       @write_type = write_type
       @batch_size = batch_size
       @flush_interval = flush_interval
+      @retry_interval = retry_interval
+      @jitter_interval = jitter_interval
     end
 
-    attr_reader :write_type, :batch_size, :flush_interval
+    attr_reader :write_type, :batch_size, :flush_interval, :retry_interval, :jitter_interval
+
+    def _check_not_negative(key, value)
+      raise ArgumentError, "The '#{key}' should be positive or zero, but is: #{value}" if value <= 0
+    end
+
+    def _check_positive(key, value)
+      raise ArgumentError, "The '#{key}' should be positive number, but is: #{value}" if value < 0
+    end
   end
 
   SYNCHRONOUS = InfluxDB2::WriteOptions.new(write_type: WriteType::SYNCHRONOUS)
@@ -210,9 +229,14 @@ module InfluxDB2
       elsif data.is_a?(Hash)
         _generate_payload(Point.from_hash(data), bucket: bucket, org: org, precision: precision)
       elsif data.respond_to? :map
-        data.map do |item|
+        payloads = data.map do |item|
           _generate_payload(item, bucket: bucket, org: org, precision: precision)
-        end.reject(&:nil?).join("\n".freeze)
+        end.reject(&:nil?)
+        if @write_options.write_type == WriteType::BATCHING
+          payloads
+        else
+          payloads.join("\n".freeze)
+        end
       end
     end
   end

data/test/influxdb/write_api_batching_test.rb

@@ -25,7 +25,7 @@ class WriteApiBatchingTest < MiniTest::Test
     WebMock.disable_net_connect!
 
     @write_options = InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING,
-                                                 batch_size: 2, flush_interval: 5_000)
+                                                 batch_size: 2, flush_interval: 5_000, retry_interval: 2_000)
     @client = InfluxDB2::Client.new('http://localhost:9999',
                                     'my-token',
                                     bucket: 'my-bucket',
@@ -44,6 +44,29 @@ class WriteApiBatchingTest < MiniTest::Test
     WebMock.reset!
   end
 
+  def test_batch_configuration
+    error = assert_raises ArgumentError do
+      InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING, batch_size: 0)
+    end
+    assert_equal "The 'batch_size' should be positive or zero, but is: 0", error.message
+
+    error = assert_raises ArgumentError do
+      InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING, flush_interval: -10)
+    end
+    assert_equal "The 'flush_interval' should be positive or zero, but is: -10", error.message
+
+    error = assert_raises ArgumentError do
+      InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING, retry_interval: 0)
+    end
+    assert_equal "The 'retry_interval' should be positive or zero, but is: 0", error.message
+
+    InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING, jitter_interval: 0)
+    error = assert_raises ArgumentError do
+      InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING, jitter_interval: -10)
+    end
+    assert_equal "The 'jitter_interval' should be positive number, but is: -10", error.message
+  end
+
   def test_batch_size
     stub_request(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
       .to_return(status: 204)
@@ -163,4 +186,107 @@ class WriteApiBatchingTest < MiniTest::Test
                      "h2o_feet,location=coyote_creek level\\ water_level=2.0 2\n" \
                      'h2o_feet,location=coyote_creek level\\ water_level=3.0 3')
   end
+
+  def test_retry_interval_by_config
+    error_body = '{"code":"temporarily unavailable","message":"Token is temporarily over quota. '\
+                 'The Retry-After header describes when to try the write again."}'
+
+    stub_request(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 429, headers: { 'X-Platform-Error-Code' => 'temporarily unavailable' }, body: error_body).then
+      .to_return(status: 204)
+
+    request = "h2o_feet,location=coyote_creek water_level=1.0 1\n" \
+               'h2o_feet,location=coyote_creek water_level=2.0 2'
+
+    @write_client.write(data: ['h2o_feet,location=coyote_creek water_level=1.0 1',
+                               'h2o_feet,location=coyote_creek water_level=2.0 2'])
+
+    sleep(0.5)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: request)
+
+    sleep(1)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: request)
+
+    sleep(1)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 2, body: request)
+  end
+
+  def test_retry_interval_by_header
+    error_body = '{"code":"temporarily unavailable","message":"Server is temporarily unavailable to accept writes. '\
+                 'The Retry-After header describes when to try the write again."}'
+
+    stub_request(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 503, headers: { 'X-Platform-Error-Code' => 'temporarily unavailable', 'Retry-After' => '3' },
+                 body: error_body).then
+      .to_return(status: 204)
+
+    request = "h2o_feet,location=coyote_creek water_level=1.0 1\n" \
+               'h2o_feet,location=coyote_creek water_level=2.0 2'
+
+    @write_client.write(data: ['h2o_feet,location=coyote_creek water_level=1.0 1',
+                               InfluxDB2::Point.new(name: 'h2o_feet')
+                                   .add_tag('location', 'coyote_creek')
+                                   .add_field('water_level', 2.0)
+                                   .time(2, InfluxDB2::WritePrecision::NANOSECOND)])
+
+    sleep(0.5)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: request)
+
+    sleep(1)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: request)
+
+    sleep(1)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: request)
+
+    sleep(1)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 2, body: request)
+  end
+
+  def test_jitter_interval
+    @client.close!
+
+    @client = InfluxDB2::Client.new('http://localhost:9999',
+                                    'my-token',
+                                    bucket: 'my-bucket',
+                                    org: 'my-org',
+                                    precision: InfluxDB2::WritePrecision::NANOSECOND,
+                                    use_ssl: false)
+
+    @write_options = InfluxDB2::WriteOptions.new(write_type: InfluxDB2::WriteType::BATCHING,
+                                                 batch_size: 2, flush_interval: 5_000, jitter_interval: 2_000)
+    @write_client = @client.create_write_api(write_options: @write_options)
+
+    stub_request(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 204)
+
+    request = "h2o_feet,location=coyote_creek water_level=1.0 1\n" \
+               'h2o_feet,location=coyote_creek water_level=2.0 2'
+
+    @write_client.write(data: ['h2o_feet,location=coyote_creek water_level=1.0 1',
+                               'h2o_feet,location=coyote_creek water_level=2.0 2'])
+
+    sleep(0.1)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 0, body: request)
+
+    sleep(2)
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: request)
+  end
 end

data/test/influxdb/write_api_test.rb

@@ -129,6 +129,25 @@ class WriteApiTest < MiniTest::Test
                      times: 1, body: expected)
   end
 
+  def test_array_of_array
+    stub_request(:any, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 204)
+    client = InfluxDB2::Client.new('http://localhost:9999', 'my-token',
+                                   bucket: 'my-bucket',
+                                   org: 'my-org',
+                                   precision: InfluxDB2::WritePrecision::NANOSECOND,
+                                   use_ssl: false)
+
+    client.create_write_api.write(data: ['h2o,location=west value=33i 15', ['h2o,location=west value=34i 16',
+                                                                            'h2o,location=west value=35i 17']])
+
+    expected = "h2o,location=west value=33i 15\nh2o,location=west value=34i 16"\
+               "\nh2o,location=west value=35i 17"
+
+    assert_requested(:post, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: expected)
+  end
+
   def test_authorization_header
     stub_request(:any, 'http://localhost:9999/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
       .to_return(status: 204)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: influxdb-client
 version: !ruby/object:Gem::Version
-  version: 1.2.0.pre.535
+  version: 1.2.0.pre.562
 platform: ruby
 authors:
 - Jakub Bednar
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-25 00:00:00.000000000 Z
+date: 2020-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler