fluent-plugin-influxdb-v2 1.9.0.pre.1169 → 1.9.0.pre.1341

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ab8851d2d37988a3ba75e224b761a25d312e006a1e5955e6fe7f7eae716db31
-  data.tar.gz: f7365f050c1a542188a816f8d5e906e8f53722decc581fcbb41f06cc6ee4dd66
+  metadata.gz: e01f201d71085c9b0a82def242dd80df362cf783cc4bd22cdd280748bb2fea8f
+  data.tar.gz: fd52d6977a87054dd6805f3c0a8e825f11a0d680119445a48d38d436e32dbf11
 SHA512:
-  metadata.gz: 2cda2d96e6fdf2d724cad9a7458f58fe55ac63fb85e09f746aaa2e7df9a26da6246a0681acecb8c1640add41547eed62a08a633cd1885f424b43bfb814427e87
-  data.tar.gz: 3f14015c32b040a8fc7614f0e3d9937f88d1b7ce6089ad942e8279163f1b4070b8b214899a95dd9ae36165d3cda55cc026c9710c8a2123fa584c80e2f37d252f
+  metadata.gz: 8ef6bb9981eaac83e2394892838edf29286672e98adf72a9183f2d088403cc24ee07c01a87726b3647a727ba5eb4e51f139b3814960c3541a9ab2535b0cf00dd
+  data.tar.gz: 52486af50894534c7d05f42c97c736a7d748e7103edc20505164a6ba6822fa13fb25de6e714df4d06f296c7e65b9b9f6c06f01b6cab1a1b58ea47a6cdbf9bd94

data/.circleci/config.yml

@@ -41,9 +41,9 @@ commands:
       - restore_cache:
           name: Restoring Gem Cache
           keys:
-            - &cache-key gem-cache-{{ checksum "fluent-plugin-influxdb-v2.gemspec" }}-<< parameters.ruby-image >>
-            - gem-cache-{{ checksum "fluent-plugin-influxdb-v2.gemspec" }}
-            - gem-cache-
+            - &cache-key gem-cache-v3-{{ checksum "fluent-plugin-influxdb-v2.gemspec" }}-<< parameters.ruby-image >>
+            - gem-cache-v3-{{ checksum "fluent-plugin-influxdb-v2.gemspec" }}
+            - gem-cache-v3-
       - run:
           name: Install dependencies
           command: |
@@ -75,7 +75,7 @@ jobs:
     parameters:
       ruby-image:
         type: string
-        default: &default-ruby-image "circleci/ruby:2.6-stretch"
+        default: &default-ruby-image "cimg/ruby:2.6"
       influxdb-image:
         type: string
         default: &default-influxdb-image "influxdb:latest"
@@ -119,7 +119,10 @@ workflows:
     jobs:
       - tests-ruby:
           name: ruby-2.7
-          ruby-image: "circleci/ruby:2.7-buster"
+          ruby-image: "cimg/ruby:2.7"
+      - tests-ruby:
+          name: ruby-3.0
+          ruby-image: "cimg/ruby:3.0"
       - tests-ruby:
           name: ruby-2.6
       - tests-ruby:
@@ -127,10 +130,10 @@ workflows:
           influxdb-image: "quay.io/influxdb/influxdb:nightly"
       - tests-ruby:
           name: ruby-2.5
-          ruby-image: "circleci/ruby:2.5-stretch"
+          ruby-image: "cimg/ruby:2.5"
       - tests-ruby:
           name: ruby-2.4
-          ruby-image: "circleci/ruby:2.4-stretch"
+          ruby-image: "cimg/ruby:2.4"
       - deploy-preview:
           requires:
             - ruby-2.7

data/.github/PULL_REQUEST_TEMPLATE

@@ -13,4 +13,4 @@ _Briefly describe your proposed changes:_
 - [ ] A test has been added if appropriate
 - [ ] `rake test` completes successfully
 - [ ] Commit messages are in [semantic format](https://seesparkbox.com/foundry/semantic_commit_messages)
-- [ ] Sign [CLA](https://influxdata.com/community/cla/) (if not already signed)
+- [ ] Sign [CLA](https://www.influxdata.com/legal/cla/) (if not already signed)

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 ## 1.9.0 [unreleased]
 
+### Features
+1. [#30](https://github.com/influxdata/influxdb-plugin-fluent/pull/30): Field value for `time_key` can be formatted date (`2021-11-05 09:15:49.487727165 +0000`, `2021-11-05T10:04:43.617216Z`)
+1. [#31](https://github.com/influxdata/influxdb-plugin-fluent/pull/31): Add possibility to use LineProtocol from Fluetd's record.
+
+### Dependencies
+1. [#30](https://github.com/influxdata/influxdb-plugin-fluent/pull/30): Update dependencies:
+    - influxdb-client to 2.1.0
+
+### CI
+1. [#27](https://github.com/influxdata/influxdb-plugin-fluent/pull/27): Switch to next-gen CircleCI's convenience images
+1. [#30](https://github.com/influxdata/influxdb-plugin-fluent/pull/30): Add Ruby 3.0 into CI
+
 ## 1.8.0 [2021-08-20]
 
 ### Features

data/README.md

@@ -45,6 +45,7 @@ Store Fluentd event to InfluxDB 2 database.
 | field_cast_to_float | Turn on/off auto casting Integer value to Float. Helper to avoid mismatch error: 'series type mismatch: already Integer but got Float'. | bool | false |
 | time_precision | The time precision of timestamp. You should specify either second (s), millisecond (ms), microsecond (us), or nanosecond (ns). | String | ns |
 | time_key | A name of the record key that used as a 'timestamp' instead of event timestamp. If a record key doesn't exists or hasn't value then is used event timestamp. | String | nil |
+| line_protocol_key | A name of the record key that contains [LineProtocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/). The value of this key is used for ingesting data into InfluxDB. If a record key doesn't exists or hasn't value then is used event timestamp. | String | nil |
 
 ##### Minimal configuration
 

data/fluent-plugin-influxdb-v2.gemspec

@@ -47,7 +47,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.2.0'
 
   spec.add_runtime_dependency 'fluentd', '~> 1.8'
-  spec.add_runtime_dependency 'influxdb-client', '1.12.0'
+  spec.add_runtime_dependency 'influxdb-client', '2.1.0'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'codecov', '~> 0.1.16'

data/lib/fluent/plugin/out_influxdb2.rb

@@ -73,6 +73,11 @@ class InfluxDBOutput < Fluent::Plugin::Output
   desc 'A name of the record key that used as a \'timestamp\' instead of event timestamp.' \
       'If a record key doesn\'t exists or hasn\'t value then is used event timestamp.'
 
+  config_param :line_protocol_key, :string, default: nil
+  desc 'A name of the record key that contains \'LineProtocol\'.' \
+      'The value of this key is used for ingesting data into InfluxDB.' \
+      'For more info see - https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/.'
+
   config_section :buffer do
     config_set_default :@type, DEFAULT_BUFFER_TYPE
     config_set_default :chunk_keys, ['tag']
@@ -121,28 +126,30 @@ class InfluxDBOutput < Fluent::Plugin::Output
     tag = chunk.metadata.tag
     bucket, measurement = expand_placeholders(chunk)
     chunk.msgpack_each do |time, record|
-      if time.is_a?(Integer)
-        time_formatted = time
+      if @line_protocol_key
+        points << record[@line_protocol_key] if record.include?(@line_protocol_key)
       else
-        nano_seconds = time.sec * 1e9
-        nano_seconds += time.nsec
-        time_formatted = @precision_formatter.call(nano_seconds)
-      end
-      point = InfluxDB2::Point
-              .new(name: measurement)
-      record.each_pair do |k, v|
-        if k.eql?(@time_key)
-          time_formatted = v
-        else
-          _parse_field(k, v, point)
+        time_formatted = _format_time(time)
+        point = InfluxDB2::Point
+                .new(name: measurement)
+        record.each_pair do |k, v|
+          if k.eql?(@time_key)
+            time_formatted = _format_time(v)
+          else
+            _parse_field(k, v, point)
+          end
+          point.add_tag('fluentd', tag) if @tag_fluentd
         end
-        point.add_tag('fluentd', tag) if @tag_fluentd
+        point.time(time_formatted, @precision)
+        points << point
       end
-      point.time(time_formatted, @precision)
-      points << point
     end
-    @write_api.write(data: points, bucket: bucket)
-    log.debug "Written points: #{points}"
+    if points.empty?
+      log.debug "Nothing to write for chunk: #{chunk.metadata}"
+    else
+      @write_api.write(data: points, bucket: bucket)
+      log.debug "Written points: #{points}"
+    end
   end
 
   private
@@ -157,6 +164,27 @@ class InfluxDBOutput < Fluent::Plugin::Output
     [bucket, measurement]
   end
 
+  def _format_time(time)
+    if time.is_a?(Integer)
+      time
+    elsif time.is_a?(Float)
+      time
+    elsif time.is_a?(Fluent::EventTime)
+      nano_seconds = time.sec * 1e9
+      nano_seconds += time.nsec
+      @precision_formatter.call(nano_seconds)
+    elsif time.is_a?(String)
+      begin
+        _format_time(Fluent::EventTime.parse(time))
+      rescue StandardError => e
+        log.debug "Cannot parse timestamp: #{time} due: #{e}"
+        time
+      end
+    else
+      time
+    end
+  end
+
   def _parse_field(key, value, point)
     if @tag_keys.include?(key)
       point.add_tag(key, value)

data/test/.rubocop.yml

@@ -0,0 +1,26 @@
+#
+# The MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+inherit_from: ../.rubocop.yml
+Metrics/ClassLength:
+  Enabled: false
+

data/test/influxdb/plugin/output_test.rb

@@ -381,6 +381,82 @@ class InfluxDBOutputTest < Minitest::Test
                      times: 1, body: 'h2o_tag,version=v.10 level=2i,location="europe" 1544897215000000000')
   end
 
+  def test_time_float
+    stub_request(:any, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 204)
+    driver = create_driver(%(
+      @type influxdb2
+      token my-token
+      bucket my-bucket
+      org my-org
+      tag_keys ["version"]
+      time_key time
+    ))
+    driver.run(default_tag: 'h2o_tag') do
+      emit_documents(driver, 'location' => 'europe', 'level' => 2, 'version' => 'v.10',
+                             'time' => 1_544_897_215_000_000_000.123)
+    end
+    assert_requested(:post, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: 'h2o_tag,version=v.10 level=2i,location="europe" 1544897215000000000')
+  end
+
+  def test_time_key_rfc_3339_format
+    stub_request(:any, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 204)
+    driver = create_driver(%(
+      @type influxdb2
+      token my-token
+      bucket my-bucket
+      org my-org
+      tag_keys ["version"]
+      time_key time
+    ))
+    driver.run(default_tag: 'h2o_tag') do
+      emit_documents(driver, 'location' => 'europe', 'level' => 2, 'version' => 'v.10',
+                             'time' => '2021-11-05T10:04:43.617216Z')
+    end
+    assert_requested(:post, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: 'h2o_tag,version=v.10 level=2i,location="europe" 1636106683617216000')
+  end
+
+  def test_time_key_log_format
+    stub_request(:any, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 204)
+    driver = create_driver(%(
+      @type influxdb2
+      token my-token
+      bucket my-bucket
+      org my-org
+      tag_keys ["version"]
+      time_key time
+    ))
+    driver.run(default_tag: 'h2o_tag') do
+      emit_documents(driver, 'location' => 'europe', 'level' => 2, 'version' => 'v.10',
+                             'time' => '2021-11-05 09:15:49.487727165 +0000')
+    end
+    assert_requested(:post, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: 'h2o_tag,version=v.10 level=2i,location="europe" 1636103749487727104')
+  end
+
+  def test_time_key_not_parseable
+    stub_request(:any, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 204)
+    driver = create_driver(%(
+      @type influxdb2
+      token my-token
+      bucket my-bucket
+      org my-org
+      tag_keys ["version"]
+      time_key time
+    ))
+    driver.run(default_tag: 'h2o_tag') do
+      emit_documents(driver, 'location' => 'europe', 'level' => 2, 'version' => 'v.10',
+                             'time' => '1544897215000000000')
+    end
+    assert_requested(:post, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: 'h2o_tag,version=v.10 level=2i,location="europe" 1544897215000000000')
+  end
+
   def test_field_cast_to_float
     stub_request(:any, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
       .to_return(status: 204)
@@ -510,4 +586,39 @@ class InfluxDBOutputTest < Minitest::Test
     assert_requested(:post, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
                      times: 1, body: 'placeholder_h2o_tag level=2i,location="europe" 1444897215000000000')
   end
+
+  def test_line_protocol
+    stub_request(:any, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 204)
+    driver = create_driver(%(
+      @type influxdb2
+      token my-token
+      bucket my-bucket
+      org my-org
+      time_precision ns
+      line_protocol_key lp_key
+    ))
+    driver.run(default_tag: 'h2o_tag') do
+      emit_documents(driver, 'location' => 'europe', 'level' => 2, 'lp_key' => 'mem,tag=1 field=10 102030')
+    end
+    assert_requested(:post, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns',
+                     times: 1, body: 'mem,tag=1 field=10 102030')
+  end
+
+  def test_line_protocol_not_defined
+    stub_request(:any, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns')
+      .to_return(status: 204)
+    driver = create_driver(%(
+      @type influxdb2
+      token my-token
+      bucket my-bucket
+      org my-org
+      time_precision ns
+      line_protocol_key lp_key
+    ))
+    driver.run(default_tag: 'h2o_tag') do
+      emit_documents(driver)
+    end
+    assert_requested(:post, 'https://localhost:8086/api/v2/write?bucket=my-bucket&org=my-org&precision=ns', times: 0)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-influxdb-v2
 version: !ruby/object:Gem::Version
-  version: 1.9.0.pre.1169
+  version: 1.9.0.pre.1341
 platform: ruby
 authors:
 - Jakub Bednar
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-20 00:00:00.000000000 Z
+date: 2021-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.12.0
+        version: 2.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.12.0
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -201,6 +201,7 @@ files:
 - lib/fluent/plugin/fluent.rb
 - lib/fluent/plugin/out_influxdb2.rb
 - lib/fluent/plugin/version.rb
+- test/.rubocop.yml
 - test/influxdb/plugin/output_test.rb
 - test/influxdb/plugin/version_test.rb
 - test/test_helper.rb
@@ -226,11 +227,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.0.3.1
+rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: InfluxDB 2 output plugin for Fluentd
 test_files:
+- test/.rubocop.yml
 - test/influxdb/plugin/output_test.rb
 - test/influxdb/plugin/version_test.rb
 - test/test_helper.rb