fluent-plugin-influxdb-deduplication 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aac38463aad973daa8abf8f04c51c51e3654be687c70fbfa4d2ded133075fe0d
-  data.tar.gz: 5fff6893e6240e861740968e45754665f50484e44ac08ea37f93d472381b4c6f
+  metadata.gz: e2c3760445648631a72cd9fa02f5f18ec0391b3e5e9172f49448a62dc5cc895e
+  data.tar.gz: c354954939bc6e3209a71e173e34e796a0073bf345276701ad0aba386dd3a662
 SHA512:
-  metadata.gz: 91b7f1f3c930937daeec8811bddb31f3e5e5718de6842fa57d7243534455cf8570835035017749ee22f1fea2ecef7c228deb08e8a47a4a2a74b5bfb1e6be6821
-  data.tar.gz: 5efdea5ff2243289eb2b73420f6bedae059d0b5157f2d6711f16f781f48cf8935dfd12c9a54c36f937c93ac8f4f724e23db4b1078a936864d4ea9221d6ae7921
+  metadata.gz: e7e54a5bc97b6ba35b29582fb1ac4487fa486c2503c3b0f174591dd522c9ab124678350fe87c5fbecedd3c17da5ee85ed9231a5c04fb365a7293b4eec151ce11
+  data.tar.gz: 0d0e5c0853ffdf0f661f23102fe6e6ef5ce71087d3ffc1897353df4ed03fcb96def70a9040812d030ea616c6034bd590c5d23f56e07fd368f5404ec5366d4c2a

data/.github/workflows/test.yml

@@ -0,0 +1,14 @@
+name: test
+on:
+  - push
+  - pull_request
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.6
+          bundler-cache: true
+      - run: bundle exec rake test

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Marc Adams
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -1,2 +1,86 @@
-# Deduplication records for [Fluentd](http://fluentd.org) with a influxdb output plugin
+# [Fluentd](https://www.fluentd.org/) filter plugin to deduplicate records for InfluxDB
 
+A filter plugin that implements the deduplication techniques described in the [InfluxDB doc](https://docs.influxdata.com/influxdb/v2.0/write-data/best-practices/duplicate-points/).
+
+
+## Installation
+
+Using RubyGems:
+
+```
+fluent-gem install fluent-plugin-influxdb-deduplication
+```
+
+
+## Configuration
+
+### Deduplicate by incrementing the timestamp
+
+The filter plugin reads the fluentd record event time with a precision to the second, and stores it in the `time_key` field.
+Any following record with the same timestamp has a `time_key` incremented by 1 nanosecond.
+
+    <filter pattern>
+      @type influxdb_deduplication
+    
+      # field to store the deduplicated timestamp
+      time_key my_key_field
+    </filter>
+
+For example, the following input records:
+
+    1613910640 { "k1" => 0, "k2" => "value0" }
+    1613910640 { "k1" => 1, "k2" => "value1" }
+    1613910640 { "k1" => 2, "k2" => "value2" }
+    1613910641 { "k1" => 3, "k3" => "value3" }
+
+Would create on output:
+
+    1613910640 { "k1" => 0, "k2" => "value0", "my_key_field" => 1613910640000000000 }
+    1613910640 { "k1" => 1, "k2" => "value1", "my_key_field" => 1613910640000000001 }
+    1613910640 { "k1" => 2, "k2" => "value2", "my_key_field" => 1613910640000000002 }
+    1613910641 { "k1" => 3, "k3" => "value3", "my_key_field" => 1613910643000000000 }
+
+The time key field can then be passed as is to the [fluent-plugin-influxdb-v2](https://github.com/influxdata/influxdb-plugin-fluent).
+Example configuration on nginx logs:
+
+    <filter nginx.access>
+      @type influxdb_deduplication
+    
+      # field to store the deduplicated timestamp
+      time_key my_key_field
+    </filter>
+
+    <match nginx.access>
+        @type influxdb2
+
+        # setup the access to your InfluxDB v2 instance
+        url             https://localhost:8086
+        token           my-token
+        bucket          my-bucket
+        org             my-org
+
+        # the influxdb2 timekey must be set to the same value as the influxdb_deduplication time_key
+        time_key my_key_field
+
+        # the timestamp precision must be set to ns
+        time_precision ns
+
+        tag_keys ["request_method", "status"]
+        field_keys ["remote_addr", "request_uri"]
+    </match>
+
+The data can then be queried as a table and viewed in [Grafana](https://grafana.com/) for example with the flux query:
+
+    from(bucket: "my-bucket")
+      |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
+      |> pivot(
+        rowKey:["_time"],
+        columnKey: ["_field"],
+        valueColumn: "_value"
+      )
+      |> keep(columns: ["_time", "request_method", "status", "remote_addr", "request_uri"])
+
+
+### Deduplicate by adding a sequence tag
+
+TODO

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/fluent-plugin-influxdb-deduplication.gemspec

@@ -4,6 +4,7 @@ $:.push File.expand_path('../lib', __FILE__)
 Gem::Specification.new do |gem|
   gem.name = "fluent-plugin-influxdb-deduplication"
   gem.description = "Filter plugin for deduplicating records for influxdb"
+  gem.homepage = "https://github.com/marcadamsge/fluent-plugin-influxdb-deduplication"
   gem.summary = gem.description
   gem.version = File.read("VERSION").strip
   gem.authors = ["Marc Adams"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-influxdb-deduplication
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Marc Adams
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-23 00:00:00.000000000 Z
+date: 2021-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -64,15 +64,17 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ChangeLog
 - Gemfile
+- LICENSE
 - README.md
 - Rakefile
 - VERSION
 - fluent-plugin-influxdb-deduplication.gemspec
 - lib/fluent/plugin/filter_influxdb_deduplication.rb
 - test/test_filter_influxdb_deduplication.rb
-homepage: 
+homepage: https://github.com/marcadamsge/fluent-plugin-influxdb-deduplication
 licenses:
 - MIT
 metadata: {}