fluent-plugin-influxdb-deduplication 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: aac38463aad973daa8abf8f04c51c51e3654be687c70fbfa4d2ded133075fe0d
+  data.tar.gz: 5fff6893e6240e861740968e45754665f50484e44ac08ea37f93d472381b4c6f
+SHA512:
+  metadata.gz: 91b7f1f3c930937daeec8811bddb31f3e5e5718de6842fa57d7243534455cf8570835035017749ee22f1fea2ecef7c228deb08e8a47a4a2a74b5bfb1e6be6821
+  data.tar.gz: 5efdea5ff2243289eb2b73420f6bedae059d0b5157f2d6711f16f781f48cf8935dfd12c9a54c36f937c93ac8f4f724e23db4b1078a936864d4ea9221d6ae7921

data/ChangeLog

@@ -0,0 +1,3 @@
+Release 0.1.0 - 2020/02/23
+
+  * First release

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,2 @@
+# Deduplication records for [Fluentd](http://fluentd.org) with a influxdb output plugin
+

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler'
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/fluent-plugin-influxdb-deduplication.gemspec

@@ -0,0 +1,21 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name = "fluent-plugin-influxdb-deduplication"
+  gem.description = "Filter plugin for deduplicating records for influxdb"
+  gem.summary = gem.description
+  gem.version = File.read("VERSION").strip
+  gem.authors = ["Marc Adams"]
+  gem.email = "marc.adams.ge@gmail.com"
+  #gem.platform    = Gem::Platform::RUBY
+  gem.license = 'MIT'
+  gem.files = `git ls-files`.split("\n")
+  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", [">= 1.0", "< 2"]
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency("test-unit", ["~> 3.1.4"])
+end

data/lib/fluent/plugin/filter_influxdb_deduplication.rb

@@ -0,0 +1,75 @@
+require 'fluent/plugin/filter'
+
+module Fluent
+  class Plugin::InfluxdbDeduplicationFilter < Plugin::Filter
+    Fluent::Plugin.register_filter('influxdb_deduplication', self)
+
+    config_param :time_key, :string, default: nil,
+                 desc: <<-DESC
+The output time key to use.
+    DESC
+
+    config_param :out_of_order, :string, default: nil,
+                 desc: <<-DESC
+If not nil, the field takes the value true if the record arrives in order and false otherwise
+    DESC
+
+    def configure(conf)
+      super
+
+      unless @time_key
+        raise Fluent::ConfigError, "a time key must be set"
+      end
+    end
+
+    def start
+      super
+
+      @last_timestamp = 0
+      @sequence = 0
+    end
+
+    def filter(tag, time, record)
+      if time.is_a?(Integer)
+        input_time = Fluent::EventTime.new(time)
+      elsif time.is_a?(Fluent::EventTime)
+        input_time = time
+      else
+        @log.error("unreadable time")
+        return nil
+      end
+
+      nano_time = input_time.sec * 1000000000
+
+      if input_time.sec < @last_timestamp
+        @log.debug("out of sequence timestamp")
+        if @out_of_order
+          record[@out_of_order] = true
+          record[@time_key] = nano_time
+        else
+          @log.debug("out of order record dropped")
+          return nil
+        end
+      elsif input_time.sec == @last_timestamp && @sequence < 999999999
+        @sequence = @sequence + 1
+        record[@time_key] = nano_time + @sequence
+        if @out_of_order
+          record[@out_of_order] = false
+        end
+      elsif input_time.sec == @last_timestamp && @sequence == 999999999
+        @log.error("received more then 999999999 records in a second")
+        return nil
+      else
+        @sequence = 0
+        @last_timestamp = input_time.sec
+        record[@time_key] = nano_time
+        if @out_of_order
+          record[@out_of_order] = false
+        end
+      end
+
+      record
+    end
+
+  end
+end

data/test/test_filter_influxdb_deduplication.rb

@@ -0,0 +1,105 @@
+# coding: utf-8
+require 'fluent/test'
+require 'fluent/test/driver/filter'
+require 'fluent/plugin/filter_influxdb_deduplication'
+require 'test/unit'
+
+class InfluxdbDeduplicationFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @tag = 'test.tag'
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::InfluxdbDeduplicationFilter).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver %[
+      time_key my_time_key
+    ]
+
+    time_key = d.instance.instance_variable_get(:@time_key)
+    assert time_key == "my_time_key"
+
+    assert_raises Fluent::ConfigError do
+      create_driver ""
+    end
+  end
+
+  def test_in_sequence
+    d = create_driver %[
+      time_key time_key
+    ]
+
+    time0 = Fluent::EventTime.new(1613910640)
+    time1 = Fluent::EventTime.new(1613910643)
+
+    d.run(default_tag: @tag) do
+      d.feed(time0, { "k1" => 0 })
+      d.feed(time0, { "k1" => 1 })
+      d.feed(time0, { "k1" => 2 })
+      d.feed(time1, { "k1" => 3 })
+      d.feed(time1, { "k1" => 4 })
+    end
+
+    assert_equal [
+                   [time0, { "k1" => 0, "time_key" => 1613910640000000000 }],
+                   [time0, { "k1" => 1, "time_key" => 1613910640000000001 }],
+                   [time0, { "k1" => 2, "time_key" => 1613910640000000002 }],
+                   [time1, { "k1" => 3, "time_key" => 1613910643000000000 }],
+                   [time1, { "k1" => 4, "time_key" => 1613910643000000001 }]
+                 ], d.filtered
+  end
+
+  def test_out_of_sequence_dropped
+    d = create_driver %[
+      time_key time_key
+    ]
+
+    time0 = Fluent::EventTime.new(1613910640)
+    time1 = Fluent::EventTime.new(1613910643)
+
+    d.run(default_tag: @tag) do
+      d.feed(time0, { "k1" => 0 })
+      d.feed(time1, { "k1" => 1 })
+      d.feed(time0, { "k1" => 2 })
+      d.feed(time1, { "k1" => 3 })
+      d.feed(time1, { "k1" => 4 })
+    end
+
+    assert_equal [
+                   [time0, { "k1" => 0, "time_key" => 1613910640000000000 }],
+                   [time1, { "k1" => 1, "time_key" => 1613910643000000000 }],
+                   [time1, { "k1" => 3, "time_key" => 1613910643000000001 }],
+                   [time1, { "k1" => 4, "time_key" => 1613910643000000002 }]
+                 ], d.filtered
+  end
+
+  def test_out_of_sequence_field
+    d = create_driver %[
+      time_key time_key
+      out_of_order ooo_field
+    ]
+
+    time0 = Fluent::EventTime.new(1613910640)
+    time1 = Fluent::EventTime.new(1613910643)
+
+    d.run(default_tag: @tag) do
+      d.feed(time0, { "k1" => 0 })
+      d.feed(time1, { "k1" => 1 })
+      d.feed(time0, { "k1" => 2 })
+      d.feed(time1, { "k1" => 3 })
+      d.feed(time1, { "k1" => 4 })
+    end
+
+    assert_equal [
+                   [time0, { "k1" => 0, "time_key" => 1613910640000000000, "ooo_field" => false }],
+                   [time1, { "k1" => 1, "time_key" => 1613910643000000000, "ooo_field" => false }],
+                   [time0, { "k1" => 2, "time_key" => 1613910640000000000, "ooo_field" => true }],
+                   [time1, { "k1" => 3, "time_key" => 1613910643000000001, "ooo_field" => false }],
+                   [time1, { "k1" => 4, "time_key" => 1613910643000000002, "ooo_field" => false }]
+                 ], d.filtered
+  end
+
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-influxdb-deduplication
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Marc Adams
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-02-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.4
+description: Filter plugin for deduplicating records for influxdb
+email: marc.adams.ge@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ChangeLog
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-influxdb-deduplication.gemspec
+- lib/fluent/plugin/filter_influxdb_deduplication.rb
+- test/test_filter_influxdb_deduplication.rb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Filter plugin for deduplicating records for influxdb
+test_files:
+- test/test_filter_influxdb_deduplication.rb