fluent-plugin-comparison-filter 0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a16b1f2cc160691386727f66572c81575cc90c73
+  data.tar.gz: ddd9853bc3b0d7cbb5bb785813f6ab23427bb1e5
+SHA512:
+  metadata.gz: 78feac7f5eb657b3e00a64b7f7670763b8e8e1c1b3dba49d1a4f1d6f675d40d8aa54c24750f0d7e746278b445df0361713ec979c3677754c2a370393f834794a
+  data.tar.gz: 2471defb2f6bb834b3a075416d63053f81f63d9a51c3d1fabb3c013c7a206d8e8fa7dee380941ce52a4e4cadd2d334885c3930eebe898da4bfccd59b617e3ef2

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+rvm:
+  - 2.3.3
+  - 2.4.0
+  - 2.4.1
+before_install: gem update bundler
+cache: bundler
+
+script:
+  - bundle exec rake test

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 reizist
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,105 @@
+# Fluent::Plugin::ComparisonFilter
+
+[![Build Status](https://travis-ci.org/reizist/fluent-plugin-comparison-filter.svg?branch=master)](https://travis-ci.org/reizist/fluent-plugin-comparison-filter)
+
+A Filter plugin of fluentd for filtering by comparing records between newest one and the others.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-comparison-filter'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-comparison-filter
+
+## Configuration
+
+### Comparison section
+| name            | type          | required?    | default | description                                       |
+| :-------------  | :------------ | :----------- | :-------| :-----------------------                          |
+| column_key      | string        | yes          | nil     | the column for using comparison
+| column_key_type | enum          | yes          | time    | time or numeric                   
+| time_type       | enum          | no           | float   | type of time type, float or string or unixtime
+| time_format     | string        | no           | nil     | 
+| localtime       | bool          | no           | true    | UTC if :localtime is false and :timezone is nil                     
+| utc             | bool          | no           | false   |
+| timezone        | string        | no           | nli     |
+
+## Usage
+
+### ComparisonFilter
+
+Add comparison filter.
+
+```
+<filter **>
+  @type comparison
+  <comparison>
+    column_key start_at
+    columm_key_type time
+    time_type string
+    time_format %Y-%m-%dT%H:%M:%S %z
+  </comparison>
+</filter>
+```
+
+In the case of incoming this records:
+
+```
+{"message": "hogehoge", "start_at":"2017-08-28T03:45:03+00:00"}
+{"message": "fugafuga", "start_at":"2017-08-28T03:45:05+00:00"}
+{"message": "piyopiyo", "start_at":"2017-08-28T03:45:02+00:00"}
+```
+
+Then output becomes as belows:
+
+```
+{"message": "hogehoge", "start_at":"2017-08-28T03:45:03+00:00"}
+{"message": "fugafuga", "start_at":"2017-08-28T03:45:05+00:00"}
+```
+
+In the other case like this,
+
+```
+<filter **>
+  @type comparison
+  <comparison>
+    column_key id 
+    columm_key_type numeric
+  </comparison>
+</filter>
+```
+
+input:
+
+```
+{"message": "hogehoge", "id": 1 }
+{"message": "fugafuga", "id": 5 }
+{"message": "piyopiyo", "id": 3 }
+{"message": "mogemoge", "id": 8 }
+```
+
+output:
+
+```
+{"message": "hogehoge", "id": 1 }
+{"message": "fugafuga", "id": 5 }
+{"message": "mogemoge", "id": 8 }
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/reizist/fluent-plugin-comparison-filter. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+desc 'Run test_unit based test'
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.test_files = Dir["test/**/test_*.rb"].sort
+  t.verbose = true
+  #t.warning = true
+end
+task :default => :test
+
+desc 'Open an irb session preloaded with the gem library'
+task :console do
+  sh 'irb -rubygems -I lib'
+end
+task :c => :console

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fluent/plugin/filter_comparison"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/example.conf

@@ -0,0 +1,7 @@
+type comparison
+<comparison>
+  column_key time
+  column_key_type time
+  time_key time
+  time_type unixtime
+</comparison>

data/fluent-plugin-comparison-filter.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluent/plugin/filter_comparison/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-comparison-filter"
+  spec.version       = Fluent::Plugin::FilterComparison::VERSION
+  spec.authors       = ["reizist"]
+  spec.email         = ["reizist@gmail.com"]
+
+  spec.summary       = %q{A fluent filter plugin to filter by comparing records.}
+  spec.description   = spec.summary
+  spec.homepage      = "https://github.com/reizist/fluent-plugin-comparison-filter"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split("\n")
+  spec.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  spec.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd", [">= 0.14", "< 2"]
+
+  spec.add_development_dependency "bundler", "~> 1.15"
+  spec.add_development_dependency "rake", "~> 12.0.0"
+  spec.add_development_dependency "test-unit", "~> 3.2.5"
+  spec.add_development_dependency "pry"
+end

data/lib/fluent/plugin/filter_comparison.rb

@@ -0,0 +1,93 @@
+require 'fluent/plugin/filter'
+
+module Fluent
+  module Plugin
+    class ComparisonFilter < Filter
+      class ConfigError < StandardError ; end
+      Fluent::Plugin.register_filter('comparison', self)
+      helpers :storage
+
+      config_section :comparison, required: true, multi: false, param_name: :comparison_config do
+        config_param :column_key, :string, default: nil
+        config_param :column_key_type, :enum, list: [:numeric, :time], default: :time
+
+        config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
+
+        Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
+          config_param name, type, opts
+        end
+      end
+
+      DEFAULT_STORAGE_TYPE = 'local'
+
+      def initialize
+        super
+        @column_key = nil
+        @column_key_type = nil
+        @time_type = nil
+      end
+
+      def configure(conf)
+        super
+        @column_key = @comparison_config.column_key
+        @column_key_type = @comparison_config.column_key_type
+        @time_type = @comparison_config.time_type
+
+        if compare_by_time?
+          @time_parser = case @time_type
+            when :float then Fluent::NumericTimeParser.new(:float)
+            when :unixtime then Fluent::NumericTimeParser.new(:unixtime)
+            else
+              localtime = @comparison_config.localtime && !@comparison_config.utc
+              Fluent::TimeParser.new(@comparison_config.time_format, localtime, @comparison_config.timezone)
+          end
+        else
+        end
+
+        @storage = storage_create(usage: 'comparison', conf: nil, type: DEFAULT_STORAGE_TYPE)
+      end
+
+      def filter(tag, time, record)
+        result = nil
+        last_recorded = fetch_comparison_key
+        begin
+          comparison_key = extract_comparison_column_from_record(record)
+          if last_recorded.nil? || last_recorded < comparison_key
+            set_comparison_key(comparison_key)
+            result = record
+          end
+        rescue => e
+          log.warn "failed to filter records", error: e
+          log.warn_backtrace
+        end
+        result
+      end
+
+      private
+
+      def fetch_comparison_key
+        @storage.get(:last_recorded)
+      end
+
+      def set_comparison_key(key)
+        @storage.put(:last_recorded, key)
+      end
+
+      def compare_by_time?
+        @comparison_config.column_key_type == :time
+      end
+
+      def extract_comparison_column_from_record(record)
+
+        if compare_by_time?
+          if @column_key && record.has_key?(@column_key)
+            return @time_parser.call(record[@column_key]).to_i # treats time as unixtime
+          end
+          nil
+        else
+          record[@column_key]
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/filter_comparison/version.rb

@@ -0,0 +1,7 @@
+module Fluent
+  module Plugin
+    module FilterComparison
+      VERSION = "0.2".freeze
+    end
+  end
+end

data/test/plugin/test_filter_comparison.rb

@@ -0,0 +1,98 @@
+require 'test_helper'
+require 'fluent/test/driver/filter'
+
+class ComparisonFilterTest < Test::Unit::TestCase
+  include Fluent
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf = nil)
+    Test::FilterTestDriver.new(Plugin::ComparisonFilter).configure(conf)
+  end
+
+  def filter(conf, msgs)
+    driver = create_driver(conf)
+    driver.run do
+      msgs.each do |msg|
+        driver.filter(msg, "time" => Time.now)
+      end
+    end
+    filtered = driver.filtered_as_array
+    filtered
+  end
+
+  sub_test_case 'filter' do
+    test 'execute filter by datetime string' do
+      CONFIG = <<CONF
+        type comparison
+        <comparison>
+          column_key time
+          column_key_type time
+          time_type string
+          time_format %Y-%m-%d %H:%M:%S %z
+        </comparison>
+CONF
+      base_time   = Time.now
+      future_time = base_time + 1
+      past_time   = base_time - 1
+
+      d = create_driver(CONFIG)
+
+      d.run do
+        d.emit("time" => base_time.to_s, "message" => "initial message")
+        d.emit("time" => future_time.to_s, "message" => "future message")
+        d.emit("time" => past_time.to_s, "message" => "past message")
+      end
+
+      assert_equal(
+        [{"time" => base_time.to_s, "message" => "initial message"}, {"time" => future_time.to_s, "message" => "future message"}].sort_by{|h| h["time"]},
+        d.filtered.instance_variable_get(:@record_array).sort_by{|h| h["time"]}
+      )
+    end
+
+    test 'execute filter by unixtime' do
+      conf_file   = IO.read("#{File.expand_path('../../../example.conf', __FILE__)}")
+      base_time   = Time.now.to_i
+      future_time = base_time + 1
+      past_time   = base_time - 1
+
+      d = create_driver(conf_file)
+
+      d.run do
+        d.emit("time" => base_time, "message" => "initial message")
+        d.emit("time" => future_time, "message" => "future message")
+        d.emit("time" => past_time, "message" => "past message")
+      end
+
+      assert_equal(
+        [{"time" => base_time, "message" => "initial message"}, {"time" => future_time, "message" => "future message"}].sort_by{|h| h["time"]},
+        d.filtered.instance_variable_get(:@record_array).sort_by{|h| h["time"]}
+      )
+    end
+
+    test 'execute filter by numeric' do
+      CONFIG = <<CONF
+        type comparison 
+        <comparison>
+          column_key id
+          column_key_type numeric
+        </comparison>
+CONF
+
+      d = create_driver(CONFIG)
+
+      d.run do
+        d.emit("id" => 1, "message" => "initial message")
+        d.emit("id" => 5, "message" => "future message")
+        d.emit("id" => 3, "message" => "past message")
+      end
+
+      assert_equal(
+        [{"id" => 1, "message" => "initial message"}, {"id" => 5, "message" => "future message"}].sort_by{|h| h["id"]},
+        d.filtered.instance_variable_get(:@record_array).sort_by{|h| h["id"]}
+      )
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/filter_comparison'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-comparison-filter
+version: !ruby/object:Gem::Version
+  version: '0.2'
+platform: ruby
+authors:
+- reizist
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-09-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.14'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.14'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.0.0
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.5
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A fluent filter plugin to filter by comparing records.
+email:
+- reizist@gmail.com
+executables:
+- console
+- setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- example.conf
+- fluent-plugin-comparison-filter.gemspec
+- lib/fluent/plugin/filter_comparison.rb
+- lib/fluent/plugin/filter_comparison/version.rb
+- test/plugin/test_filter_comparison.rb
+- test/test_helper.rb
+homepage: https://github.com/reizist/fluent-plugin-comparison-filter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: A fluent filter plugin to filter by comparing records.
+test_files:
+- test/plugin/test_filter_comparison.rb
+- test/test_helper.rb