fluent-plugin-belated-record-filter 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b9e6581a21052f43f4595acecaf7770e30728a5d
+  data.tar.gz: 14018391cfc9af8b130b5fb78d48ea2f95a881ea
+SHA512:
+  metadata.gz: b8df9e0af6653ffffb60fd9ba62072a5cb8e8f91233daf2726517fd97a3a4bec66c4a0cc65b2669927773a938ed2fad1db613e9c23621c840c46e1f5a09d9130
+  data.tar.gz: 17a5f10d070bf13f5d400024ddaa5307e88d08abde104ec25220443d5386fb817a99da3a59771df75140385a69b25dfa9488aa82546eaa82852b73469bdda0e0

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+rvm:
+  - 2.3.3
+  - 2.4.0
+  - 2.4.1
+before_install: gem update bundler
+cache: bundler
+
+script:
+  - bundle exec rake test

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 reizist
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,63 @@
+# Fluent::Plugin::BelatedRecordFilter
+
+[![Build Status](https://travis-ci.org/reizist/fluent-plugin-belated-record-filter.svg?branch=master)](https://travis-ci.org/reizist/fluent-plugin-belated-record-filter)
+
+A Filter plugin of fluentd for filtering older records than newest one.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-belated-record-filter'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-belated-record-filter
+
+## Usage
+
+### BelatedRecordFilter
+
+Add belated_record filter.
+
+```
+<filter **>
+  @type belated_record
+  <extract>
+    time_key start_at
+    time_type string
+    time_format %Y-%m-%dT%H:%M:%S %z
+    keep_time_key true
+  </extract>
+</filter>
+```
+
+In the case of incoming this records:
+
+```
+{"message": "hogehoge", "start_at":"2017-08-28T03:45:03+00:00"}
+{"message": "fugafuga", "start_at":"2017-08-28T03:45:05+00:00"}
+{"message": "piyopiyo", "start_at":"2017-08-28T03:45:02+00:00"}
+```
+
+Then output becomes as belows:
+
+```
+{"message": "hogehoge", "start_at":"2017-08-28T03:45:03+00:00"}
+{"message": "fugafuga", "start_at":"2017-08-28T03:45:05+00:00"}
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/reizist/fluent-plugin-belated-record-filter. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+desc 'Run test_unit based test'
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.test_files = Dir["test/**/test_*.rb"].sort
+  t.verbose = true
+  #t.warning = true
+end
+task :default => :test
+
+desc 'Open an irb session preloaded with the gem library'
+task :console do
+  sh 'irb -rubygems -I lib'
+end
+task :c => :console

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fluent/plugin/filter_belated_record"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/example.conf

@@ -0,0 +1,6 @@
+type belated_record
+<extract>
+  time_key time
+  time_type unixtime
+  keep_time_key true
+</extract>

data/fluent-plugin-belated-record-filter.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluent/plugin/filter_belated_record/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-belated-record-filter"
+  spec.version       = Fluent::Plugin::FilterBelatedRecord::VERSION
+  spec.authors       = ["reizist"]
+  spec.email         = ["reizist@gmail.com"]
+
+  spec.summary       = %q{A fluent filter plugin to filter belated records.}
+  spec.description   = spec.summary
+  spec.homepage      = "https://github.com/reizist/fluent-plugin-belated-record-filter"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split("\n")
+  spec.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  spec.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd", [">= 0.14", "< 2"]
+
+  spec.add_development_dependency "bundler", "~> 1.15"
+  spec.add_development_dependency "rake", "~> 12.0.0"
+  spec.add_development_dependency "test-unit", "~> 3.2.5"
+  spec.add_development_dependency "pry"
+end

data/lib/fluent/plugin/filter_belated_record.rb

@@ -0,0 +1,49 @@
+require 'fluent/plugin/filter'
+
+module Fluent
+  module Plugin
+    class BelatedRecordFilter < Filter
+      class ConfigError < StandardError ; end
+      Fluent::Plugin.register_filter('belated_record', self)
+
+      helpers :extract, :storage
+      config_param :time_key, :string, default: nil
+      DEFAULT_STORAGE_TYPE = 'local'
+
+      def initialize
+        super
+      end
+
+      def configure(conf)
+        super
+        @storage = storage_create(usage: 'belated_record', conf: nil, type: DEFAULT_STORAGE_TYPE)
+      end
+
+      def filter(tag, time, record)
+        result = nil
+        last_recorded = fetch_timestamp
+        begin
+          record_time = extract_time_from_record(record).to_i # treats time as unixtime
+          if last_recorded.nil? || last_recorded < record_time
+            set_timestamp(record_time)
+            result = record
+          end
+        rescue => e
+          log.warn "failed to filter records", error: e
+          log.warn_backtrace
+        end
+        result
+      end
+
+      private
+
+      def fetch_timestamp
+        @storage.get(:last_recorded)
+      end
+
+      def set_timestamp(time)
+        @storage.put(:last_recorded, time)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/filter_belated_record/version.rb

@@ -0,0 +1,7 @@
+module Fluent
+  module Plugin
+    module FilterBelatedRecord
+      VERSION = "0.1.1".freeze
+    end
+  end
+end

data/test/plugin/test_filter_belated_record.rb

@@ -0,0 +1,76 @@
+require 'test_helper'
+require 'fluent/test/driver/filter'
+
+class BelatedRecordFilterTest < Test::Unit::TestCase
+  include Fluent
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf = nil)
+    Test::FilterTestDriver.new(Plugin::BelatedRecordFilter).configure(conf)
+  end
+
+  def filter(conf, msgs)
+    driver = create_driver(conf)
+    driver.run do
+      msgs.each do |msg|
+        driver.filter(msg, "time" => Time.now)
+      end
+    end
+    filtered = driver.filtered_as_array
+    filtered
+  end
+
+  sub_test_case 'filter' do
+    test 'execute filter by datetime string' do
+      CONFIG = <<CONF
+        type belated_record
+        <extract>
+          time_key time
+        time_type string
+        time_format %Y-%m-%d %H:%M:%S %z
+        keep_time_key true
+
+        </extract>
+CONF
+      base_time   = Time.now
+      future_time = base_time + 1
+      past_time   = base_time - 1
+
+      d = create_driver(CONFIG)
+
+      d.run do
+        d.emit("time" => base_time.to_s, "message" => "initial message")
+        d.emit("time" => future_time.to_s, "message" => "future message")
+        d.emit("time" => past_time.to_s, "message" => "past message")
+      end
+
+      assert_equal(
+        [{"time" => base_time.to_s, "message" => "initial message"}, {"time" => future_time.to_s, "message" => "future message"}].sort_by{|h| h["time"]},
+        d.filtered.instance_variable_get(:@record_array).sort_by{|h| h["time"]}
+      )
+    end
+
+    test 'execute filter by unixtime' do
+      conf_file   = IO.read("#{File.expand_path('../../../example.conf', __FILE__)}")
+      base_time   = Time.now.to_i
+      future_time = base_time + 1
+      past_time   = base_time - 1
+
+      d = create_driver(conf_file)
+
+      d.run do
+        d.emit("time" => base_time, "message" => "initial message")
+        d.emit("time" => future_time, "message" => "future message")
+        d.emit("time" => past_time, "message" => "past message")
+      end
+
+      assert_equal(
+        [{"time" => base_time, "message" => "initial message"}, {"time" => future_time, "message" => "future message"}].sort_by{|h| h["time"]},
+        d.filtered.instance_variable_get(:@record_array).sort_by{|h| h["time"]}
+      )
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/filter_belated_record'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-belated-record-filter
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- reizist
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-08-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.14'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.14'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.0.0
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.5
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A fluent filter plugin to filter belated records.
+email:
+- reizist@gmail.com
+executables:
+- console
+- setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- example.conf
+- fluent-plugin-belated-record-filter.gemspec
+- lib/fluent/plugin/filter_belated_record.rb
+- lib/fluent/plugin/filter_belated_record/version.rb
+- test/plugin/test_filter_belated_record.rb
+- test/test_helper.rb
+homepage: https://github.com/reizist/fluent-plugin-belated-record-filter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: A fluent filter plugin to filter belated records.
+test_files:
+- test/plugin/test_filter_belated_record.rb
+- test/test_helper.rb