fluent-plugin-dedup 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 75f9ab2798dd39e111ecac0a0afd0c1ac964bc05
4
+ data.tar.gz: aa7545d7df05055174952cf8edfcb87e3985658f
5
+ SHA512:
6
+ metadata.gz: f688f6d8f185e583cf6327375b40fc4eff8ca665c5d5cfad31106fa7950240c24a57b0c4bc9836f84411cb2f5c7fd302dbbc692ca25c3a74097b1a8046afdf90
7
+ data.tar.gz: 037a80be99a3156f450aada80bf75da9714ea8d19723cd46bef6207b453bf5ac9d4681a72a7a31857bc0f1847bc558254781c5c1d14b228b49cd88478e16aeba
data/.gitignore ADDED
@@ -0,0 +1,23 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ *.bundle
19
+ *.so
20
+ *.o
21
+ *.a
22
+ mkmf.log
23
+ vendor/bundle/
data/.travis.yml ADDED
@@ -0,0 +1,3 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.0.0
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-dedup.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Atsushi Takayama
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,49 @@
1
+ # fluent-plugin-dedup
2
+
3
+ A fluentd plugin to suppress emission of subsequent logs identical to the first one.
4
+
5
+ [![Build Status](https://travis-ci.org/edvakf/fluent-plugin-dedup.svg?branch=master)](https://travis-ci.org/edvakf/fluent-plugin-dedup)
6
+
7
+ ## Example Usage
8
+
9
+ It's useful when the output of a command executed by `in_exec` only returns the "latest" state of something and you want to send logs only when there is a change.
10
+
11
+ <source>
12
+ type exec
13
+ command latest_state_of_something.rb
14
+ format json
15
+ keys unique_id,foo,bar
16
+ tag some.thing
17
+ run_interval 1s
18
+ </source>
19
+
20
+ <match some.thing>
21
+ type dedup
22
+ key unique_id # required
23
+ file /tmp/dedup_state.json # optional. If set, saves the state to the file.
24
+ </match>
25
+
26
+ <match dedup.some.thing>
27
+ type stdout
28
+ </match>
29
+
30
+ All logs that are processed by this plugin will have tag prefix `dedup`.
31
+
32
+ If the optional `file` parameter is set, it dumps the state during shutdown and loads on start, so that it can still dedup after reload.
33
+
34
+ ## Testing
35
+
36
+ bundle install
37
+ bundle exec rake test
38
+
39
+ ## Installation
40
+
41
+ gem install fluent-plugin-dedup
42
+
43
+ ## Contributing
44
+
45
+ 1. Fork it ( https://github.com/[my-github-username]/fluent-plugin-dedup/fork )
46
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
47
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
48
+ 4. Push to the branch (`git push origin my-new-feature`)
49
+ 5. Create a new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,11 @@
1
+ require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+
4
+ Rake::TestTask.new(:test) do |test|
5
+ test.libs << 'lib' << 'test'
6
+ test.pattern = 'test/**/test_*.rb'
7
+ test.verbose = true
8
+ end
9
+
10
+ task :default => :test
11
+
@@ -0,0 +1,22 @@
1
+ # coding: utf-8
2
+ Gem::Specification.new do |spec|
3
+ spec.name = "fluent-plugin-dedup"
4
+ spec.version = "0.1.0"
5
+ spec.authors = ["edvakf"]
6
+ spec.email = ["taka.atsushi@gmail.com"]
7
+ spec.summary = %q{fluentd plugin for removing duplicate logs}
8
+ spec.description = %q{fluent-plugin-dedup is a fluentd plugin to suppress emission of subsequent logs identical to the first one.}
9
+ spec.homepage = "https://github.com/edvakf/fluent-plugin-dedup"
10
+ spec.license = "MIT"
11
+
12
+ spec.files = `git ls-files -z`.split("\x0")
13
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
14
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
15
+ spec.require_paths = ["lib"]
16
+
17
+ spec.add_development_dependency "bundler", "~> 1.6"
18
+ spec.add_development_dependency "rake"
19
+ spec.add_development_dependency "test-unit", "~> 3.0"
20
+
21
+ spec.add_runtime_dependency "fluentd"
22
+ end
@@ -0,0 +1,67 @@
1
+ require 'json'
2
+
3
+ class Fluent::DedupOutput < Fluent::Output
4
+ Fluent::Plugin.register_output('dedup', self)
5
+
6
+ config_param :key, :string, :default => nil
7
+ config_param :file, :string, :default => nil
8
+
9
+ # Define `log` method for v0.10.42 or earlier
10
+ unless method_defined?(:log)
11
+ define_method("log") { $log }
12
+ end
13
+
14
+ def configure(conf)
15
+ super
16
+ unless conf.include?('key')
17
+ raise Fluent::ConfigError, "config parameter `key` is required"
18
+ end
19
+ @key = conf['key']
20
+ @file = conf['file']
21
+ @states = {}
22
+ end
23
+
24
+ def start
25
+ super
26
+
27
+ if not @file.nil? and File.file?(@file)
28
+ @states = JSON.parse(File.open(@file).read) rescue {}
29
+ end
30
+ end
31
+
32
+ def shutdown
33
+ super
34
+
35
+ save_states
36
+ end
37
+
38
+ def emit(tag, es, chain)
39
+ es.each do |time, record|
40
+ next if dup?(tag, record)
41
+ update_states(tag, record)
42
+ Fluent::Engine.emit("dedup.#{tag}", time, record)
43
+ end
44
+
45
+ chain.next
46
+ end
47
+
48
+ private
49
+ def save_states
50
+ unless @file.nil?
51
+ File.open(@file, 'wb') do |f|
52
+ f.print(@states.to_json)
53
+ end
54
+ end
55
+ end
56
+
57
+ def dup?(tag, record)
58
+ unless record.include?(@key)
59
+ log.warn "record does not have key `#{@key}`, record: #{record.to_json}"
60
+ end
61
+ @states[tag] == record[@key]
62
+ end
63
+
64
+ def update_states(tag, record)
65
+ @states[tag] = record[@key]
66
+ end
67
+ end
@@ -0,0 +1,92 @@
1
+ require 'helper'
2
+
3
+ class DedupOutputTest < Test::Unit::TestCase
4
+ def setup
5
+ Fluent::Test.setup
6
+ end
7
+
8
+ CONFIG = %[
9
+ key unique_id
10
+ ]
11
+
12
+ def create_driver(conf=CONFIG, tag='test', use_v1=false)
13
+ Fluent::Test::OutputTestDriver.new(Fluent::DedupOutput, tag).configure(conf, use_v1)
14
+ end
15
+
16
+ test "`key` must be present" do
17
+ assert_raise(Fluent::ConfigError) {
18
+ d = create_driver('file abc')
19
+ }
20
+ end
21
+
22
+ test "two sequential identical logs are emitted only once" do
23
+ d = create_driver(CONFIG)
24
+ d.run do
25
+ d.emit({'unique_id' => '1'}, Time.now)
26
+ d.emit({'unique_id' => '1'}, Time.now) # dup
27
+ d.emit({'unique_id' => '2'}, Time.now)
28
+ d.emit({'unique_id' => '1'}, Time.now)
29
+ end
30
+
31
+ assert_equal 3, d.emits.length
32
+ assert_equal '1', d.emits[0][2]['unique_id']
33
+ assert_equal '2', d.emits[1][2]['unique_id']
34
+ assert_equal '1', d.emits[2][2]['unique_id']
35
+ assert_equal 'dedup.test', d.emits[0][0], 'sets a tag prefix `dedup`'
36
+ end
37
+
38
+ test "different tags are not treated as identical" do
39
+ d = create_driver(CONFIG)
40
+ d.run do
41
+ d.emit({'unique_id' => '1'}, Time.now)
42
+ d.tag = d.tag + d.tag # set a different tag from the first
43
+ d.emit({'unique_id' => '1'}, Time.now) # not dup
44
+ end
45
+
46
+ assert_equal 2, d.emits.length
47
+ end
48
+
49
+ test "state is not saved on shutdown by default" do
50
+ d1 = create_driver(CONFIG)
51
+ d1.run do
52
+ d1.emit({'unique_id' => '1'}, Time.now)
53
+ end
54
+ d2 = create_driver(CONFIG)
55
+ d2.run do
56
+ d2.emit({'unique_id' => '1'}, Time.now)
57
+ end
58
+
59
+ assert_equal 1, d1.emits.length
60
+ assert_equal 1, d2.emits.length
61
+ end
62
+
63
+ sub_test_case '`file` parameter is present' do
64
+ setup do
65
+ @statefile = File.expand_path('../../../../states.json', __FILE__)
66
+ File.unlink(@statefile) if File.file?(@statefile)
67
+ end
68
+
69
+ teardown do
70
+ File.unlink(@statefile) if File.file?(@statefile)
71
+ end
72
+
73
+ test "state is saved on shutdown if `file` parameter is present" do
74
+ config = %[
75
+ key unique_id
76
+ file #{@statefile}
77
+ ]
78
+
79
+ d1 = create_driver(config)
80
+ d1.run do
81
+ d1.emit({'unique_id' => '1'}, Time.now)
82
+ end
83
+ d2 = create_driver(config)
84
+ d2.run do
85
+ d2.emit({'unique_id' => '1'}, Time.now)
86
+ end
87
+
88
+ assert_equal 1, d1.emits.length
89
+ assert_equal 0, d2.emits.length
90
+ end
91
+ end
92
+ end
data/test/helper.rb ADDED
@@ -0,0 +1,6 @@
1
+ $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
2
+ require 'test/unit'
3
+ require 'fluent/load'
4
+ require 'fluent/test'
5
+
6
+ require 'fluent/plugin/out_dedup'
metadata ADDED
@@ -0,0 +1,113 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-dedup
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - edvakf
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-12-07 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.6'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.6'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: test-unit
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: fluentd
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ description: fluent-plugin-dedup is a fluentd plugin to suppress emission of subsequent
70
+ logs identical to the first one.
71
+ email:
72
+ - taka.atsushi@gmail.com
73
+ executables: []
74
+ extensions: []
75
+ extra_rdoc_files: []
76
+ files:
77
+ - ".gitignore"
78
+ - ".travis.yml"
79
+ - Gemfile
80
+ - LICENSE.txt
81
+ - README.md
82
+ - Rakefile
83
+ - fluent-plugin-dedup.gemspec
84
+ - lib/fluent/plugin/out_dedup.rb
85
+ - test/fluent/plugin/test_out_dedup.rb
86
+ - test/helper.rb
87
+ homepage: https://github.com/edvakf/fluent-plugin-dedup
88
+ licenses:
89
+ - MIT
90
+ metadata: {}
91
+ post_install_message:
92
+ rdoc_options: []
93
+ require_paths:
94
+ - lib
95
+ required_ruby_version: !ruby/object:Gem::Requirement
96
+ requirements:
97
+ - - ">="
98
+ - !ruby/object:Gem::Version
99
+ version: '0'
100
+ required_rubygems_version: !ruby/object:Gem::Requirement
101
+ requirements:
102
+ - - ">="
103
+ - !ruby/object:Gem::Version
104
+ version: '0'
105
+ requirements: []
106
+ rubyforge_project:
107
+ rubygems_version: 2.2.2
108
+ signing_key:
109
+ specification_version: 4
110
+ summary: fluentd plugin for removing duplicate logs
111
+ test_files:
112
+ - test/fluent/plugin/test_out_dedup.rb
113
+ - test/helper.rb