fluent-plugin-dedup 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/fluent-plugin-dedup.gemspec +2 -1
- data/lib/fluent/plugin/out_dedup.rb +31 -9
- data/test/fluent/plugin/test_out_dedup.rb +21 -0
- metadata +15 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 504ed27a9800ed32acf1e4c2872520394d935ee4
|
4
|
+
data.tar.gz: 3c48385579ce83f21be03e5d6052f3a1bdf59b55
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 35f86e880c21e1b40015d95c1684e2d9117e324fda2781a49fc233dd0dbed6b02894792e049226053a5cabf814a589207ef31a7353ccd5aca34d3bde4def4b9e
|
7
|
+
data.tar.gz: 95726e9ef89719c7cc382748b03842e0eee6872ea6e8693486de26f3202bd173ae35bcac879a8fcb0e89ecc2a940cbcc7df7d45284ea9850fb94a77cb18f17a5
|
data/fluent-plugin-dedup.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "fluent-plugin-dedup"
|
4
|
-
spec.version = "0.
|
4
|
+
spec.version = "0.2.0"
|
5
5
|
spec.authors = ["edvakf"]
|
6
6
|
spec.email = ["taka.atsushi@gmail.com"]
|
7
7
|
spec.summary = %q{fluentd plugin for removing duplicate logs}
|
@@ -19,4 +19,5 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.add_development_dependency "test-unit", "~> 3.0"
|
20
20
|
|
21
21
|
spec.add_runtime_dependency "fluentd"
|
22
|
+
spec.add_runtime_dependency "lru_redux"
|
22
23
|
end
|
@@ -1,10 +1,12 @@
|
|
1
1
|
require 'json'
|
2
|
+
require 'lru_redux'
|
2
3
|
|
3
4
|
class Fluent::DedupOutput < Fluent::Output
|
4
5
|
Fluent::Plugin.register_output('dedup', self)
|
5
6
|
|
6
7
|
config_param :key, :string, :default => nil
|
7
8
|
config_param :file, :string, :default => nil
|
9
|
+
config_param :cache_per_tag, :size, :default => 1
|
8
10
|
|
9
11
|
# Define `log` method for v0.10.42 or earlier
|
10
12
|
unless method_defined?(:log)
|
@@ -24,9 +26,7 @@ class Fluent::DedupOutput < Fluent::Output
|
|
24
26
|
def start
|
25
27
|
super
|
26
28
|
|
27
|
-
|
28
|
-
@states = JSON.parse(File.open(@file).read) rescue {}
|
29
|
-
end
|
29
|
+
restore_states
|
30
30
|
end
|
31
31
|
|
32
32
|
def shutdown
|
@@ -38,7 +38,6 @@ class Fluent::DedupOutput < Fluent::Output
|
|
38
38
|
def emit(tag, es, chain)
|
39
39
|
es.each do |time, record|
|
40
40
|
next if dup?(tag, record)
|
41
|
-
update_states(tag, record)
|
42
41
|
Fluent::Engine.emit("dedup.#{tag}", time, record)
|
43
42
|
end
|
44
43
|
|
@@ -46,22 +45,45 @@ class Fluent::DedupOutput < Fluent::Output
|
|
46
45
|
end
|
47
46
|
|
48
47
|
private
|
48
|
+
def restore_states
|
49
|
+
if not @file.nil? and File.file?(@file)
|
50
|
+
dump = JSON.parse(File.open(@file).read) rescue {}
|
51
|
+
dump.each do |tag, ids|
|
52
|
+
lru = new_lru
|
53
|
+
ids.each {|id| lru[id] = true}
|
54
|
+
@states[tag] = lru
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
49
59
|
def save_states
|
50
60
|
unless @file.nil?
|
51
61
|
File.open(@file, 'wb') do |f|
|
52
|
-
|
62
|
+
dump = {}
|
63
|
+
@states.each do |tag, lru|
|
64
|
+
dump[tag] = lru.to_a.map(&:first)
|
65
|
+
end
|
66
|
+
f.print(dump.to_json)
|
53
67
|
end
|
54
68
|
end
|
55
69
|
end
|
56
70
|
|
57
71
|
def dup?(tag, record)
|
58
|
-
|
72
|
+
is_dup = false
|
73
|
+
if record.include?(@key)
|
74
|
+
@states[tag] = new_lru unless @states.include?(tag)
|
75
|
+
if @states[tag].fetch(record[@key])
|
76
|
+
is_dup = true
|
77
|
+
else
|
78
|
+
@states[tag][record[@key]] = true
|
79
|
+
end
|
80
|
+
else
|
59
81
|
log.warn "record does not have key `#{@key}`, record: #{record.to_json}"
|
60
82
|
end
|
61
|
-
|
83
|
+
is_dup
|
62
84
|
end
|
63
85
|
|
64
|
-
def
|
65
|
-
@
|
86
|
+
def new_lru
|
87
|
+
LruRedux::ThreadSafeCache.new(@cache_per_tag)
|
66
88
|
end
|
67
89
|
end
|
@@ -89,4 +89,25 @@ class DedupOutputTest < Test::Unit::TestCase
|
|
89
89
|
assert_equal 0, d2.emits.length
|
90
90
|
end
|
91
91
|
end
|
92
|
+
|
93
|
+
sub_test_case '`cache_per_tag` parameter is present' do
|
94
|
+
test "a record identical to most recent N records is suppressed" do
|
95
|
+
config = %[
|
96
|
+
key unique_id
|
97
|
+
cache_per_tag 2
|
98
|
+
]
|
99
|
+
|
100
|
+
d = create_driver(config)
|
101
|
+
d.run do
|
102
|
+
d.emit({'unique_id' => '1'}, Time.now)
|
103
|
+
d.emit({'unique_id' => '1'}, Time.now) # dup
|
104
|
+
d.emit({'unique_id' => '2'}, Time.now)
|
105
|
+
d.emit({'unique_id' => '1'}, Time.now) # dup
|
106
|
+
end
|
107
|
+
|
108
|
+
assert_equal 2, d.emits.length
|
109
|
+
assert_equal '1', d.emits[0][2]['unique_id']
|
110
|
+
assert_equal '2', d.emits[1][2]['unique_id']
|
111
|
+
end
|
112
|
+
end
|
92
113
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-dedup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- edvakf
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: lru_redux
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
69
83
|
description: fluent-plugin-dedup is a fluentd plugin to suppress emission of subsequent
|
70
84
|
logs identical to the first one.
|
71
85
|
email:
|