fluent-plugin-record-modifier 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ChangeLog +6 -0
- data/Rakefile +2 -3
- data/VERSION +1 -1
- data/fluent-plugin-record-modifier.gemspec +1 -1
- data/lib/fluent/plugin/filter_record_modifier.rb +86 -0
- data/test/test_filter_record_modifier.rb +111 -0
- data/test/{out_record_modifier.rb → test_out_record_modifier.rb} +0 -0
- metadata +17 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 344506e1d7deb2d3211df17f02386283fa8a30ab
|
4
|
+
data.tar.gz: 44e6d422823ddf0c686abe066b7da36fc96e4e19
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 45f15173d92fa1d91e20c62337ffe757852bf42bd006c7847459ca61521cd21cc63b604715489cfec05555f8ce95ba91d182816e470c64771bb3e81cecacad17
|
7
|
+
data.tar.gz: 7c63568717b13b17bdec90bf43c6edf30884639dd9d586efe85bd90f02787d99a7cbbd26484f6ab2676bdddb2facae032148307b0a35aec6c36428f0e602f3ca
|
data/ChangeLog
CHANGED
data/Rakefile
CHANGED
@@ -1,12 +1,11 @@
|
|
1
|
-
|
2
1
|
require 'bundler'
|
3
|
-
|
2
|
+
require 'bundler/gem_tasks'
|
4
3
|
|
5
4
|
require 'rake/testtask'
|
6
5
|
|
7
6
|
Rake::TestTask.new(:test) do |test|
|
8
7
|
test.libs << 'lib' << 'test'
|
9
|
-
test.test_files = FileList['test
|
8
|
+
test.test_files = FileList['test/test_*.rb']
|
10
9
|
test.verbose = true
|
11
10
|
end
|
12
11
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.3.0
|
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
|
|
17
17
|
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
18
|
gem.require_paths = ['lib']
|
19
19
|
|
20
|
-
gem.add_dependency "fluentd", "
|
20
|
+
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
21
21
|
gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
|
22
22
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
23
23
|
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'fluent/mixin/config_placeholders'
|
2
|
+
|
3
|
+
module Fluent
|
4
|
+
class RecordModifierFilter < Filter
|
5
|
+
Fluent::Plugin.register_filter('record_modifier', self)
|
6
|
+
|
7
|
+
config_param :char_encoding, :string, :default => nil
|
8
|
+
config_param :remove_keys, :string, :default => nil
|
9
|
+
|
10
|
+
include SetTagKeyMixin
|
11
|
+
include Fluent::Mixin::ConfigPlaceholders
|
12
|
+
|
13
|
+
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id include_tag_key tag_key char_encoding remove_keys)
|
14
|
+
|
15
|
+
def configure(conf)
|
16
|
+
super
|
17
|
+
|
18
|
+
@map = {}
|
19
|
+
conf.each_pair { |k, v|
|
20
|
+
unless BUILTIN_CONFIGURATIONS.include?(k)
|
21
|
+
conf.has_key?(k)
|
22
|
+
@map[k] = v
|
23
|
+
end
|
24
|
+
}
|
25
|
+
|
26
|
+
if @char_encoding
|
27
|
+
from, to = @char_encoding.split(':', 2)
|
28
|
+
@from_enc = Encoding.find(from)
|
29
|
+
@to_enc = Encoding.find(to) if to
|
30
|
+
|
31
|
+
m = if @to_enc
|
32
|
+
method(:convert_encoding)
|
33
|
+
else
|
34
|
+
method(:set_encoding)
|
35
|
+
end
|
36
|
+
|
37
|
+
(class << self; self; end).module_eval do
|
38
|
+
define_method(:change_encoding, m)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
if @remove_keys
|
43
|
+
@remove_keys = @remove_keys.split(',').map {|e| e.strip }
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def filter(tag, time, record)
|
48
|
+
filter_record(tag, time, record)
|
49
|
+
modify_record(record)
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
|
54
|
+
def modify_record(record)
|
55
|
+
@map.each_pair { |k, v|
|
56
|
+
record[k] = v
|
57
|
+
}
|
58
|
+
|
59
|
+
if @remove_keys
|
60
|
+
@remove_keys.each { |v|
|
61
|
+
record.delete(v)
|
62
|
+
}
|
63
|
+
end
|
64
|
+
|
65
|
+
record = change_encoding(record) if @char_encoding
|
66
|
+
record
|
67
|
+
end
|
68
|
+
|
69
|
+
def set_encoding(record)
|
70
|
+
record.each_pair { |k, v|
|
71
|
+
if v.is_a?(String)
|
72
|
+
v.force_encoding(@from_enc)
|
73
|
+
end
|
74
|
+
}
|
75
|
+
end
|
76
|
+
|
77
|
+
def convert_encoding(record)
|
78
|
+
record.each_pair { |k, v|
|
79
|
+
if v.is_a?(String)
|
80
|
+
v.force_encoding(@from_enc) if v.encoding == Encoding::BINARY
|
81
|
+
v.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
|
82
|
+
end
|
83
|
+
}
|
84
|
+
end
|
85
|
+
end if defined?(Filter)
|
86
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require 'fluent/test'
|
2
|
+
require 'fluent/plugin/filter_record_modifier'
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
exit unless defined?(Fluent::Filter)
|
6
|
+
|
7
|
+
class RecordModifierFilterTest < Test::Unit::TestCase
|
8
|
+
def setup
|
9
|
+
Fluent::Test.setup
|
10
|
+
@tag = 'test.tag'
|
11
|
+
end
|
12
|
+
|
13
|
+
CONFIG = %[
|
14
|
+
type record_modifier
|
15
|
+
|
16
|
+
gen_host ${hostname}
|
17
|
+
foo bar
|
18
|
+
include_tag_key
|
19
|
+
tag_key included_tag
|
20
|
+
remove_keys hoge
|
21
|
+
]
|
22
|
+
|
23
|
+
def create_driver(conf = CONFIG)
|
24
|
+
Fluent::Test::FilterTestDriver.new(Fluent::RecordModifierFilter, @tag).configure(conf)
|
25
|
+
end
|
26
|
+
|
27
|
+
def get_hostname
|
28
|
+
require 'socket'
|
29
|
+
Socket.gethostname.chomp
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_configure
|
33
|
+
d = create_driver
|
34
|
+
map = d.instance.instance_variable_get(:@map)
|
35
|
+
|
36
|
+
assert_equal get_hostname, map['gen_host']
|
37
|
+
assert_equal 'bar', map['foo']
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_format
|
41
|
+
d = create_driver
|
42
|
+
|
43
|
+
d.run do
|
44
|
+
d.emit("a" => 1)
|
45
|
+
d.emit("a" => 2)
|
46
|
+
end
|
47
|
+
|
48
|
+
mapped = {'gen_host' => get_hostname, 'foo' => 'bar', 'included_tag' => @tag}
|
49
|
+
assert_equal [
|
50
|
+
{"a" => 1}.merge(mapped),
|
51
|
+
{"a" => 2}.merge(mapped),
|
52
|
+
], d.filtered_as_array.map { |e| e.last }
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_set_char_encoding
|
56
|
+
d = create_driver %[
|
57
|
+
type record_modifier
|
58
|
+
|
59
|
+
char_encoding utf-8
|
60
|
+
]
|
61
|
+
|
62
|
+
|
63
|
+
d.run do
|
64
|
+
d.emit("k" => 'v'.force_encoding('BINARY'))
|
65
|
+
end
|
66
|
+
|
67
|
+
assert_equal [{"k" => 'v'.force_encoding('UTF-8')}], d.filtered_as_array.map { |e| e.last }
|
68
|
+
end
|
69
|
+
|
70
|
+
def test_convert_char_encoding
|
71
|
+
d = create_driver %[
|
72
|
+
type record_modifier
|
73
|
+
|
74
|
+
char_encoding utf-8:cp932
|
75
|
+
]
|
76
|
+
|
77
|
+
d.run do
|
78
|
+
d.emit("k" => 'v'.force_encoding('utf-8'))
|
79
|
+
end
|
80
|
+
|
81
|
+
assert_equal [{"k" => 'v'.force_encoding('cp932')}], d.filtered_as_array.map { |e| e.last }
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_remove_one_key
|
85
|
+
d = create_driver %[
|
86
|
+
type record_modifier
|
87
|
+
|
88
|
+
remove_keys k1
|
89
|
+
]
|
90
|
+
|
91
|
+
d.run do
|
92
|
+
d.emit("k1" => 'v', "k2" => 'v')
|
93
|
+
end
|
94
|
+
|
95
|
+
assert_equal [{"k2" => 'v'}], d.filtered_as_array.map { |e| e.last }
|
96
|
+
end
|
97
|
+
|
98
|
+
def test_remove_multiple_keys
|
99
|
+
d = create_driver %[
|
100
|
+
type record_modifier
|
101
|
+
|
102
|
+
remove_keys k1, k2, k3
|
103
|
+
]
|
104
|
+
|
105
|
+
d.run do
|
106
|
+
d.emit("k1" => 'v', "k2" => 'v', "k4" => 'v')
|
107
|
+
end
|
108
|
+
|
109
|
+
assert_equal [{"k4" => 'v'}], d.filtered_as_array.map { |e| e.last }
|
110
|
+
end
|
111
|
+
end
|
File without changes
|
metadata
CHANGED
@@ -1,29 +1,35 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-record-modifier
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Nakagawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-06-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.10.58
|
20
|
+
- - "<"
|
18
21
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
22
|
+
version: '2'
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
|
-
- - "
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.10.58
|
30
|
+
- - "<"
|
25
31
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
32
|
+
version: '2'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: fluent-mixin-config-placeholders
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -65,8 +71,10 @@ files:
|
|
65
71
|
- Rakefile
|
66
72
|
- VERSION
|
67
73
|
- fluent-plugin-record-modifier.gemspec
|
74
|
+
- lib/fluent/plugin/filter_record_modifier.rb
|
68
75
|
- lib/fluent/plugin/out_record_modifier.rb
|
69
|
-
- test/
|
76
|
+
- test/test_filter_record_modifier.rb
|
77
|
+
- test/test_out_record_modifier.rb
|
70
78
|
homepage: https://github.com/repeatedly/fluent-plugin-record-modifier
|
71
79
|
licenses:
|
72
80
|
- MIT
|
@@ -92,4 +100,5 @@ signing_key:
|
|
92
100
|
specification_version: 4
|
93
101
|
summary: Output filter plugin for modifying each event record
|
94
102
|
test_files:
|
95
|
-
- test/
|
103
|
+
- test/test_filter_record_modifier.rb
|
104
|
+
- test/test_out_record_modifier.rb
|