fluent-plugin-key-picker 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +4 -0
- data/Gemfile +2 -0
- data/LICENSE +2 -0
- data/README.md +85 -0
- data/Rakefile +9 -0
- data/fluent-plugin-key-picker.gemspec +18 -0
- data/lib/fluent/plugin/out_key_picker.rb +57 -0
- data/test/plugin/test_out_key_picker.rb +118 -0
- data/test/test_helper.rb +21 -0
- metadata +83 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: d7d01f21018a0789fb1216c54cbaa32b3338b53d
|
4
|
+
data.tar.gz: 751d86fe1d11bca30250763d83c357c07c30a51b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 5dd5dd150f144a1e724abbe05ddc513ae4ff837b691f408c8244fe4ee0bca3bbe874183a2e437d931562bca949168cefc37eac369204848f8bd69c0ade74e42e
|
7
|
+
data.tar.gz: 4b70cf884c3251eeb723d12f6d49eb20e297039602a795ba4d91ee3500860e1ea10407386548588d75a26e9167eeee1353066b0440688d14611c7bfd55ec000d
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
data/README.md
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
# fluent-plugin-time_parser, a plugin for [Fluentd](http://fluentd.org)
|
2
|
+
|
3
|
+
## Component
|
4
|
+
KeyPickerOutput
|
5
|
+
|
6
|
+
Take a record and pick only the keys you want to pass
|
7
|
+
|
8
|
+
## wat?!
|
9
|
+
|
10
|
+
```
|
11
|
+
<match test.**>
|
12
|
+
type key_picker
|
13
|
+
|
14
|
+
keys foo, baz
|
15
|
+
add_tag_prefix picked.
|
16
|
+
</match>
|
17
|
+
```
|
18
|
+
|
19
|
+
And you feed such a value into fluentd:
|
20
|
+
|
21
|
+
```
|
22
|
+
"test" => {
|
23
|
+
"foo" => "a",
|
24
|
+
"bar" => "b",
|
25
|
+
"baz" => "c",
|
26
|
+
"bah" => "e",
|
27
|
+
}
|
28
|
+
```
|
29
|
+
|
30
|
+
Then you'll get re-emmited tags/records like so:
|
31
|
+
|
32
|
+
```
|
33
|
+
"picked.test" => {
|
34
|
+
"foo" => "a",
|
35
|
+
"baz" => "c"
|
36
|
+
}
|
37
|
+
```
|
38
|
+
### WARNING
|
39
|
+
Unmatched keys will DISAPPEAR, please use this with caution.
|
40
|
+
|
41
|
+
## Configuration
|
42
|
+
|
43
|
+
### keys
|
44
|
+
|
45
|
+
`keys` should include the keys you want to pick.
|
46
|
+
|
47
|
+
### remove_tag_prefix, remove_tag_suffix, add_tag_prefix, add_tag_suffix
|
48
|
+
|
49
|
+
These params are included from `Fluent::HandleTagNameMixin`. See the code for details.
|
50
|
+
|
51
|
+
You must add at least one of these params.
|
52
|
+
|
53
|
+
|
54
|
+
## Installation
|
55
|
+
|
56
|
+
Add this line to your application's Gemfile:
|
57
|
+
|
58
|
+
gem 'fluent-plugin-key-picker'
|
59
|
+
|
60
|
+
And then execute:
|
61
|
+
|
62
|
+
$ bundle
|
63
|
+
|
64
|
+
Or install it yourself as:
|
65
|
+
|
66
|
+
$ gem install fluent-plugin-key-picker
|
67
|
+
|
68
|
+
|
69
|
+
## Contributing
|
70
|
+
|
71
|
+
1. Fork it
|
72
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
73
|
+
3. Commit your changes (`git commit -am 'Added some feature'`)
|
74
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
75
|
+
5. Create new Pull Request
|
76
|
+
|
77
|
+
## Copyright
|
78
|
+
|
79
|
+
### Copyright
|
80
|
+
|
81
|
+
Copyright (c) 2014- Carlos Donderis (@CaDs)
|
82
|
+
|
83
|
+
### License
|
84
|
+
|
85
|
+
Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
Gem::Specification.new do |gem|
|
2
|
+
gem.name = 'fluent-plugin-key-picker'
|
3
|
+
gem.version = '0.0.1'
|
4
|
+
gem.authors = ['Carlos Donderis']
|
5
|
+
gem.email = ['cdonderis@gmail.com']
|
6
|
+
gem.homepage = 'http://github.com/cads/fluent-plugin-key-picker'
|
7
|
+
gem.description = %q{Fluentd plugin for filtering / picking desired keys.}
|
8
|
+
gem.summary = %q{This plugin allows you specify only keys you want to use.}
|
9
|
+
gem.license = 'Apache License, Version 2.0'
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.require_paths = ['lib']
|
15
|
+
|
16
|
+
gem.add_development_dependency 'rake', '~> 0'
|
17
|
+
gem.add_development_dependency 'fluentd', '~> 0'
|
18
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
3
|
+
module Fluent
|
4
|
+
class KeyPickerOutput < Output
|
5
|
+
include Fluent::HandleTagNameMixin
|
6
|
+
|
7
|
+
Fluent::Plugin.register_output('key_picker', self)
|
8
|
+
|
9
|
+
config_param :keys, :string, :default => nil
|
10
|
+
config_param :add_tag_and_reemit, :bool, :default => false
|
11
|
+
|
12
|
+
def configure(conf)
|
13
|
+
super
|
14
|
+
|
15
|
+
if (
|
16
|
+
!remove_tag_prefix &&
|
17
|
+
!remove_tag_suffix &&
|
18
|
+
!add_tag_prefix &&
|
19
|
+
!add_tag_suffix
|
20
|
+
)
|
21
|
+
raise ConfigError, "out_key_picker: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
|
22
|
+
end
|
23
|
+
|
24
|
+
unless keys && !keys.nil?
|
25
|
+
raise ConfigError, "keys: You need to specify the keys you want to pick."
|
26
|
+
end
|
27
|
+
|
28
|
+
@keys = keys && keys.split(/\s*,\s*/)
|
29
|
+
end
|
30
|
+
|
31
|
+
def start
|
32
|
+
super
|
33
|
+
end
|
34
|
+
|
35
|
+
def shutdown
|
36
|
+
super
|
37
|
+
end
|
38
|
+
|
39
|
+
def emit(tag, es, chain)
|
40
|
+
es.each {|time,record|
|
41
|
+
t = tag.dup
|
42
|
+
filter_record(t, time, record)
|
43
|
+
Engine.emit(t, time, record)
|
44
|
+
}
|
45
|
+
chain.next
|
46
|
+
end
|
47
|
+
|
48
|
+
def filter_record(tag, time, record)
|
49
|
+
begin
|
50
|
+
record.keep_if{|key, value| @keys.include?(key.to_s)}
|
51
|
+
rescue ArgumentError => error
|
52
|
+
$log.warn("out_key_picker: #{error.message}")
|
53
|
+
end
|
54
|
+
super(tag, time, record)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
require 'byebug'
|
2
|
+
# -*- encoding: utf-8 -*-
|
3
|
+
require '../test_helper'
|
4
|
+
|
5
|
+
class KeyPickerOutputTest < Test::Unit::TestCase
|
6
|
+
|
7
|
+
TEST_RECORD = {'foo' => 'a', 'bar' => 'b', 'baz' => 'c'}
|
8
|
+
EMPTY_HASH = {}
|
9
|
+
|
10
|
+
def setup
|
11
|
+
Fluent::Test.setup
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_driver(conf, tag = 'test')
|
15
|
+
Fluent::Test::OutputTestDriver.new(
|
16
|
+
Fluent::KeyPickerOutput, tag
|
17
|
+
).configure(conf)
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_configure
|
21
|
+
d = create_driver(%[
|
22
|
+
keys foo, bar
|
23
|
+
add_tag_prefix picked.
|
24
|
+
])
|
25
|
+
assert_equal ['foo', 'bar'], d.instance.keys
|
26
|
+
assert_equal 'picked.', d.instance.add_tag_prefix
|
27
|
+
|
28
|
+
#No Prefix
|
29
|
+
assert_raise(Fluent::ConfigError) do
|
30
|
+
create_driver(%[
|
31
|
+
keys foo, bar
|
32
|
+
])
|
33
|
+
end
|
34
|
+
|
35
|
+
#No Keys
|
36
|
+
assert_raise(Fluent::ConfigError) do
|
37
|
+
create_driver(%[
|
38
|
+
add_tag_prefix picked.
|
39
|
+
])
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_filter_record
|
44
|
+
d = create_driver(%[
|
45
|
+
keys foo, bar
|
46
|
+
add_tag_prefix picked.
|
47
|
+
])
|
48
|
+
|
49
|
+
record = TEST_RECORD.dup
|
50
|
+
d.instance.filter_record('test', Time.now, record)
|
51
|
+
assert_equal record, {'foo' => 'a', 'bar' => 'b'}
|
52
|
+
assert_equal record['foo'], 'a'
|
53
|
+
assert_equal record['bar'], 'b'
|
54
|
+
end
|
55
|
+
|
56
|
+
|
57
|
+
def test_no_matching_record
|
58
|
+
d = create_driver(%[
|
59
|
+
keys foo, bar
|
60
|
+
add_tag_prefix picked.
|
61
|
+
])
|
62
|
+
|
63
|
+
tag = 'test'
|
64
|
+
record = {}
|
65
|
+
|
66
|
+
assert_equal record, {}
|
67
|
+
end
|
68
|
+
|
69
|
+
def test_emit
|
70
|
+
d = create_driver(%[
|
71
|
+
keys foo, bar
|
72
|
+
add_tag_prefix picked.
|
73
|
+
])
|
74
|
+
|
75
|
+
d.run { d.emit(TEST_RECORD.dup) }
|
76
|
+
emits = d.emits
|
77
|
+
|
78
|
+
assert_equal 1, emits.count
|
79
|
+
assert_equal 'picked.test', emits[0][0]
|
80
|
+
assert_equal 'a', emits[0][2]['foo']
|
81
|
+
assert_equal 'b', emits[0][2]['bar']
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_emit_multi
|
85
|
+
d = create_driver(%[
|
86
|
+
keys foo, bar
|
87
|
+
add_tag_prefix picked.
|
88
|
+
])
|
89
|
+
d.run do
|
90
|
+
d.emit(TEST_RECORD.dup)
|
91
|
+
d.emit(TEST_RECORD.dup)
|
92
|
+
d.emit(TEST_RECORD.dup)
|
93
|
+
end
|
94
|
+
emits = d.emits
|
95
|
+
|
96
|
+
assert_equal 3, emits.count
|
97
|
+
|
98
|
+
emits.each do |e|
|
99
|
+
assert_equal 'picked.test', e[0]
|
100
|
+
assert_equal 'a', e[2]['foo']
|
101
|
+
assert_equal 'b', e[2]['bar']
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def test_emit_without_match_key
|
106
|
+
d = create_driver(%[
|
107
|
+
keys fiz, faz
|
108
|
+
add_tag_prefix picked.
|
109
|
+
])
|
110
|
+
d.run { d.emit(TEST_RECORD.dup) }
|
111
|
+
emits = d.emits
|
112
|
+
|
113
|
+
assert_equal 1, emits.count
|
114
|
+
assert_equal 'picked.test', emits[0][0]
|
115
|
+
assert_equal EMPTY_HASH, emits[0][2]
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
4
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
5
|
+
|
6
|
+
require 'fluent/test'
|
7
|
+
|
8
|
+
unless ENV.has_key?('VERBOSE')
|
9
|
+
nulllogger = Object.new
|
10
|
+
nulllogger.instance_eval {|obj|
|
11
|
+
def method_missing(method, *args)
|
12
|
+
# pass
|
13
|
+
end
|
14
|
+
}
|
15
|
+
$log = nulllogger
|
16
|
+
end
|
17
|
+
|
18
|
+
require 'fluent/plugin/out_key_picker'
|
19
|
+
|
20
|
+
class Test::Unit::TestCase
|
21
|
+
end
|
metadata
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-key-picker
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Carlos Donderis
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-09-18 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: fluentd
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
description: Fluentd plugin for filtering / picking desired keys.
|
42
|
+
email:
|
43
|
+
- cdonderis@gmail.com
|
44
|
+
executables: []
|
45
|
+
extensions: []
|
46
|
+
extra_rdoc_files: []
|
47
|
+
files:
|
48
|
+
- ".gitignore"
|
49
|
+
- Gemfile
|
50
|
+
- LICENSE
|
51
|
+
- README.md
|
52
|
+
- Rakefile
|
53
|
+
- fluent-plugin-key-picker.gemspec
|
54
|
+
- lib/fluent/plugin/out_key_picker.rb
|
55
|
+
- test/plugin/test_out_key_picker.rb
|
56
|
+
- test/test_helper.rb
|
57
|
+
homepage: http://github.com/cads/fluent-plugin-key-picker
|
58
|
+
licenses:
|
59
|
+
- Apache License, Version 2.0
|
60
|
+
metadata: {}
|
61
|
+
post_install_message:
|
62
|
+
rdoc_options: []
|
63
|
+
require_paths:
|
64
|
+
- lib
|
65
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '0'
|
75
|
+
requirements: []
|
76
|
+
rubyforge_project:
|
77
|
+
rubygems_version: 2.2.2
|
78
|
+
signing_key:
|
79
|
+
specification_version: 4
|
80
|
+
summary: This plugin allows you specify only keys you want to use.
|
81
|
+
test_files:
|
82
|
+
- test/plugin/test_out_key_picker.rb
|
83
|
+
- test/test_helper.rb
|