fluent-plugin-lookup 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +4 -0
- data/README.md +91 -0
- data/Rakefile +12 -0
- data/fluent-plugin-lookup.gemspec +19 -0
- data/lib/fluent/plugin/out_lookup.rb +101 -0
- data/test/plugin/correct.csv +5 -0
- data/test/plugin/duplicates.csv +6 -0
- data/test/plugin/empty.csv +0 -0
- data/test/plugin/test_out_lookup.rb +375 -0
- data/test/test_helper.rb +19 -0
- metadata +101 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: bc552128863c56880e1ed789e3dc9956f8063bfe
|
4
|
+
data.tar.gz: a870d43673d7027df0f90fdf6343dae2f4f397f1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: a4752b33dcba97bd4aaa359e63678efc02437f61315f3ea5c858715bb406ba22ca134aebc7299bdff21f0be591341c84851dfe88bbe986893b1533ddb01cddcc
|
7
|
+
data.tar.gz: 45bcabbafc264a61d6686ed6c305997d88b7726760da4102428c26db0537caca08697e50e6ae1a93a070abd176d75b53602c975a057fecacc90ec90e4c169f3d
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
fluent-plugin-lookup
|
2
|
+
====================
|
3
|
+
|
4
|
+
(Yet another Fluentd plugin)
|
5
|
+
|
6
|
+
What
|
7
|
+
----
|
8
|
+
|
9
|
+
Allows to replace record values for specific keys, using a lookup table from a *CSV* file.
|
10
|
+
|
11
|
+
How
|
12
|
+
---
|
13
|
+
|
14
|
+
You basically want to define :
|
15
|
+
- Input field : **field**.
|
16
|
+
- Output field : **output_field** (omitting this parameter will replace *input* field value).
|
17
|
+
- Lookup table CSV file : **table_file** (two columns per row, separated by a comma).
|
18
|
+
- Sanity check, raises error if empty, malformed file or duplicates entries inside the file : **strict** (omitting this parameter will default to *false*).
|
19
|
+
|
20
|
+
Use this filter multiple times if you need to repalce multiple fields.
|
21
|
+
|
22
|
+
Examples
|
23
|
+
========
|
24
|
+
|
25
|
+
This is our *lookup.csv* file :
|
26
|
+
|
27
|
+
```
|
28
|
+
value,other_value
|
29
|
+
nicolas,cage
|
30
|
+
input,output
|
31
|
+
1,one
|
32
|
+
two,2
|
33
|
+
```
|
34
|
+
|
35
|
+
Example 1
|
36
|
+
---------
|
37
|
+
|
38
|
+
```
|
39
|
+
<match *.test>
|
40
|
+
type lookup
|
41
|
+
add_tag_prefix lookup.
|
42
|
+
table_file /usr/share/my/lookup.csv
|
43
|
+
field key1
|
44
|
+
output_field key2
|
45
|
+
</match>
|
46
|
+
```
|
47
|
+
|
48
|
+
Example of records :
|
49
|
+
```
|
50
|
+
{
|
51
|
+
'key1' => "nicolas",
|
52
|
+
'foo' => "bar"
|
53
|
+
}
|
54
|
+
```
|
55
|
+
... will output :
|
56
|
+
```
|
57
|
+
{
|
58
|
+
'key1' => "nicolas",
|
59
|
+
'key2' => "cage",
|
60
|
+
'foo' => "bar"
|
61
|
+
}
|
62
|
+
```
|
63
|
+
|
64
|
+
Example 2
|
65
|
+
---------
|
66
|
+
|
67
|
+
```
|
68
|
+
<match *.test>
|
69
|
+
type lookup
|
70
|
+
add_tag_prefix lookup.
|
71
|
+
table_file /usr/share/my/lookup.csv
|
72
|
+
field key1
|
73
|
+
</match>
|
74
|
+
```
|
75
|
+
|
76
|
+
Example of records :
|
77
|
+
```
|
78
|
+
{
|
79
|
+
'key1' => "nicolas",
|
80
|
+
'foo' => "bar"
|
81
|
+
}
|
82
|
+
```
|
83
|
+
... will output :
|
84
|
+
```
|
85
|
+
{
|
86
|
+
'key1' => "cage",
|
87
|
+
'foo' => "bar"
|
88
|
+
}
|
89
|
+
```
|
90
|
+
|
91
|
+
Since *output_field* is not defined, the input *field* value is replaced.
|
data/Rakefile
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
require 'rake/testtask'
|
4
|
+
|
5
|
+
desc 'Default: run test.'
|
6
|
+
task :default => :test
|
7
|
+
|
8
|
+
Rake::TestTask.new(:test) do |test|
|
9
|
+
test.libs << 'lib' << 'test'
|
10
|
+
test.pattern = 'test/**/test_*.rb'
|
11
|
+
test.verbose = true
|
12
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
Gem::Specification.new do |spec|
|
2
|
+
spec.name = "fluent-plugin-lookup"
|
3
|
+
spec.version = "0.0.1"
|
4
|
+
spec.authors = ["Neozaru"]
|
5
|
+
spec.email = ["neozaru@mailoo.org"]
|
6
|
+
spec.description = %q{Fluentd custom plugin to replace fields values using lookup table file}
|
7
|
+
spec.summary = %q{Fluentd custom plugin to replace fields values using lookup table file}
|
8
|
+
spec.homepage = "https://github.com/Neozaru/fluent-plugin-lookup.git"
|
9
|
+
spec.license = "WTFPL"
|
10
|
+
|
11
|
+
spec.files = `git ls-files`.split($/)
|
12
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
13
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
14
|
+
spec.require_paths = ["lib"]
|
15
|
+
|
16
|
+
spec.add_development_dependency "bundler"
|
17
|
+
spec.add_development_dependency "rake"
|
18
|
+
spec.add_development_dependency "fluentd"
|
19
|
+
end
|
@@ -0,0 +1,101 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
require "csv"
|
3
|
+
|
4
|
+
module Fluent
|
5
|
+
class LookupOutput < Output
|
6
|
+
include Fluent::HandleTagNameMixin
|
7
|
+
|
8
|
+
Fluent::Plugin.register_output('lookup', self)
|
9
|
+
|
10
|
+
config_param :table_file, :string, :default => nil
|
11
|
+
config_param :field, :string, :default => nil
|
12
|
+
config_param :output_field, :string, :default => nil
|
13
|
+
config_param :strict, :bool, :default => false
|
14
|
+
|
15
|
+
def handle_row(lookup_table, row)
|
16
|
+
if (row.length < 2)
|
17
|
+
return handle_row_error(row, "Too few columns : #{row.length} instead of 2")
|
18
|
+
end
|
19
|
+
|
20
|
+
# If too much columns
|
21
|
+
if (strict && row.length > 2)
|
22
|
+
return handle_row_error(row, "Too much columns : #{row.length} instead of 2")
|
23
|
+
end
|
24
|
+
|
25
|
+
# If duplicates
|
26
|
+
if (strict && lookup_table.has_key?(row[0]))
|
27
|
+
return handle_row_error(row, "Duplicate entry")
|
28
|
+
end
|
29
|
+
|
30
|
+
lookup_table[row[0]] = row[1]
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
|
35
|
+
def create_lookup_table(file)
|
36
|
+
lookup_table = {}
|
37
|
+
CSV.foreach(file) do |row|
|
38
|
+
handle_row(lookup_table, row)
|
39
|
+
end
|
40
|
+
|
41
|
+
if (strict && lookup_table.length == 0)
|
42
|
+
raise ConfigError, "Lookup file is empty"
|
43
|
+
end
|
44
|
+
|
45
|
+
return lookup_table
|
46
|
+
rescue Errno::ENOENT => e
|
47
|
+
handle_file_err(file, e)
|
48
|
+
rescue Errno::EACCES => e
|
49
|
+
handle_file_err(file, e)
|
50
|
+
end
|
51
|
+
|
52
|
+
|
53
|
+
def configure(conf)
|
54
|
+
super
|
55
|
+
|
56
|
+
if (field.nil? || table_file.nil?)
|
57
|
+
raise ConfigError, "lookup: Both 'field', and 'table_file' are required to be set."
|
58
|
+
end
|
59
|
+
|
60
|
+
@lookup_table = create_lookup_table(table_file)
|
61
|
+
@field = field
|
62
|
+
@output_field = output_field || field
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
def emit(tag, es, chain)
|
67
|
+
es.each { |time, record|
|
68
|
+
t = tag.dup
|
69
|
+
filter_record(t, time, record)
|
70
|
+
Engine.emit(t, time, record)
|
71
|
+
}
|
72
|
+
|
73
|
+
chain.next
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def filter_record(tag, time, record)
|
79
|
+
super(tag, time, record)
|
80
|
+
if (not record.has_key?(@field))
|
81
|
+
return
|
82
|
+
end
|
83
|
+
record[@output_field] = process(record[@field]) || record[@field]
|
84
|
+
end
|
85
|
+
|
86
|
+
def process(value)
|
87
|
+
return @lookup_table[value]
|
88
|
+
end
|
89
|
+
|
90
|
+
|
91
|
+
def handle_row_error(row, e)
|
92
|
+
raise ConfigError, "Error at row #{row} : #{e}"
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
def handle_file_err(file, e)
|
97
|
+
raise ConfigError, "Unable to open file '#{file}' : #{e.message}"
|
98
|
+
end
|
99
|
+
|
100
|
+
end
|
101
|
+
end
|
File without changes
|
@@ -0,0 +1,375 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
3
|
+
require 'test_helper'
|
4
|
+
require 'fluent/plugin/out_lookup'
|
5
|
+
|
6
|
+
class LookupOutputTest < Test::Unit::TestCase
|
7
|
+
def setup
|
8
|
+
Fluent::Test.setup
|
9
|
+
dir = File.dirname(__FILE__)
|
10
|
+
@nonexisting_file = "#{dir}/tobeornottobe.csv"
|
11
|
+
@correct_file = "#{dir}/correct.csv"
|
12
|
+
@duplicates_file = "#{dir}/duplicates.csv"
|
13
|
+
@empty_file = "#{dir}/empty.csv"
|
14
|
+
end
|
15
|
+
|
16
|
+
def create_driver(conf, tag = 'test')
|
17
|
+
Fluent::Test::OutputTestDriver.new(
|
18
|
+
Fluent::LookupOutput, tag
|
19
|
+
).configure(conf)
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
def test_configure_on_success
|
24
|
+
|
25
|
+
|
26
|
+
# All set
|
27
|
+
d = create_driver(%[
|
28
|
+
strict true
|
29
|
+
add_tag_prefix lookup.
|
30
|
+
table_file #{@correct_file}
|
31
|
+
field key1
|
32
|
+
output_field key2
|
33
|
+
])
|
34
|
+
|
35
|
+
assert_equal 'lookup.', d.instance.add_tag_prefix
|
36
|
+
assert_equal 'key1', d.instance.field
|
37
|
+
assert_equal 'key2', d.instance.output_field
|
38
|
+
assert_equal true, d.instance.strict
|
39
|
+
assert_equal @correct_file, d.instance.table_file
|
40
|
+
|
41
|
+
|
42
|
+
# "Strict" omitted
|
43
|
+
d = create_driver(%[
|
44
|
+
add_tag_prefix lookup.
|
45
|
+
table_file #{@correct_file}
|
46
|
+
field key1
|
47
|
+
output_field key2
|
48
|
+
])
|
49
|
+
|
50
|
+
assert_equal 'lookup.', d.instance.add_tag_prefix
|
51
|
+
assert_equal 'key1', d.instance.field
|
52
|
+
assert_equal 'key2', d.instance.output_field
|
53
|
+
assert_equal false, d.instance.strict
|
54
|
+
assert_equal @correct_file, d.instance.table_file
|
55
|
+
|
56
|
+
|
57
|
+
# "output_field" omitted
|
58
|
+
d = create_driver(%[
|
59
|
+
strict true
|
60
|
+
add_tag_prefix lookup.
|
61
|
+
table_file #{@correct_file}
|
62
|
+
field key1
|
63
|
+
])
|
64
|
+
|
65
|
+
assert_equal 'lookup.', d.instance.add_tag_prefix
|
66
|
+
assert_equal 'key1', d.instance.field
|
67
|
+
assert_equal 'key1', d.instance.output_field
|
68
|
+
assert_equal true, d.instance.strict
|
69
|
+
assert_equal @correct_file, d.instance.table_file
|
70
|
+
|
71
|
+
|
72
|
+
# File with duplicates in non-strict mode
|
73
|
+
d = create_driver(%[
|
74
|
+
strict false
|
75
|
+
add_tag_prefix lookup.
|
76
|
+
table_file #{@duplicates_file}
|
77
|
+
field key1
|
78
|
+
output_field key2
|
79
|
+
])
|
80
|
+
|
81
|
+
assert_equal 'lookup.', d.instance.add_tag_prefix
|
82
|
+
assert_equal 'key1', d.instance.field
|
83
|
+
assert_equal 'key2', d.instance.output_field
|
84
|
+
assert_equal false, d.instance.strict
|
85
|
+
assert_equal @duplicates_file, d.instance.table_file
|
86
|
+
|
87
|
+
|
88
|
+
# Empty file in non-strict mode
|
89
|
+
d = create_driver(%[
|
90
|
+
strict false
|
91
|
+
add_tag_prefix lookup.
|
92
|
+
table_file #{@empty_file}
|
93
|
+
field key1
|
94
|
+
output_field key2
|
95
|
+
])
|
96
|
+
|
97
|
+
assert_equal 'lookup.', d.instance.add_tag_prefix
|
98
|
+
assert_equal 'key1', d.instance.field
|
99
|
+
assert_equal 'key2', d.instance.output_field
|
100
|
+
assert_equal false, d.instance.strict
|
101
|
+
assert_equal @empty_file, d.instance.table_file
|
102
|
+
|
103
|
+
end
|
104
|
+
|
105
|
+
def test_configure_on_failure
|
106
|
+
# when mandatory keys not set
|
107
|
+
assert_raise(Fluent::ConfigError) do
|
108
|
+
create_driver(%[
|
109
|
+
blah blah
|
110
|
+
])
|
111
|
+
end
|
112
|
+
|
113
|
+
# 'field' is missing
|
114
|
+
assert_raise(Fluent::ConfigError) do
|
115
|
+
create_driver(%[
|
116
|
+
strict true
|
117
|
+
add_tag_prefix lookup.
|
118
|
+
table_file #{@correct_file}
|
119
|
+
output_field key2
|
120
|
+
])
|
121
|
+
end
|
122
|
+
|
123
|
+
# 'table_file' is missing
|
124
|
+
assert_raise(Fluent::ConfigError) do
|
125
|
+
create_driver(%[
|
126
|
+
strict true
|
127
|
+
add_tag_prefix lookup.
|
128
|
+
field key1
|
129
|
+
output_field key2
|
130
|
+
])
|
131
|
+
end
|
132
|
+
|
133
|
+
# 'table_file' is not readable in strict mode
|
134
|
+
assert_raise(Fluent::ConfigError) do
|
135
|
+
create_driver(%[
|
136
|
+
strict true
|
137
|
+
add_tag_prefix lookup.
|
138
|
+
table_file #{@nonexisting_file}
|
139
|
+
field key1
|
140
|
+
output_field key2
|
141
|
+
])
|
142
|
+
end
|
143
|
+
|
144
|
+
# 'table_file' is not readable in non-strict mode
|
145
|
+
assert_raise(Fluent::ConfigError) do
|
146
|
+
create_driver(%[
|
147
|
+
strict false
|
148
|
+
add_tag_prefix lookup.
|
149
|
+
table_file #{@nonexisting_file}
|
150
|
+
field key1
|
151
|
+
output_field key2
|
152
|
+
])
|
153
|
+
end
|
154
|
+
|
155
|
+
# 'table_file' contains duplicates in strict mode
|
156
|
+
assert_raise(Fluent::ConfigError) do
|
157
|
+
create_driver(%[
|
158
|
+
strict true
|
159
|
+
add_tag_prefix lookup.
|
160
|
+
table_file #{@duplicates_file}
|
161
|
+
field key1
|
162
|
+
output_field key2
|
163
|
+
])
|
164
|
+
end
|
165
|
+
|
166
|
+
# 'table_file' is empty in strict mode
|
167
|
+
assert_raise(Fluent::ConfigError) do
|
168
|
+
create_driver(%[
|
169
|
+
strict true
|
170
|
+
add_tag_prefix lookup.
|
171
|
+
table_file #{@empty_file}
|
172
|
+
field key1
|
173
|
+
output_field key2
|
174
|
+
])
|
175
|
+
end
|
176
|
+
|
177
|
+
end
|
178
|
+
|
179
|
+
|
180
|
+
def test_unit_create_lookup_table
|
181
|
+
|
182
|
+
# Correct file
|
183
|
+
d = create_driver(%[
|
184
|
+
add_tag_prefix lookup.
|
185
|
+
table_file #{@correct_file}
|
186
|
+
field key1
|
187
|
+
output_field key2
|
188
|
+
])
|
189
|
+
|
190
|
+
table = d.instance.create_lookup_table(@correct_file)
|
191
|
+
assert_equal({"foo"=>"bar", "nicolas"=>"cage", "input"=>"output", "1"=>"one", "two"=>"2"}, table)
|
192
|
+
|
193
|
+
# With duplicates
|
194
|
+
d = create_driver(%[
|
195
|
+
add_tag_prefix lookup.
|
196
|
+
table_file #{@duplicates_file}
|
197
|
+
field key1
|
198
|
+
output_field key2
|
199
|
+
])
|
200
|
+
|
201
|
+
table = d.instance.create_lookup_table(@duplicates_file)
|
202
|
+
assert_equal({"foo"=>"bar", "nicolas"=>"cage", "input"=>"output", "1"=>"one", "two"=>"2"}, table)
|
203
|
+
|
204
|
+
# Empty file
|
205
|
+
d = create_driver(%[
|
206
|
+
add_tag_prefix lookup.
|
207
|
+
table_file #{@empty_file}
|
208
|
+
field key1
|
209
|
+
output_field key2
|
210
|
+
])
|
211
|
+
|
212
|
+
table = d.instance.create_lookup_table(@empty_file)
|
213
|
+
assert_equal({}, table)
|
214
|
+
|
215
|
+
end
|
216
|
+
|
217
|
+
|
218
|
+
def test_unit_handle_row
|
219
|
+
# Correct row
|
220
|
+
d = create_driver(%[
|
221
|
+
add_tag_prefix lookup.
|
222
|
+
table_file #{@correct_file}
|
223
|
+
field key1
|
224
|
+
])
|
225
|
+
|
226
|
+
table = {}
|
227
|
+
d.instance.handle_row(table, ["foo", "bar"])
|
228
|
+
assert_equal({"foo"=>"bar"}, table)
|
229
|
+
|
230
|
+
# Too small row
|
231
|
+
d = create_driver(%[
|
232
|
+
add_tag_prefix lookup.
|
233
|
+
table_file #{@correct_file}
|
234
|
+
field key1
|
235
|
+
])
|
236
|
+
|
237
|
+
table = {}
|
238
|
+
assert_raise do d.instance.handle_row(table, ["foo"]) end
|
239
|
+
assert_equal({}, table)
|
240
|
+
|
241
|
+
# Too large row non strict
|
242
|
+
d = create_driver(%[
|
243
|
+
add_tag_prefix lookup.
|
244
|
+
table_file #{@correct_file}
|
245
|
+
field key1
|
246
|
+
])
|
247
|
+
|
248
|
+
table = {}
|
249
|
+
d.instance.handle_row(table, ["foo", "bar", "baz"])
|
250
|
+
assert_equal({"foo" => "bar"}, table)
|
251
|
+
|
252
|
+
# Too large row strict
|
253
|
+
d = create_driver(%[
|
254
|
+
add_tag_prefix lookup.
|
255
|
+
strict true
|
256
|
+
table_file #{@correct_file}
|
257
|
+
field key1
|
258
|
+
])
|
259
|
+
|
260
|
+
table = {}
|
261
|
+
assert_raise do d.instance.handle_row(table, ["foo", "bar", "baz"]) end
|
262
|
+
assert_equal({}, table)
|
263
|
+
|
264
|
+
|
265
|
+
# Too duplicate row non strict
|
266
|
+
d = create_driver(%[
|
267
|
+
add_tag_prefix lookup.
|
268
|
+
table_file #{@correct_file}
|
269
|
+
field key1
|
270
|
+
])
|
271
|
+
|
272
|
+
table = {}
|
273
|
+
d.instance.handle_row(table, ["foo", "bar"])
|
274
|
+
d.instance.handle_row(table, ["foo", "baz"])
|
275
|
+
assert_equal({"foo" => "baz"}, table)
|
276
|
+
|
277
|
+
|
278
|
+
# Too duplicate row strict
|
279
|
+
d = create_driver(%[
|
280
|
+
add_tag_prefix lookup.
|
281
|
+
strict true
|
282
|
+
table_file #{@correct_file}
|
283
|
+
field key1
|
284
|
+
])
|
285
|
+
|
286
|
+
table = {}
|
287
|
+
d.instance.handle_row(table, ["foo", "bar"])
|
288
|
+
assert_raise do d.instance.handle_row(table, ["foo", "baz"]) end
|
289
|
+
assert_equal({"foo" => "bar"}, table)
|
290
|
+
end
|
291
|
+
|
292
|
+
|
293
|
+
def test_emit_with_output_field
|
294
|
+
d = create_driver(%[
|
295
|
+
add_tag_prefix lookup.
|
296
|
+
table_file #{@correct_file}
|
297
|
+
field key1
|
298
|
+
output_field key2
|
299
|
+
])
|
300
|
+
|
301
|
+
record = {
|
302
|
+
'key1' => "nicolas",
|
303
|
+
'foo' => "bar"
|
304
|
+
}
|
305
|
+
|
306
|
+
d.run { d.emit(record) }
|
307
|
+
emits = d.emits
|
308
|
+
|
309
|
+
assert_equal 1, emits.count
|
310
|
+
assert_equal 'lookup.test', emits[0][0]
|
311
|
+
assert_equal 'cage', emits[0][2]['key2']
|
312
|
+
end
|
313
|
+
|
314
|
+
def test_emit_with_output_field_no_correspondance
|
315
|
+
d = create_driver(%[
|
316
|
+
add_tag_prefix lookup.
|
317
|
+
table_file #{@correct_file}
|
318
|
+
field key1
|
319
|
+
output_field key2
|
320
|
+
])
|
321
|
+
|
322
|
+
record = {
|
323
|
+
'key1' => "myvalue",
|
324
|
+
'foo' => "bar"
|
325
|
+
}
|
326
|
+
|
327
|
+
d.run { d.emit(record) }
|
328
|
+
emits = d.emits
|
329
|
+
|
330
|
+
assert_equal 1, emits.count
|
331
|
+
assert_equal 'lookup.test', emits[0][0]
|
332
|
+
assert_equal 'myvalue', emits[0][2]['key2']
|
333
|
+
end
|
334
|
+
|
335
|
+
def test_emit_without_output_field
|
336
|
+
d = create_driver(%[
|
337
|
+
add_tag_prefix lookup.
|
338
|
+
table_file #{@correct_file}
|
339
|
+
field key1
|
340
|
+
])
|
341
|
+
|
342
|
+
record = {
|
343
|
+
'key1' => "nicolas",
|
344
|
+
'foo' => "bar"
|
345
|
+
}
|
346
|
+
|
347
|
+
d.run { d.emit(record) }
|
348
|
+
emits = d.emits
|
349
|
+
|
350
|
+
assert_equal 1, emits.count
|
351
|
+
assert_equal 'lookup.test', emits[0][0]
|
352
|
+
assert_equal 'cage', emits[0][2]['key1']
|
353
|
+
end
|
354
|
+
|
355
|
+
def test_emit_without_output_field_no_correspondance
|
356
|
+
d = create_driver(%[
|
357
|
+
add_tag_prefix lookup.
|
358
|
+
table_file #{@correct_file}
|
359
|
+
field key1
|
360
|
+
])
|
361
|
+
|
362
|
+
record = {
|
363
|
+
'key1' => "myvalue",
|
364
|
+
'foo' => "bar"
|
365
|
+
}
|
366
|
+
|
367
|
+
d.run { d.emit(record) }
|
368
|
+
emits = d.emits
|
369
|
+
|
370
|
+
assert_equal 1, emits.count
|
371
|
+
assert_equal 'lookup.test', emits[0][0]
|
372
|
+
assert_equal 'myvalue', emits[0][2]['key1']
|
373
|
+
end
|
374
|
+
|
375
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
4
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
5
|
+
|
6
|
+
require 'fluent/test'
|
7
|
+
|
8
|
+
unless ENV.has_key?('VERBOSE')
|
9
|
+
nulllogger = Object.new
|
10
|
+
nulllogger.instance_eval {|obj|
|
11
|
+
def method_missing(method, *args)
|
12
|
+
# pass
|
13
|
+
end
|
14
|
+
}
|
15
|
+
$log = nulllogger
|
16
|
+
end
|
17
|
+
|
18
|
+
class Test::Unit::TestCase
|
19
|
+
end
|
metadata
ADDED
@@ -0,0 +1,101 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-lookup
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Neozaru
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-03-29 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: fluentd
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description: Fluentd custom plugin to replace fields values using lookup table file
|
56
|
+
email:
|
57
|
+
- neozaru@mailoo.org
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- Gemfile
|
63
|
+
- README.md
|
64
|
+
- Rakefile
|
65
|
+
- fluent-plugin-lookup.gemspec
|
66
|
+
- lib/fluent/plugin/out_lookup.rb
|
67
|
+
- test/plugin/correct.csv
|
68
|
+
- test/plugin/duplicates.csv
|
69
|
+
- test/plugin/empty.csv
|
70
|
+
- test/plugin/test_out_lookup.rb
|
71
|
+
- test/test_helper.rb
|
72
|
+
homepage: https://github.com/Neozaru/fluent-plugin-lookup.git
|
73
|
+
licenses:
|
74
|
+
- WTFPL
|
75
|
+
metadata: {}
|
76
|
+
post_install_message:
|
77
|
+
rdoc_options: []
|
78
|
+
require_paths:
|
79
|
+
- lib
|
80
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
81
|
+
requirements:
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: '0'
|
85
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
requirements: []
|
91
|
+
rubyforge_project:
|
92
|
+
rubygems_version: 2.4.5
|
93
|
+
signing_key:
|
94
|
+
specification_version: 4
|
95
|
+
summary: Fluentd custom plugin to replace fields values using lookup table file
|
96
|
+
test_files:
|
97
|
+
- test/plugin/correct.csv
|
98
|
+
- test/plugin/duplicates.csv
|
99
|
+
- test/plugin/empty.csv
|
100
|
+
- test/plugin/test_out_lookup.rb
|
101
|
+
- test/test_helper.rb
|