fluent-plugin-lookup 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +59 -1
- data/fluent-plugin-lookup.gemspec +1 -1
- data/lib/fluent/plugin/out_lookup.rb +24 -4
- data/test/plugin/test_out_lookup.rb +119 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 752a9b8b44e789257e24b27a8d08a2d538397aa0
|
4
|
+
data.tar.gz: 9b02e0d6a7f5f67543561c3e933c6c3fd57573d3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 80769b096d7e29a452bd59c70d4384025350a0d533473a859013f48ef84319a19a06bf5afe0502f75542c60790887fb9ab8057ed5d89302fe05359477583e695
|
7
|
+
data.tar.gz: 8e69986b886057d210bf23f8fe6f6903f6df163dd1fe05f3bf4a9d2efe9b3c49ccfa0004604648f057294f559bffb6aac39b7b385c0688af5fd855ca84905adf
|
data/README.md
CHANGED
@@ -87,5 +87,63 @@ Example of records :
|
|
87
87
|
'foo' => "bar"
|
88
88
|
}
|
89
89
|
```
|
90
|
+
Example 3
|
91
|
+
---------
|
92
|
+
|
93
|
+
```
|
94
|
+
<match *.test>
|
95
|
+
type lookup
|
96
|
+
add_tag_prefix lookup.
|
97
|
+
table_file /usr/share/my/lookup.csv
|
98
|
+
field nested.key1
|
99
|
+
</match>
|
100
|
+
```
|
101
|
+
|
102
|
+
Example of records :
|
103
|
+
```
|
104
|
+
{
|
105
|
+
'nested' => {
|
106
|
+
'key1' => "nicolas",
|
107
|
+
},
|
108
|
+
'foo' => "bar"
|
109
|
+
}
|
110
|
+
```
|
111
|
+
... will output :
|
112
|
+
```
|
113
|
+
{
|
114
|
+
'nested' => {
|
115
|
+
'key1' => "cage",
|
116
|
+
},
|
117
|
+
'foo' => "bar"
|
118
|
+
}
|
119
|
+
```
|
120
|
+
|
121
|
+
Example 4
|
122
|
+
---------
|
123
|
+
|
124
|
+
Renaming key
|
125
|
+
|
126
|
+
```
|
127
|
+
<match *.test>
|
128
|
+
type lookup
|
129
|
+
add_tag_prefix lookup.
|
130
|
+
table_file /usr/share/my/lookup.csv
|
131
|
+
field key1
|
132
|
+
rename_key true
|
133
|
+
</match>
|
134
|
+
```
|
135
|
+
|
136
|
+
Example of records :
|
137
|
+
```
|
138
|
+
{
|
139
|
+
'input' => "nicolas",
|
140
|
+
'foo' => "bar"
|
141
|
+
}
|
142
|
+
```
|
143
|
+
... will output :
|
144
|
+
```
|
145
|
+
{
|
146
|
+
'output' => "nicolas",
|
147
|
+
'foo' => "bar"
|
148
|
+
}
|
90
149
|
|
91
|
-
Since *output_field* is not defined, the input *field* value is replaced.
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "fluent-plugin-lookup"
|
3
|
-
spec.version = "0.0.
|
3
|
+
spec.version = "0.0.3"
|
4
4
|
spec.authors = ["Neozaru"]
|
5
5
|
spec.email = ["neozaru@mailoo.org"]
|
6
6
|
spec.description = %q{Fluentd custom plugin to replace fields values using lookup table file}
|
@@ -11,6 +11,7 @@ module Fluent
|
|
11
11
|
config_param :field, :string, :default => nil
|
12
12
|
config_param :output_field, :string, :default => nil
|
13
13
|
config_param :strict, :bool, :default => false
|
14
|
+
config_param :rename_key, :bool, :default => false
|
14
15
|
|
15
16
|
def handle_row(lookup_table, row)
|
16
17
|
if (row.length < 2)
|
@@ -53,6 +54,11 @@ module Fluent
|
|
53
54
|
def configure(conf)
|
54
55
|
super
|
55
56
|
|
57
|
+
@assign_method = method(:assign)
|
58
|
+
@assign_self_method = method(:assign_self)
|
59
|
+
@return_method = method(:return)
|
60
|
+
@rename_method = method(:rename)
|
61
|
+
|
56
62
|
if (field.nil? || table_file.nil?)
|
57
63
|
raise ConfigError, "lookup: Both 'field', and 'table_file' are required to be set."
|
58
64
|
end
|
@@ -60,6 +66,11 @@ module Fluent
|
|
60
66
|
@lookup_table = create_lookup_table(table_file)
|
61
67
|
@field = field.split(".")
|
62
68
|
|
69
|
+
if (rename_key)
|
70
|
+
@filter_method = method(:filter_rename_key)
|
71
|
+
return
|
72
|
+
end
|
73
|
+
|
63
74
|
if (output_field.nil?)
|
64
75
|
@filter_method = method(:filter_no_output)
|
65
76
|
else
|
@@ -67,9 +78,6 @@ module Fluent
|
|
67
78
|
@filter_method = method(:filter_with_output)
|
68
79
|
end
|
69
80
|
|
70
|
-
@assign_method = method(:assign)
|
71
|
-
@assign_self_method = method(:assign_self)
|
72
|
-
@return_method = method(:return)
|
73
81
|
|
74
82
|
end
|
75
83
|
|
@@ -93,10 +101,17 @@ module Fluent
|
|
93
101
|
record[key] = process(value) || value
|
94
102
|
end
|
95
103
|
|
96
|
-
def return(record, key,
|
104
|
+
def return(record, key, value_nouse)
|
97
105
|
return record[key]
|
98
106
|
end
|
99
107
|
|
108
|
+
def rename(record, key, value_nouse)
|
109
|
+
new_key = process(key) || return
|
110
|
+
field_value = record[key]
|
111
|
+
record.delete(key)
|
112
|
+
record[new_key] = field_value
|
113
|
+
end
|
114
|
+
|
100
115
|
|
101
116
|
def filter_record(tag, time, record)
|
102
117
|
super(tag, time, record)
|
@@ -116,6 +131,11 @@ module Fluent
|
|
116
131
|
end
|
117
132
|
end
|
118
133
|
|
134
|
+
# Rename key (will NOT copy or move the field, just rename the key)
|
135
|
+
def filter_rename_key(record)
|
136
|
+
value = dig_cb(record, @field, nil, false, @rename_method)
|
137
|
+
end
|
138
|
+
|
119
139
|
# Generic function to dig into map.
|
120
140
|
def dig_cb(record, path, value, alter, cb)
|
121
141
|
digged_record = record
|
@@ -415,6 +415,39 @@ class LookupOutputTest < Test::Unit::TestCase
|
|
415
415
|
assert_equal 'lookup.test', emits[0][0]
|
416
416
|
assert_equal nil, emits[0][2]['nested']
|
417
417
|
assert_equal nil, emits[0][2]['new']
|
418
|
+
assert_equal 'bar', emits[0][2]['foo']
|
419
|
+
end
|
420
|
+
|
421
|
+
# Checks that the record is not modified if the input_field is not found
|
422
|
+
def test_emit_nested_with_partial_output_structure
|
423
|
+
d = create_driver(%[
|
424
|
+
add_tag_prefix lookup.
|
425
|
+
table_file #{@correct_file}
|
426
|
+
field nested.key1
|
427
|
+
output_field new.foo
|
428
|
+
])
|
429
|
+
|
430
|
+
record = {
|
431
|
+
'foo' => "bar",
|
432
|
+
'nested' => {
|
433
|
+
'key1' => "nicolas",
|
434
|
+
},
|
435
|
+
'new' => {
|
436
|
+
'field1' => "value1",
|
437
|
+
'field2' => "value2"
|
438
|
+
}
|
439
|
+
}
|
440
|
+
|
441
|
+
d.run { d.emit(record) }
|
442
|
+
emits = d.emits
|
443
|
+
|
444
|
+
assert_equal 1, emits.count
|
445
|
+
assert_equal 'lookup.test', emits[0][0]
|
446
|
+
assert_equal 'nicolas', emits[0][2]['nested']['key1']
|
447
|
+
assert_equal 'cage', emits[0][2]['new']['foo']
|
448
|
+
assert_equal 'value1', emits[0][2]['new']['field1']
|
449
|
+
assert_equal 'value2', emits[0][2]['new']['field2']
|
450
|
+
|
418
451
|
end
|
419
452
|
|
420
453
|
# Checks that the record is not modified if the input_field is not found
|
@@ -457,4 +490,90 @@ class LookupOutputTest < Test::Unit::TestCase
|
|
457
490
|
assert_equal 'myvalue', emits[0][2]['key1']
|
458
491
|
end
|
459
492
|
|
493
|
+
def test_emit_rename_key
|
494
|
+
d = create_driver(%[
|
495
|
+
add_tag_prefix lookup.
|
496
|
+
table_file #{@correct_file}
|
497
|
+
field nicolas
|
498
|
+
rename_key true
|
499
|
+
])
|
500
|
+
|
501
|
+
record = {
|
502
|
+
'nicolas' => "bar"
|
503
|
+
}
|
504
|
+
|
505
|
+
d.run { d.emit(record) }
|
506
|
+
emits = d.emits
|
507
|
+
|
508
|
+
assert_equal 1, emits.count
|
509
|
+
assert_equal 'lookup.test', emits[0][0]
|
510
|
+
assert_equal 'bar', emits[0][2]['cage']
|
511
|
+
assert_equal nil, emits[0][2]['nicolas']
|
512
|
+
end
|
513
|
+
|
514
|
+
def test_emit_rename_key_erase
|
515
|
+
d = create_driver(%[
|
516
|
+
add_tag_prefix lookup.
|
517
|
+
table_file #{@correct_file}
|
518
|
+
field nicolas
|
519
|
+
rename_key true
|
520
|
+
])
|
521
|
+
|
522
|
+
record = {
|
523
|
+
'nicolas' => "bar",
|
524
|
+
'cage' => "foo"
|
525
|
+
}
|
526
|
+
|
527
|
+
d.run { d.emit(record) }
|
528
|
+
emits = d.emits
|
529
|
+
|
530
|
+
assert_equal 1, emits.count
|
531
|
+
assert_equal 'lookup.test', emits[0][0]
|
532
|
+
assert_equal 'bar', emits[0][2]['cage']
|
533
|
+
assert_equal nil, emits[0][2]['nicolas']
|
534
|
+
end
|
535
|
+
|
536
|
+
def test_emit_rename_key_nested
|
537
|
+
d = create_driver(%[
|
538
|
+
add_tag_prefix lookup.
|
539
|
+
table_file #{@correct_file}
|
540
|
+
field nested.nicolas
|
541
|
+
rename_key true
|
542
|
+
])
|
543
|
+
|
544
|
+
record = {
|
545
|
+
'nested' => {
|
546
|
+
'nicolas' => "bar",
|
547
|
+
}
|
548
|
+
}
|
549
|
+
|
550
|
+
d.run { d.emit(record) }
|
551
|
+
emits = d.emits
|
552
|
+
|
553
|
+
assert_equal 1, emits.count
|
554
|
+
assert_equal 'lookup.test', emits[0][0]
|
555
|
+
assert_equal 'bar', emits[0][2]['nested']['cage']
|
556
|
+
assert_equal nil, emits[0][2]['nested']['nicolas']
|
557
|
+
end
|
558
|
+
|
559
|
+
def test_emit_rename_key_not_existing
|
560
|
+
d = create_driver(%[
|
561
|
+
add_tag_prefix lookup.
|
562
|
+
table_file #{@correct_file}
|
563
|
+
field dummy
|
564
|
+
rename_key true
|
565
|
+
])
|
566
|
+
|
567
|
+
record = {
|
568
|
+
'nicolas' => "bar"
|
569
|
+
}
|
570
|
+
|
571
|
+
d.run { d.emit(record) }
|
572
|
+
emits = d.emits
|
573
|
+
|
574
|
+
assert_equal 1, emits.count
|
575
|
+
assert_equal 'lookup.test', emits[0][0]
|
576
|
+
assert_equal 'bar', emits[0][2]['nicolas']
|
577
|
+
end
|
578
|
+
|
460
579
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-lookup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Neozaru
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-04-
|
11
|
+
date: 2015-04-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|