fluent-plugin-json_expander 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/fluent-plugin-json_expander.gemspec +1 -1
- data/lib/fluent/plugin/out_json_expander.rb +55 -12
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7a35f9e721811377cd62973f2f5cf3577d564b64
|
|
4
|
+
data.tar.gz: 92cf9d852f7c38c992bb472eb1d9b56510928af1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: eaa5671117602bf5d421e1ba48ed0e1c1e3acebce2cd451658ea2e53d710be2329275cc0c58672e299a13ae8bf1025b94d296f44dd5914d3eeafb2eccccf7dd4
|
|
7
|
+
data.tar.gz: 3948c8542fcc827267a476023c7dde83e5e956fb435f500eaa2c1eee4ba3c83a6cd8477dd03798fbdda7c770f72ee7ab856e950aafaab84bb78b684f6a910329
|
|
@@ -28,6 +28,8 @@ class Fluent::JsonExpanderOutput < Fluent::MultiOutput
|
|
|
28
28
|
|
|
29
29
|
@outputs = []
|
|
30
30
|
@mutex = Mutex.new
|
|
31
|
+
@mappings = {}
|
|
32
|
+
@invalid_mapping_keys = []
|
|
31
33
|
|
|
32
34
|
templates = conf.elements.select{|e| e.name == 'template' }
|
|
33
35
|
if templates.size != 1
|
|
@@ -35,6 +37,7 @@ class Fluent::JsonExpanderOutput < Fluent::MultiOutput
|
|
|
35
37
|
end
|
|
36
38
|
|
|
37
39
|
@template = templates.first
|
|
40
|
+
@expand_target_keys = scan_keys(@template)
|
|
38
41
|
end
|
|
39
42
|
|
|
40
43
|
def emit(tag, es, chain)
|
|
@@ -57,24 +60,60 @@ class Fluent::JsonExpanderOutput < Fluent::MultiOutput
|
|
|
57
60
|
chain.next
|
|
58
61
|
end
|
|
59
62
|
|
|
63
|
+
def shutdown
|
|
64
|
+
super
|
|
65
|
+
@mappings.values.each do |output|
|
|
66
|
+
output.shutdown
|
|
67
|
+
end
|
|
68
|
+
@mappings.clear
|
|
69
|
+
end
|
|
70
|
+
|
|
60
71
|
private
|
|
61
72
|
|
|
73
|
+
SCAN_DATA_RE = /\$\{data\[([_a-zA-Z][_a-zA-Z0-9]*)\]\}/
|
|
74
|
+
def scan_keys(elm)
|
|
75
|
+
elm.inject([]) { |dst, (attr, value)|
|
|
76
|
+
dst.concat(value.scan(SCAN_DATA_RE).flatten)
|
|
77
|
+
}.sort.uniq
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def to_mapping_key(data)
|
|
81
|
+
data.select{|k, _| @expand_target_keys.include? k }
|
|
82
|
+
.to_a
|
|
83
|
+
.sort_by(&:first)
|
|
84
|
+
.flatten
|
|
85
|
+
.join("::")
|
|
86
|
+
end
|
|
87
|
+
|
|
62
88
|
def new_output(data)
|
|
63
89
|
o = nil
|
|
64
90
|
t = @template
|
|
91
|
+
map_key = to_mapping_key(data)
|
|
92
|
+
if @invalid_mapping_keys.include?(map_key)
|
|
93
|
+
return o, data
|
|
94
|
+
end
|
|
95
|
+
|
|
65
96
|
begin
|
|
66
|
-
@
|
|
67
|
-
|
|
68
|
-
if
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
97
|
+
o = @mappings[map_key]
|
|
98
|
+
if o
|
|
99
|
+
if @delete_used_key
|
|
100
|
+
@expand_target_keys.each{|k| data.delete(k) }
|
|
101
|
+
end
|
|
102
|
+
else
|
|
103
|
+
@mutex.synchronize do
|
|
104
|
+
e, data = expand_elm(t, data)
|
|
105
|
+
if e
|
|
106
|
+
o = Fluent::Plugin.new_output(@subtype)
|
|
107
|
+
o.configure(e)
|
|
108
|
+
o.start
|
|
109
|
+
|
|
110
|
+
@outputs.push(o)
|
|
111
|
+
@mappings[map_key] = o
|
|
112
|
+
end
|
|
74
113
|
end
|
|
75
|
-
end
|
|
76
114
|
|
|
77
|
-
|
|
115
|
+
log.info "[out_json_expand] Expanded new output: #{@subtype}"
|
|
116
|
+
end
|
|
78
117
|
rescue Fluent::ConfigError => e
|
|
79
118
|
log.error "failed to configure sub output #{@subtype}: #{e.message}"
|
|
80
119
|
log.error e.backtrace.join("\n")
|
|
@@ -87,16 +126,20 @@ class Fluent::JsonExpanderOutput < Fluent::MultiOutput
|
|
|
87
126
|
o = nil
|
|
88
127
|
end
|
|
89
128
|
|
|
129
|
+
unless o
|
|
130
|
+
@invalid_mapping_keys << map_key
|
|
131
|
+
end
|
|
132
|
+
|
|
90
133
|
return o, data
|
|
91
134
|
end
|
|
92
135
|
|
|
93
|
-
|
|
136
|
+
GSUB_DATA_RE = /\$\{data\[(?:[_a-zA-Z][_a-zA-Z0-9]*)\]\}/
|
|
94
137
|
SCAN_KEY_NAME_RE = /\[([_a-zA-Z][_a-zA-Z0-9]*)\]/
|
|
95
138
|
|
|
96
139
|
def expand_elm(template, data)
|
|
97
140
|
attr = {}
|
|
98
141
|
template.each do |k, v|
|
|
99
|
-
v = v.gsub(
|
|
142
|
+
v = v.gsub(GSUB_DATA_RE) do |matched|
|
|
100
143
|
key_matched = matched.scan(SCAN_KEY_NAME_RE)[0]
|
|
101
144
|
if !key_matched or !key_matched[0]
|
|
102
145
|
raise(Fluent::ConfigError, "[BUG] data matched in template, but could not find key name")
|