fluent-plugin-json_expander 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/fluent-plugin-json_expander.gemspec +1 -1
- data/lib/fluent/plugin/out_json_expander.rb +55 -12
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7a35f9e721811377cd62973f2f5cf3577d564b64
|
|
4
|
+
data.tar.gz: 92cf9d852f7c38c992bb472eb1d9b56510928af1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: eaa5671117602bf5d421e1ba48ed0e1c1e3acebce2cd451658ea2e53d710be2329275cc0c58672e299a13ae8bf1025b94d296f44dd5914d3eeafb2eccccf7dd4
|
|
7
|
+
data.tar.gz: 3948c8542fcc827267a476023c7dde83e5e956fb435f500eaa2c1eee4ba3c83a6cd8477dd03798fbdda7c770f72ee7ab856e950aafaab84bb78b684f6a910329
|
|
@@ -28,6 +28,8 @@ class Fluent::JsonExpanderOutput < Fluent::MultiOutput
|
|
|
28
28
|
|
|
29
29
|
@outputs = []
|
|
30
30
|
@mutex = Mutex.new
|
|
31
|
+
@mappings = {}
|
|
32
|
+
@invalid_mapping_keys = []
|
|
31
33
|
|
|
32
34
|
templates = conf.elements.select{|e| e.name == 'template' }
|
|
33
35
|
if templates.size != 1
|
|
@@ -35,6 +37,7 @@ class Fluent::JsonExpanderOutput < Fluent::MultiOutput
|
|
|
35
37
|
end
|
|
36
38
|
|
|
37
39
|
@template = templates.first
|
|
40
|
+
@expand_target_keys = scan_keys(@template)
|
|
38
41
|
end
|
|
39
42
|
|
|
40
43
|
def emit(tag, es, chain)
|
|
@@ -57,24 +60,60 @@ class Fluent::JsonExpanderOutput < Fluent::MultiOutput
|
|
|
57
60
|
chain.next
|
|
58
61
|
end
|
|
59
62
|
|
|
63
|
+
def shutdown
|
|
64
|
+
super
|
|
65
|
+
@mappings.values.each do |output|
|
|
66
|
+
output.shutdown
|
|
67
|
+
end
|
|
68
|
+
@mappings.clear
|
|
69
|
+
end
|
|
70
|
+
|
|
60
71
|
private
|
|
61
72
|
|
|
73
|
+
SCAN_DATA_RE = /\$\{data\[([_a-zA-Z][_a-zA-Z0-9]*)\]\}/
|
|
74
|
+
def scan_keys(elm)
|
|
75
|
+
elm.inject([]) { |dst, (attr, value)|
|
|
76
|
+
dst.concat(value.scan(SCAN_DATA_RE).flatten)
|
|
77
|
+
}.sort.uniq
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def to_mapping_key(data)
|
|
81
|
+
data.select{|k, _| @expand_target_keys.include? k }
|
|
82
|
+
.to_a
|
|
83
|
+
.sort_by(&:first)
|
|
84
|
+
.flatten
|
|
85
|
+
.join("::")
|
|
86
|
+
end
|
|
87
|
+
|
|
62
88
|
def new_output(data)
|
|
63
89
|
o = nil
|
|
64
90
|
t = @template
|
|
91
|
+
map_key = to_mapping_key(data)
|
|
92
|
+
if @invalid_mapping_keys.include?(map_key)
|
|
93
|
+
return o, data
|
|
94
|
+
end
|
|
95
|
+
|
|
65
96
|
begin
|
|
66
|
-
@
|
|
67
|
-
|
|
68
|
-
if
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
97
|
+
o = @mappings[map_key]
|
|
98
|
+
if o
|
|
99
|
+
if @delete_used_key
|
|
100
|
+
@expand_target_keys.each{|k| data.delete(k) }
|
|
101
|
+
end
|
|
102
|
+
else
|
|
103
|
+
@mutex.synchronize do
|
|
104
|
+
e, data = expand_elm(t, data)
|
|
105
|
+
if e
|
|
106
|
+
o = Fluent::Plugin.new_output(@subtype)
|
|
107
|
+
o.configure(e)
|
|
108
|
+
o.start
|
|
109
|
+
|
|
110
|
+
@outputs.push(o)
|
|
111
|
+
@mappings[map_key] = o
|
|
112
|
+
end
|
|
74
113
|
end
|
|
75
|
-
end
|
|
76
114
|
|
|
77
|
-
|
|
115
|
+
log.info "[out_json_expand] Expanded new output: #{@subtype}"
|
|
116
|
+
end
|
|
78
117
|
rescue Fluent::ConfigError => e
|
|
79
118
|
log.error "failed to configure sub output #{@subtype}: #{e.message}"
|
|
80
119
|
log.error e.backtrace.join("\n")
|
|
@@ -87,16 +126,20 @@ class Fluent::JsonExpanderOutput < Fluent::MultiOutput
|
|
|
87
126
|
o = nil
|
|
88
127
|
end
|
|
89
128
|
|
|
129
|
+
unless o
|
|
130
|
+
@invalid_mapping_keys << map_key
|
|
131
|
+
end
|
|
132
|
+
|
|
90
133
|
return o, data
|
|
91
134
|
end
|
|
92
135
|
|
|
93
|
-
|
|
136
|
+
GSUB_DATA_RE = /\$\{data\[(?:[_a-zA-Z][_a-zA-Z0-9]*)\]\}/
|
|
94
137
|
SCAN_KEY_NAME_RE = /\[([_a-zA-Z][_a-zA-Z0-9]*)\]/
|
|
95
138
|
|
|
96
139
|
def expand_elm(template, data)
|
|
97
140
|
attr = {}
|
|
98
141
|
template.each do |k, v|
|
|
99
|
-
v = v.gsub(
|
|
142
|
+
v = v.gsub(GSUB_DATA_RE) do |matched|
|
|
100
143
|
key_matched = matched.scan(SCAN_KEY_NAME_RE)[0]
|
|
101
144
|
if !key_matched or !key_matched[0]
|
|
102
145
|
raise(Fluent::ConfigError, "[BUG] data matched in template, but could not find key name")
|