fluentd 0.14.1 → 0.14.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ChangeLog +110 -1
- data/Rakefile +5 -1
- data/appveyor.yml +7 -1
- data/example/in_forward.conf +4 -0
- data/lib/fluent/compat/exec_util.rb +129 -0
- data/lib/fluent/compat/file_util.rb +54 -0
- data/lib/fluent/compat/filter.rb +21 -3
- data/lib/fluent/compat/formatter.rb +4 -2
- data/lib/fluent/compat/formatter_utils.rb +85 -0
- data/lib/fluent/compat/handle_tag_and_time_mixin.rb +60 -0
- data/lib/fluent/compat/input.rb +1 -3
- data/lib/fluent/compat/output.rb +95 -39
- data/lib/fluent/compat/parser.rb +17 -0
- data/lib/fluent/compat/parser_utils.rb +40 -0
- data/lib/fluent/compat/socket_util.rb +165 -0
- data/lib/fluent/compat/string_util.rb +34 -0
- data/lib/fluent/{test/driver/owner.rb → compat/structured_format_mixin.rb} +5 -11
- data/lib/fluent/config/element.rb +2 -2
- data/lib/fluent/configurable.rb +2 -1
- data/lib/fluent/event.rb +61 -7
- data/lib/fluent/event_router.rb +1 -1
- data/lib/fluent/plugin.rb +7 -7
- data/lib/fluent/plugin/buf_file.rb +5 -2
- data/lib/fluent/plugin/buffer.rb +194 -64
- data/lib/fluent/plugin/buffer/chunk.rb +28 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +5 -21
- data/lib/fluent/plugin/buffer/memory_chunk.rb +1 -11
- data/lib/fluent/plugin/exec_util.rb +2 -112
- data/lib/fluent/plugin/file_util.rb +3 -38
- data/lib/fluent/plugin/file_wrapper.rb +1 -1
- data/lib/fluent/plugin/filter_grep.rb +3 -7
- data/lib/fluent/plugin/filter_record_transformer.rb +5 -5
- data/lib/fluent/plugin/filter_stdout.rb +18 -11
- data/lib/fluent/plugin/formatter.rb +0 -48
- data/lib/fluent/plugin/formatter_csv.rb +7 -8
- data/lib/fluent/plugin/formatter_hash.rb +1 -4
- data/lib/fluent/plugin/formatter_json.rb +1 -4
- data/lib/fluent/plugin/formatter_ltsv.rb +5 -6
- data/lib/fluent/plugin/formatter_msgpack.rb +1 -4
- data/lib/fluent/plugin/formatter_out_file.rb +36 -3
- data/lib/fluent/plugin/formatter_stdout.rb +36 -1
- data/lib/fluent/plugin/in_dummy.rb +9 -2
- data/lib/fluent/plugin/in_exec.rb +20 -57
- data/lib/fluent/plugin/in_forward.rb +4 -3
- data/lib/fluent/plugin/in_object_space.rb +8 -44
- data/lib/fluent/plugin/in_syslog.rb +13 -24
- data/lib/fluent/plugin/in_tail.rb +3 -0
- data/lib/fluent/plugin/out_buffered_stdout.rb +14 -4
- data/lib/fluent/plugin/out_exec.rb +7 -5
- data/lib/fluent/plugin/out_exec_filter.rb +10 -10
- data/lib/fluent/plugin/out_file.rb +1 -3
- data/lib/fluent/plugin/out_forward.rb +38 -57
- data/lib/fluent/plugin/out_stdout.rb +14 -5
- data/lib/fluent/plugin/out_stream.rb +3 -0
- data/lib/fluent/plugin/output.rb +31 -14
- data/lib/fluent/plugin/parser.rb +0 -69
- data/lib/fluent/plugin/parser_apache.rb +10 -6
- data/lib/fluent/plugin/parser_apache_error.rb +8 -3
- data/lib/fluent/plugin/parser_csv.rb +3 -1
- data/lib/fluent/plugin/parser_json.rb +1 -1
- data/lib/fluent/plugin/parser_multiline.rb +5 -3
- data/lib/fluent/plugin/parser_nginx.rb +10 -6
- data/lib/fluent/plugin/parser_regexp.rb +73 -0
- data/lib/fluent/plugin/socket_util.rb +2 -148
- data/lib/fluent/plugin/storage_local.rb +1 -1
- data/lib/fluent/plugin/string_util.rb +3 -18
- data/lib/fluent/plugin_helper.rb +1 -0
- data/lib/fluent/plugin_helper/compat_parameters.rb +166 -41
- data/lib/fluent/plugin_helper/formatter.rb +30 -19
- data/lib/fluent/plugin_helper/inject.rb +25 -12
- data/lib/fluent/plugin_helper/parser.rb +22 -13
- data/lib/fluent/plugin_helper/storage.rb +22 -13
- data/lib/fluent/registry.rb +19 -6
- data/lib/fluent/supervisor.rb +27 -1
- data/lib/fluent/test/driver/base.rb +16 -92
- data/lib/fluent/test/driver/base_owned.rb +17 -53
- data/lib/fluent/test/driver/base_owner.rb +125 -0
- data/lib/fluent/test/driver/filter.rb +24 -2
- data/lib/fluent/test/driver/input.rb +2 -2
- data/lib/fluent/test/driver/multi_output.rb +2 -2
- data/lib/fluent/test/driver/output.rb +3 -5
- data/lib/fluent/test/helpers.rb +25 -0
- data/lib/fluent/test/input_test.rb +4 -4
- data/lib/fluent/test/output_test.rb +3 -3
- data/lib/fluent/version.rb +1 -1
- data/test/config/test_element.rb +135 -6
- data/test/plugin/test_buf_file.rb +71 -3
- data/test/plugin/test_buffer.rb +305 -86
- data/test/plugin/test_buffer_chunk.rb +60 -2
- data/test/plugin/test_buffer_file_chunk.rb +4 -3
- data/test/plugin/test_filter_grep.rb +25 -21
- data/test/plugin/test_filter_record_transformer.rb +75 -67
- data/test/plugin/test_filter_stdout.rb +171 -74
- data/test/plugin/test_formatter_csv.rb +94 -0
- data/test/plugin/test_formatter_json.rb +30 -0
- data/test/plugin/test_formatter_ltsv.rb +52 -0
- data/test/plugin/test_formatter_msgpack.rb +28 -0
- data/test/plugin/test_formatter_out_file.rb +95 -0
- data/test/plugin/test_formatter_single_value.rb +38 -0
- data/test/plugin/test_in_dummy.rb +95 -0
- data/test/plugin/test_in_exec.rb +27 -31
- data/test/plugin/test_in_forward.rb +24 -0
- data/test/plugin/test_in_gc_stat.rb +5 -5
- data/test/plugin/test_in_object_space.rb +4 -4
- data/test/plugin/test_in_syslog.rb +60 -35
- data/test/plugin/test_out_buffered_stdout.rb +17 -3
- data/test/plugin/test_out_forward.rb +93 -5
- data/test/plugin/test_out_stdout.rb +14 -3
- data/test/plugin/test_output_as_buffered_retries.rb +20 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +16 -0
- data/test/plugin/test_output_as_standard.rb +22 -22
- data/test/plugin/test_parser_apache.rb +13 -9
- data/test/plugin/test_parser_apache_error.rb +11 -6
- data/test/plugin/test_parser_csv.rb +35 -25
- data/test/plugin/test_parser_nginx.rb +11 -5
- data/test/plugin/test_parser_regexp.rb +235 -68
- data/test/plugin/test_parser_tsv.rb +54 -58
- data/test/plugin_helper/test_compat_parameters.rb +111 -46
- data/test/plugin_helper/test_formatter.rb +40 -0
- data/test/plugin_helper/test_inject.rb +101 -2
- data/test/plugin_helper/test_parser.rb +40 -0
- data/test/plugin_helper/test_storage.rb +43 -0
- data/test/test_event.rb +93 -0
- data/test/test_event_router.rb +13 -4
- data/test/test_event_time.rb +0 -3
- data/test/test_formatter.rb +7 -164
- data/test/test_plugin_classes.rb +28 -1
- metadata +24 -3
|
@@ -23,88 +23,255 @@ class RegexpParserTest < ::Test::Unit::TestCase
|
|
|
23
23
|
}
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
26
|
+
sub_test_case "Fluent::Compat::TextParser::RegexpParser" do
|
|
27
|
+
def create_driver(regexp, conf = {}, initialize_conf: false)
|
|
28
|
+
if initialize_conf
|
|
29
|
+
Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
|
|
30
|
+
else
|
|
31
|
+
Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
30
34
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
35
|
+
def test_parse_with_typed
|
|
36
|
+
# Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0
|
|
37
|
+
regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
|
|
38
|
+
conf = {
|
|
39
|
+
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
|
40
|
+
'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
|
|
41
|
+
}
|
|
42
|
+
d = create_driver(regexp, conf, initialize_conf: true)
|
|
43
|
+
internal_test_case(d.instance)
|
|
44
|
+
end
|
|
40
45
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
46
|
+
def test_parse_with_configure
|
|
47
|
+
# Specify conf by configure method instaed of intializer
|
|
48
|
+
regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
|
|
49
|
+
conf = {
|
|
50
|
+
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
|
51
|
+
'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
|
|
52
|
+
}
|
|
53
|
+
d = create_driver(regexp, conf)
|
|
54
|
+
internal_test_case(d.instance)
|
|
55
|
+
assert_equal(regexp, d.instance.patterns['format'])
|
|
56
|
+
assert_equal("%d/%b/%Y:%H:%M:%S %z", d.instance.patterns['time_format'])
|
|
57
|
+
end
|
|
44
58
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
59
|
+
def test_parse_with_typed_and_name_separator
|
|
60
|
+
regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
|
|
61
|
+
conf = {
|
|
62
|
+
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
|
63
|
+
'types' => 'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer',
|
|
64
|
+
'types_label_delimiter' => '|'
|
|
65
|
+
}
|
|
66
|
+
d = create_driver(regexp, conf)
|
|
67
|
+
internal_test_case(d.instance)
|
|
54
68
|
end
|
|
55
|
-
end
|
|
56
69
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
70
|
+
def test_parse_with_time_key
|
|
71
|
+
conf = {
|
|
72
|
+
'time_format' => "%Y-%m-%d %H:%M:%S %z",
|
|
73
|
+
'time_key' => 'logtime'
|
|
74
|
+
}
|
|
75
|
+
d = create_driver(/(?<logtime>[^\]]*)/, conf)
|
|
76
|
+
text = '2013-02-28 12:00:00 +0900'
|
|
77
|
+
d.instance.parse(text) do |time, _record|
|
|
78
|
+
assert_equal Fluent::EventTime.parse(text), time
|
|
79
|
+
end
|
|
80
|
+
end
|
|
60
81
|
|
|
61
|
-
|
|
62
|
-
|
|
82
|
+
def test_parse_without_time
|
|
83
|
+
time_at_start = Time.now.to_i
|
|
84
|
+
text = "tagomori_satoshi tagomoris 34\n"
|
|
63
85
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
}
|
|
86
|
+
regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)
|
|
87
|
+
conf = {
|
|
88
|
+
'types' => 'name:string,user:string,age:integer'
|
|
89
|
+
}
|
|
90
|
+
d = create_driver(regexp, conf)
|
|
70
91
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
92
|
+
d.instance.parse(text) { |time, record|
|
|
93
|
+
assert time && time >= time_at_start, "parser puts current time without time input"
|
|
94
|
+
assert_equal "tagomori_satoshi", record["name"]
|
|
95
|
+
assert_equal "tagomoris", record["user"]
|
|
96
|
+
assert_equal 34, record["age"]
|
|
97
|
+
}
|
|
98
|
+
end
|
|
74
99
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
100
|
+
def test_parse_without_time_estimate_curent_event_false
|
|
101
|
+
text = "tagomori_satoshi tagomoris 34\n"
|
|
102
|
+
regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)
|
|
103
|
+
conf = {
|
|
104
|
+
'types' => 'name:string,user:string,age:integer'
|
|
105
|
+
}
|
|
106
|
+
d = create_driver(regexp, conf)
|
|
107
|
+
d.instance.estimate_current_event = false
|
|
108
|
+
d.instance.parse(text) { |time, record|
|
|
109
|
+
assert_equal "tagomori_satoshi", record["name"]
|
|
110
|
+
assert_equal "tagomoris", record["user"]
|
|
111
|
+
assert_equal 34, record["age"]
|
|
79
112
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
113
|
+
assert_nil time, "parser returns nil if configured so"
|
|
114
|
+
}
|
|
115
|
+
end
|
|
83
116
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
117
|
+
def test_parse_with_keep_time_key
|
|
118
|
+
regexp = Regexp.new(%q!(?<time>.*)!)
|
|
119
|
+
conf = {
|
|
120
|
+
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
|
121
|
+
'keep_time_key' => 'true',
|
|
122
|
+
}
|
|
123
|
+
d = create_driver(regexp, conf)
|
|
124
|
+
text = '28/Feb/2013:12:00:00 +0900'
|
|
125
|
+
d.instance.parse(text) do |_time, record|
|
|
126
|
+
assert_equal text, record['time']
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def test_parse_with_keep_time_key_with_typecast
|
|
131
|
+
regexp = Regexp.new(%q!(?<time>.*)!)
|
|
132
|
+
conf = {
|
|
133
|
+
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
|
134
|
+
'keep_time_key' => 'true',
|
|
135
|
+
'types' => 'time:time:%d/%b/%Y:%H:%M:%S %z',
|
|
136
|
+
}
|
|
137
|
+
d = create_driver(regexp, conf)
|
|
138
|
+
text = '28/Feb/2013:12:00:00 +0900'
|
|
139
|
+
d.instance.parse(text) do |_time, record|
|
|
140
|
+
assert_equal 1362020400, record['time']
|
|
141
|
+
end
|
|
94
142
|
end
|
|
95
143
|
end
|
|
96
144
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
145
|
+
sub_test_case "Fluent::Plugin::RegexpParser" do
|
|
146
|
+
def create_driver(conf)
|
|
147
|
+
Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new).configure(conf)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
sub_test_case "configure" do
|
|
151
|
+
def test_default_options
|
|
152
|
+
conf = {
|
|
153
|
+
'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
|
|
154
|
+
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
|
155
|
+
'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
|
|
156
|
+
}
|
|
157
|
+
d = create_driver(conf)
|
|
158
|
+
regexp = d.instance.instance_variable_get(:@regexp)
|
|
159
|
+
assert_equal(0, regexp.options)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
data(
|
|
163
|
+
ignorecase: [{ "ignorecase" => true }, Regexp::IGNORECASE],
|
|
164
|
+
multiline: [{ "multiline" => true }, Regexp::MULTILINE],
|
|
165
|
+
ignorecase_multiline: [{ "ignorecase" => true, "multiline" => true }, Regexp::IGNORECASE | Regexp::MULTILINE],
|
|
166
|
+
)
|
|
167
|
+
def test_options(data)
|
|
168
|
+
regexp_option, expected = data
|
|
169
|
+
conf = {
|
|
170
|
+
'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
|
|
171
|
+
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
|
172
|
+
'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
|
|
173
|
+
}
|
|
174
|
+
conf = conf.merge(regexp_option)
|
|
175
|
+
d = create_driver(conf)
|
|
176
|
+
regexp = d.instance.instance_variable_get(:@regexp)
|
|
177
|
+
assert_equal(expected, regexp.options)
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def test_parse_with_typed
|
|
182
|
+
conf = {
|
|
183
|
+
'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
|
|
184
|
+
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
|
185
|
+
'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
|
|
186
|
+
}
|
|
187
|
+
d = create_driver(conf)
|
|
188
|
+
internal_test_case(d.instance)
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def test_parse_with_typed_and_name_separator
|
|
192
|
+
conf = {
|
|
193
|
+
'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
|
|
194
|
+
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
|
195
|
+
'types' => 'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer',
|
|
196
|
+
'types_label_delimiter' => '|'
|
|
197
|
+
}
|
|
198
|
+
d = create_driver(conf)
|
|
199
|
+
internal_test_case(d.instance)
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def test_parse_with_time_key
|
|
203
|
+
conf = {
|
|
204
|
+
'expression' => %q!/(?<logtime>[^\]]*)/!,
|
|
205
|
+
'time_format' => "%Y-%m-%d %H:%M:%S %z",
|
|
206
|
+
'time_key' => 'logtime'
|
|
207
|
+
}
|
|
208
|
+
d = create_driver(conf)
|
|
209
|
+
text = '2013-02-28 12:00:00 +0900'
|
|
210
|
+
d.instance.parse(text) do |time, _record|
|
|
211
|
+
assert_equal Fluent::EventTime.parse(text), time
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def test_parse_without_time
|
|
216
|
+
time_at_start = Time.now.to_i
|
|
217
|
+
text = "tagomori_satoshi tagomoris 34\n"
|
|
218
|
+
|
|
219
|
+
conf = {
|
|
220
|
+
'expression' => %q!/^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$/!,
|
|
221
|
+
'types' => 'name:string,user:string,age:integer'
|
|
222
|
+
}
|
|
223
|
+
d = create_driver(conf)
|
|
224
|
+
|
|
225
|
+
d.instance.parse(text) { |time, record|
|
|
226
|
+
assert time && time >= time_at_start, "parser puts current time without time input"
|
|
227
|
+
assert_equal "tagomori_satoshi", record["name"]
|
|
228
|
+
assert_equal "tagomoris", record["user"]
|
|
229
|
+
assert_equal 34, record["age"]
|
|
230
|
+
}
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
def test_parse_without_time_estimate_curent_event_false
|
|
234
|
+
text = "tagomori_satoshi tagomoris 34\n"
|
|
235
|
+
conf = {
|
|
236
|
+
'expression' => %q!/^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$/!,
|
|
237
|
+
'types' => 'name:string,user:string,age:integer'
|
|
238
|
+
}
|
|
239
|
+
d = create_driver(conf)
|
|
240
|
+
d.instance.estimate_current_event = false
|
|
241
|
+
d.instance.parse(text) { |time, record|
|
|
242
|
+
assert_equal "tagomori_satoshi", record["name"]
|
|
243
|
+
assert_equal "tagomoris", record["user"]
|
|
244
|
+
assert_equal 34, record["age"]
|
|
245
|
+
|
|
246
|
+
assert_nil time, "parser returns nil if configured so"
|
|
247
|
+
}
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def test_parse_with_keep_time_key
|
|
251
|
+
conf = {
|
|
252
|
+
'expression' => %q!/(?<time>.*)/!,
|
|
253
|
+
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
|
254
|
+
'keep_time_key' => 'true',
|
|
255
|
+
}
|
|
256
|
+
d = create_driver(conf)
|
|
257
|
+
text = '28/Feb/2013:12:00:00 +0900'
|
|
258
|
+
d.instance.parse(text) do |_time, record|
|
|
259
|
+
assert_equal text, record['time']
|
|
260
|
+
end
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
def test_parse_with_keep_time_key_with_typecast
|
|
264
|
+
conf = {
|
|
265
|
+
'expression' => %q!/(?<time>.*)/!,
|
|
266
|
+
'time_format' => "%d/%b/%Y:%H:%M:%S %z",
|
|
267
|
+
'keep_time_key' => 'true',
|
|
268
|
+
'types' => 'time:time:%d/%b/%Y:%H:%M:%S %z',
|
|
269
|
+
}
|
|
270
|
+
d = create_driver(conf)
|
|
271
|
+
text = '28/Feb/2013:12:00:00 +0900'
|
|
272
|
+
d.instance.parse(text) do |_time, record|
|
|
273
|
+
assert_equal 1362020400, record['time']
|
|
274
|
+
end
|
|
108
275
|
end
|
|
109
276
|
end
|
|
110
277
|
end
|
|
@@ -1,57 +1,59 @@
|
|
|
1
1
|
require_relative '../helper'
|
|
2
2
|
require 'fluent/test/driver/parser'
|
|
3
|
-
require 'fluent/plugin/
|
|
3
|
+
require 'fluent/plugin/parser_tsv'
|
|
4
4
|
|
|
5
5
|
class TSVParserTest < ::Test::Unit::TestCase
|
|
6
6
|
def setup
|
|
7
7
|
Fluent::Test.setup
|
|
8
8
|
end
|
|
9
9
|
|
|
10
|
+
def create_driver(conf={})
|
|
11
|
+
Fluent::Test::Driver::Parser.new(Fluent::TextParser::TSVParser).configure(conf)
|
|
12
|
+
end
|
|
13
|
+
|
|
10
14
|
data('array param' => '["a","b"]', 'string param' => 'a,b')
|
|
11
15
|
def test_config_params(param)
|
|
12
|
-
|
|
16
|
+
d = create_driver
|
|
13
17
|
|
|
14
|
-
assert_equal "\t",
|
|
18
|
+
assert_equal "\t", d.instance.delimiter
|
|
15
19
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
+
d = create_driver(
|
|
21
|
+
'keys' => param,
|
|
22
|
+
'delimiter' => ',',
|
|
23
|
+
)
|
|
20
24
|
|
|
21
|
-
assert_equal ['a', 'b'],
|
|
22
|
-
assert_equal ",",
|
|
25
|
+
assert_equal ['a', 'b'], d.instance.keys
|
|
26
|
+
assert_equal ",", d.instance.delimiter
|
|
23
27
|
end
|
|
24
28
|
|
|
25
29
|
data('array param' => '["time","a","b"]', 'string param' => 'time,a,b')
|
|
26
30
|
def test_parse(param)
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
parser.instance.parse("2013/02/28 12:00:00\t192.168.0.1\t111") { |time, record|
|
|
31
|
+
d = create_driver('keys' => param, 'time_key' => 'time')
|
|
32
|
+
d.instance.parse("2013/02/28 12:00:00\t192.168.0.1\t111") { |time, record|
|
|
30
33
|
assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
|
|
31
34
|
assert_equal({
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
+
'a' => '192.168.0.1',
|
|
36
|
+
'b' => '111',
|
|
37
|
+
}, record)
|
|
35
38
|
}
|
|
36
39
|
end
|
|
37
40
|
|
|
38
41
|
def test_parse_with_time
|
|
39
42
|
time_at_start = Time.now.to_i
|
|
40
43
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
parser.instance.parse("192.168.0.1\t111") { |time, record|
|
|
44
|
+
d = create_driver('keys' => 'a,b')
|
|
45
|
+
d.instance.parse("192.168.0.1\t111") { |time, record|
|
|
44
46
|
assert time && time >= time_at_start, "parser puts current time without time input"
|
|
45
47
|
assert_equal({
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
48
|
+
'a' => '192.168.0.1',
|
|
49
|
+
'b' => '111',
|
|
50
|
+
}, record)
|
|
49
51
|
}
|
|
50
52
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)
|
|
54
|
+
d.instance.estimate_current_event = false
|
|
55
|
+
d.configure('keys' => 'a,b', 'time_key' => 'time')
|
|
56
|
+
d.instance.parse("192.168.0.1\t111") { |time, record|
|
|
55
57
|
assert_equal({
|
|
56
58
|
'a' => '192.168.0.1',
|
|
57
59
|
'b' => '111',
|
|
@@ -61,45 +63,41 @@ class TSVParserTest < ::Test::Unit::TestCase
|
|
|
61
63
|
end
|
|
62
64
|
|
|
63
65
|
data(
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
'left blank column' => ["\t@\t@", {"1" => "","2" => "@","3" => "@"}],
|
|
67
|
+
'center blank column' => ["@\t\t@", {"1" => "@","2" => "","3" => "@"}],
|
|
68
|
+
'right blank column' => ["@\t@\t", {"1" => "@","2" => "@","3" => ""}],
|
|
69
|
+
'2 right blank columns' => ["@\t\t", {"1" => "@","2" => "","3" => ""}],
|
|
70
|
+
'left blank columns' => ["\t\t@", {"1" => "","2" => "","3" => "@"}],
|
|
71
|
+
'all blank columns' => ["\t\t", {"1" => "","2" => "","3" => ""}])
|
|
70
72
|
def test_black_column(data)
|
|
71
73
|
line, expected = data
|
|
72
74
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
parser.instance.parse(line) { |time, record|
|
|
75
|
+
d = create_driver('keys' => '1,2,3')
|
|
76
|
+
d.instance.parse(line) { |time, record|
|
|
76
77
|
assert_equal(expected, record)
|
|
77
78
|
}
|
|
78
79
|
end
|
|
79
80
|
|
|
80
81
|
def test_parse_with_keep_time_key
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
)
|
|
82
|
+
d = create_driver(
|
|
83
|
+
'keys'=>'time',
|
|
84
|
+
'time_key'=>'time',
|
|
85
|
+
'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
|
|
86
|
+
'keep_time_key'=>'true',
|
|
87
|
+
)
|
|
88
88
|
text = '28/Feb/2013:12:00:00 +0900'
|
|
89
|
-
|
|
89
|
+
d.instance.parse(text) do |time, record|
|
|
90
90
|
assert_equal text, record['time']
|
|
91
91
|
end
|
|
92
92
|
end
|
|
93
93
|
|
|
94
94
|
data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
|
|
95
|
-
def test_parse_with_null_value_pattern
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
)
|
|
102
|
-
parser.instance.parse("-\tnull\tNULL\t\t--\tnuLL") do |time, record|
|
|
95
|
+
def test_parse_with_null_value_pattern(param)
|
|
96
|
+
d = create_driver(
|
|
97
|
+
'keys'=>param,
|
|
98
|
+
'null_value_pattern'=>'^(-|null|NULL)$'
|
|
99
|
+
)
|
|
100
|
+
d.instance.parse("-\tnull\tNULL\t\t--\tnuLL") do |time, record|
|
|
103
101
|
assert_nil record['a']
|
|
104
102
|
assert_nil record['b']
|
|
105
103
|
assert_nil record['c']
|
|
@@ -110,14 +108,12 @@ class TSVParserTest < ::Test::Unit::TestCase
|
|
|
110
108
|
end
|
|
111
109
|
|
|
112
110
|
data('array param' => '["a","b"]', 'string param' => 'a,b')
|
|
113
|
-
def test_parse_with_null_empty_string
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
)
|
|
120
|
-
parser.instance.parse("\t ") do |time, record|
|
|
111
|
+
def test_parse_with_null_empty_string(param)
|
|
112
|
+
d = create_driver(
|
|
113
|
+
'keys'=>param,
|
|
114
|
+
'null_empty_string'=>true
|
|
115
|
+
)
|
|
116
|
+
d.instance.parse("\t ") do |time, record|
|
|
121
117
|
assert_nil record['a']
|
|
122
118
|
assert_equal record['b'], ' '
|
|
123
119
|
end
|