fluentd 0.14.1 → 0.14.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. checksums.yaml +4 -4
  2. data/ChangeLog +110 -1
  3. data/Rakefile +5 -1
  4. data/appveyor.yml +7 -1
  5. data/example/in_forward.conf +4 -0
  6. data/lib/fluent/compat/exec_util.rb +129 -0
  7. data/lib/fluent/compat/file_util.rb +54 -0
  8. data/lib/fluent/compat/filter.rb +21 -3
  9. data/lib/fluent/compat/formatter.rb +4 -2
  10. data/lib/fluent/compat/formatter_utils.rb +85 -0
  11. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +60 -0
  12. data/lib/fluent/compat/input.rb +1 -3
  13. data/lib/fluent/compat/output.rb +95 -39
  14. data/lib/fluent/compat/parser.rb +17 -0
  15. data/lib/fluent/compat/parser_utils.rb +40 -0
  16. data/lib/fluent/compat/socket_util.rb +165 -0
  17. data/lib/fluent/compat/string_util.rb +34 -0
  18. data/lib/fluent/{test/driver/owner.rb → compat/structured_format_mixin.rb} +5 -11
  19. data/lib/fluent/config/element.rb +2 -2
  20. data/lib/fluent/configurable.rb +2 -1
  21. data/lib/fluent/event.rb +61 -7
  22. data/lib/fluent/event_router.rb +1 -1
  23. data/lib/fluent/plugin.rb +7 -7
  24. data/lib/fluent/plugin/buf_file.rb +5 -2
  25. data/lib/fluent/plugin/buffer.rb +194 -64
  26. data/lib/fluent/plugin/buffer/chunk.rb +28 -3
  27. data/lib/fluent/plugin/buffer/file_chunk.rb +5 -21
  28. data/lib/fluent/plugin/buffer/memory_chunk.rb +1 -11
  29. data/lib/fluent/plugin/exec_util.rb +2 -112
  30. data/lib/fluent/plugin/file_util.rb +3 -38
  31. data/lib/fluent/plugin/file_wrapper.rb +1 -1
  32. data/lib/fluent/plugin/filter_grep.rb +3 -7
  33. data/lib/fluent/plugin/filter_record_transformer.rb +5 -5
  34. data/lib/fluent/plugin/filter_stdout.rb +18 -11
  35. data/lib/fluent/plugin/formatter.rb +0 -48
  36. data/lib/fluent/plugin/formatter_csv.rb +7 -8
  37. data/lib/fluent/plugin/formatter_hash.rb +1 -4
  38. data/lib/fluent/plugin/formatter_json.rb +1 -4
  39. data/lib/fluent/plugin/formatter_ltsv.rb +5 -6
  40. data/lib/fluent/plugin/formatter_msgpack.rb +1 -4
  41. data/lib/fluent/plugin/formatter_out_file.rb +36 -3
  42. data/lib/fluent/plugin/formatter_stdout.rb +36 -1
  43. data/lib/fluent/plugin/in_dummy.rb +9 -2
  44. data/lib/fluent/plugin/in_exec.rb +20 -57
  45. data/lib/fluent/plugin/in_forward.rb +4 -3
  46. data/lib/fluent/plugin/in_object_space.rb +8 -44
  47. data/lib/fluent/plugin/in_syslog.rb +13 -24
  48. data/lib/fluent/plugin/in_tail.rb +3 -0
  49. data/lib/fluent/plugin/out_buffered_stdout.rb +14 -4
  50. data/lib/fluent/plugin/out_exec.rb +7 -5
  51. data/lib/fluent/plugin/out_exec_filter.rb +10 -10
  52. data/lib/fluent/plugin/out_file.rb +1 -3
  53. data/lib/fluent/plugin/out_forward.rb +38 -57
  54. data/lib/fluent/plugin/out_stdout.rb +14 -5
  55. data/lib/fluent/plugin/out_stream.rb +3 -0
  56. data/lib/fluent/plugin/output.rb +31 -14
  57. data/lib/fluent/plugin/parser.rb +0 -69
  58. data/lib/fluent/plugin/parser_apache.rb +10 -6
  59. data/lib/fluent/plugin/parser_apache_error.rb +8 -3
  60. data/lib/fluent/plugin/parser_csv.rb +3 -1
  61. data/lib/fluent/plugin/parser_json.rb +1 -1
  62. data/lib/fluent/plugin/parser_multiline.rb +5 -3
  63. data/lib/fluent/plugin/parser_nginx.rb +10 -6
  64. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  65. data/lib/fluent/plugin/socket_util.rb +2 -148
  66. data/lib/fluent/plugin/storage_local.rb +1 -1
  67. data/lib/fluent/plugin/string_util.rb +3 -18
  68. data/lib/fluent/plugin_helper.rb +1 -0
  69. data/lib/fluent/plugin_helper/compat_parameters.rb +166 -41
  70. data/lib/fluent/plugin_helper/formatter.rb +30 -19
  71. data/lib/fluent/plugin_helper/inject.rb +25 -12
  72. data/lib/fluent/plugin_helper/parser.rb +22 -13
  73. data/lib/fluent/plugin_helper/storage.rb +22 -13
  74. data/lib/fluent/registry.rb +19 -6
  75. data/lib/fluent/supervisor.rb +27 -1
  76. data/lib/fluent/test/driver/base.rb +16 -92
  77. data/lib/fluent/test/driver/base_owned.rb +17 -53
  78. data/lib/fluent/test/driver/base_owner.rb +125 -0
  79. data/lib/fluent/test/driver/filter.rb +24 -2
  80. data/lib/fluent/test/driver/input.rb +2 -2
  81. data/lib/fluent/test/driver/multi_output.rb +2 -2
  82. data/lib/fluent/test/driver/output.rb +3 -5
  83. data/lib/fluent/test/helpers.rb +25 -0
  84. data/lib/fluent/test/input_test.rb +4 -4
  85. data/lib/fluent/test/output_test.rb +3 -3
  86. data/lib/fluent/version.rb +1 -1
  87. data/test/config/test_element.rb +135 -6
  88. data/test/plugin/test_buf_file.rb +71 -3
  89. data/test/plugin/test_buffer.rb +305 -86
  90. data/test/plugin/test_buffer_chunk.rb +60 -2
  91. data/test/plugin/test_buffer_file_chunk.rb +4 -3
  92. data/test/plugin/test_filter_grep.rb +25 -21
  93. data/test/plugin/test_filter_record_transformer.rb +75 -67
  94. data/test/plugin/test_filter_stdout.rb +171 -74
  95. data/test/plugin/test_formatter_csv.rb +94 -0
  96. data/test/plugin/test_formatter_json.rb +30 -0
  97. data/test/plugin/test_formatter_ltsv.rb +52 -0
  98. data/test/plugin/test_formatter_msgpack.rb +28 -0
  99. data/test/plugin/test_formatter_out_file.rb +95 -0
  100. data/test/plugin/test_formatter_single_value.rb +38 -0
  101. data/test/plugin/test_in_dummy.rb +95 -0
  102. data/test/plugin/test_in_exec.rb +27 -31
  103. data/test/plugin/test_in_forward.rb +24 -0
  104. data/test/plugin/test_in_gc_stat.rb +5 -5
  105. data/test/plugin/test_in_object_space.rb +4 -4
  106. data/test/plugin/test_in_syslog.rb +60 -35
  107. data/test/plugin/test_out_buffered_stdout.rb +17 -3
  108. data/test/plugin/test_out_forward.rb +93 -5
  109. data/test/plugin/test_out_stdout.rb +14 -3
  110. data/test/plugin/test_output_as_buffered_retries.rb +20 -0
  111. data/test/plugin/test_output_as_buffered_secondary.rb +16 -0
  112. data/test/plugin/test_output_as_standard.rb +22 -22
  113. data/test/plugin/test_parser_apache.rb +13 -9
  114. data/test/plugin/test_parser_apache_error.rb +11 -6
  115. data/test/plugin/test_parser_csv.rb +35 -25
  116. data/test/plugin/test_parser_nginx.rb +11 -5
  117. data/test/plugin/test_parser_regexp.rb +235 -68
  118. data/test/plugin/test_parser_tsv.rb +54 -58
  119. data/test/plugin_helper/test_compat_parameters.rb +111 -46
  120. data/test/plugin_helper/test_formatter.rb +40 -0
  121. data/test/plugin_helper/test_inject.rb +101 -2
  122. data/test/plugin_helper/test_parser.rb +40 -0
  123. data/test/plugin_helper/test_storage.rb +43 -0
  124. data/test/test_event.rb +93 -0
  125. data/test/test_event_router.rb +13 -4
  126. data/test/test_event_time.rb +0 -3
  127. data/test/test_formatter.rb +7 -164
  128. data/test/test_plugin_classes.rb +28 -1
  129. metadata +24 -3
@@ -23,88 +23,255 @@ class RegexpParserTest < ::Test::Unit::TestCase
23
23
  }
24
24
  end
25
25
 
26
- def test_parse_with_typed
27
- # Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0
28
- internal_test_case(Fluent::Plugin::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'))
29
- end
26
+ sub_test_case "Fluent::Compat::TextParser::RegexpParser" do
27
+ def create_driver(regexp, conf = {}, initialize_conf: false)
28
+ if initialize_conf
29
+ Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
30
+ else
31
+ Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
32
+ end
33
+ end
30
34
 
31
- def test_parse_with_configure
32
- # Specify conf by configure method instaed of intializer
33
- regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
34
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new(regexp))
35
- parser.configure('time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer')
36
- internal_test_case(parser.instance)
37
- assert_equal(regexp, parser.instance.patterns['format'])
38
- assert_equal("%d/%b/%Y:%H:%M:%S %z", parser.instance.patterns['time_format'])
39
- end
35
+ def test_parse_with_typed
36
+ # Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0
37
+ regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
38
+ conf = {
39
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
40
+ 'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
41
+ }
42
+ d = create_driver(regexp, conf, initialize_conf: true)
43
+ internal_test_case(d.instance)
44
+ end
40
45
 
41
- def test_parse_with_typed_and_name_separator
42
- internal_test_case(Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|')).instance)
43
- end
46
+ def test_parse_with_configure
47
+ # Specify conf by configure method instaed of intializer
48
+ regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
49
+ conf = {
50
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
51
+ 'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
52
+ }
53
+ d = create_driver(regexp, conf)
54
+ internal_test_case(d.instance)
55
+ assert_equal(regexp, d.instance.patterns['format'])
56
+ assert_equal("%d/%b/%Y:%H:%M:%S %z", d.instance.patterns['time_format'])
57
+ end
44
58
 
45
- def test_parse_with_time_key
46
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new(/(?<logtime>[^\]]*)/))
47
- parser.instance.configure(
48
- 'time_format'=>"%Y-%m-%d %H:%M:%S %z",
49
- 'time_key'=>'logtime',
50
- )
51
- text = '2013-02-28 12:00:00 +0900'
52
- parser.instance.parse(text) do |time, record|
53
- assert_equal Fluent::EventTime.parse(text), time
59
+ def test_parse_with_typed_and_name_separator
60
+ regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
61
+ conf = {
62
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
63
+ 'types' => 'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer',
64
+ 'types_label_delimiter' => '|'
65
+ }
66
+ d = create_driver(regexp, conf)
67
+ internal_test_case(d.instance)
54
68
  end
55
- end
56
69
 
57
- def test_parse_without_time
58
- time_at_start = Time.now.to_i
59
- text = "tagomori_satoshi tagomoris 34\n"
70
+ def test_parse_with_time_key
71
+ conf = {
72
+ 'time_format' => "%Y-%m-%d %H:%M:%S %z",
73
+ 'time_key' => 'logtime'
74
+ }
75
+ d = create_driver(/(?<logtime>[^\]]*)/, conf)
76
+ text = '2013-02-28 12:00:00 +0900'
77
+ d.instance.parse(text) do |time, _record|
78
+ assert_equal Fluent::EventTime.parse(text), time
79
+ end
80
+ end
60
81
 
61
- parser = Fluent::Plugin::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
62
- parser.configure('types'=>'name:string,user:string,age:integer')
82
+ def test_parse_without_time
83
+ time_at_start = Time.now.to_i
84
+ text = "tagomori_satoshi tagomoris 34\n"
63
85
 
64
- parser.parse(text) { |time, record|
65
- assert time && time >= time_at_start, "parser puts current time without time input"
66
- assert_equal "tagomori_satoshi", record["name"]
67
- assert_equal "tagomoris", record["user"]
68
- assert_equal 34, record["age"]
69
- }
86
+ regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)
87
+ conf = {
88
+ 'types' => 'name:string,user:string,age:integer'
89
+ }
90
+ d = create_driver(regexp, conf)
70
91
 
71
- parser2 = Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)))
72
- parser2.configure('types'=>'name:string,user:string,age:integer')
73
- parser2.instance.estimate_current_event = false
92
+ d.instance.parse(text) { |time, record|
93
+ assert time && time >= time_at_start, "parser puts current time without time input"
94
+ assert_equal "tagomori_satoshi", record["name"]
95
+ assert_equal "tagomoris", record["user"]
96
+ assert_equal 34, record["age"]
97
+ }
98
+ end
74
99
 
75
- parser2.instance.parse(text) { |time, record|
76
- assert_equal "tagomori_satoshi", record["name"]
77
- assert_equal "tagomoris", record["user"]
78
- assert_equal 34, record["age"]
100
+ def test_parse_without_time_estimate_curent_event_false
101
+ text = "tagomori_satoshi tagomoris 34\n"
102
+ regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)
103
+ conf = {
104
+ 'types' => 'name:string,user:string,age:integer'
105
+ }
106
+ d = create_driver(regexp, conf)
107
+ d.instance.estimate_current_event = false
108
+ d.instance.parse(text) { |time, record|
109
+ assert_equal "tagomori_satoshi", record["name"]
110
+ assert_equal "tagomoris", record["user"]
111
+ assert_equal 34, record["age"]
79
112
 
80
- assert_nil time, "parser returns nil if configured so"
81
- }
82
- end
113
+ assert_nil time, "parser returns nil if configured so"
114
+ }
115
+ end
83
116
 
84
- def test_parse_with_keep_time_key
85
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new(
86
- Regexp.new(%q!(?<time>.*)!),
87
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
88
- 'keep_time_key'=>'true',
89
- )
90
- )
91
- text = '28/Feb/2013:12:00:00 +0900'
92
- parser.instance.parse(text) do |time, record|
93
- assert_equal text, record['time']
117
+ def test_parse_with_keep_time_key
118
+ regexp = Regexp.new(%q!(?<time>.*)!)
119
+ conf = {
120
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
121
+ 'keep_time_key' => 'true',
122
+ }
123
+ d = create_driver(regexp, conf)
124
+ text = '28/Feb/2013:12:00:00 +0900'
125
+ d.instance.parse(text) do |_time, record|
126
+ assert_equal text, record['time']
127
+ end
128
+ end
129
+
130
+ def test_parse_with_keep_time_key_with_typecast
131
+ regexp = Regexp.new(%q!(?<time>.*)!)
132
+ conf = {
133
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
134
+ 'keep_time_key' => 'true',
135
+ 'types' => 'time:time:%d/%b/%Y:%H:%M:%S %z',
136
+ }
137
+ d = create_driver(regexp, conf)
138
+ text = '28/Feb/2013:12:00:00 +0900'
139
+ d.instance.parse(text) do |_time, record|
140
+ assert_equal 1362020400, record['time']
141
+ end
94
142
  end
95
143
  end
96
144
 
97
- def test_parse_with_keep_time_key_with_typecast
98
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new(
99
- Regexp.new(%q!(?<time>.*)!),
100
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
101
- 'keep_time_key'=>'true',
102
- 'types'=>'time:time:%d/%b/%Y:%H:%M:%S %z',
103
- )
104
- )
105
- text = '28/Feb/2013:12:00:00 +0900'
106
- parser.instance.parse(text) do |time, record|
107
- assert_equal 1362020400, record['time']
145
+ sub_test_case "Fluent::Plugin::RegexpParser" do
146
+ def create_driver(conf)
147
+ Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new).configure(conf)
148
+ end
149
+
150
+ sub_test_case "configure" do
151
+ def test_default_options
152
+ conf = {
153
+ 'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
154
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
155
+ 'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
156
+ }
157
+ d = create_driver(conf)
158
+ regexp = d.instance.instance_variable_get(:@regexp)
159
+ assert_equal(0, regexp.options)
160
+ end
161
+
162
+ data(
163
+ ignorecase: [{ "ignorecase" => true }, Regexp::IGNORECASE],
164
+ multiline: [{ "multiline" => true }, Regexp::MULTILINE],
165
+ ignorecase_multiline: [{ "ignorecase" => true, "multiline" => true }, Regexp::IGNORECASE | Regexp::MULTILINE],
166
+ )
167
+ def test_options(data)
168
+ regexp_option, expected = data
169
+ conf = {
170
+ 'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
171
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
172
+ 'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
173
+ }
174
+ conf = conf.merge(regexp_option)
175
+ d = create_driver(conf)
176
+ regexp = d.instance.instance_variable_get(:@regexp)
177
+ assert_equal(expected, regexp.options)
178
+ end
179
+ end
180
+
181
+ def test_parse_with_typed
182
+ conf = {
183
+ 'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
184
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
185
+ 'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
186
+ }
187
+ d = create_driver(conf)
188
+ internal_test_case(d.instance)
189
+ end
190
+
191
+ def test_parse_with_typed_and_name_separator
192
+ conf = {
193
+ 'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
194
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
195
+ 'types' => 'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer',
196
+ 'types_label_delimiter' => '|'
197
+ }
198
+ d = create_driver(conf)
199
+ internal_test_case(d.instance)
200
+ end
201
+
202
+ def test_parse_with_time_key
203
+ conf = {
204
+ 'expression' => %q!/(?<logtime>[^\]]*)/!,
205
+ 'time_format' => "%Y-%m-%d %H:%M:%S %z",
206
+ 'time_key' => 'logtime'
207
+ }
208
+ d = create_driver(conf)
209
+ text = '2013-02-28 12:00:00 +0900'
210
+ d.instance.parse(text) do |time, _record|
211
+ assert_equal Fluent::EventTime.parse(text), time
212
+ end
213
+ end
214
+
215
+ def test_parse_without_time
216
+ time_at_start = Time.now.to_i
217
+ text = "tagomori_satoshi tagomoris 34\n"
218
+
219
+ conf = {
220
+ 'expression' => %q!/^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$/!,
221
+ 'types' => 'name:string,user:string,age:integer'
222
+ }
223
+ d = create_driver(conf)
224
+
225
+ d.instance.parse(text) { |time, record|
226
+ assert time && time >= time_at_start, "parser puts current time without time input"
227
+ assert_equal "tagomori_satoshi", record["name"]
228
+ assert_equal "tagomoris", record["user"]
229
+ assert_equal 34, record["age"]
230
+ }
231
+ end
232
+
233
+ def test_parse_without_time_estimate_curent_event_false
234
+ text = "tagomori_satoshi tagomoris 34\n"
235
+ conf = {
236
+ 'expression' => %q!/^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$/!,
237
+ 'types' => 'name:string,user:string,age:integer'
238
+ }
239
+ d = create_driver(conf)
240
+ d.instance.estimate_current_event = false
241
+ d.instance.parse(text) { |time, record|
242
+ assert_equal "tagomori_satoshi", record["name"]
243
+ assert_equal "tagomoris", record["user"]
244
+ assert_equal 34, record["age"]
245
+
246
+ assert_nil time, "parser returns nil if configured so"
247
+ }
248
+ end
249
+
250
+ def test_parse_with_keep_time_key
251
+ conf = {
252
+ 'expression' => %q!/(?<time>.*)/!,
253
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
254
+ 'keep_time_key' => 'true',
255
+ }
256
+ d = create_driver(conf)
257
+ text = '28/Feb/2013:12:00:00 +0900'
258
+ d.instance.parse(text) do |_time, record|
259
+ assert_equal text, record['time']
260
+ end
261
+ end
262
+
263
+ def test_parse_with_keep_time_key_with_typecast
264
+ conf = {
265
+ 'expression' => %q!/(?<time>.*)/!,
266
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
267
+ 'keep_time_key' => 'true',
268
+ 'types' => 'time:time:%d/%b/%Y:%H:%M:%S %z',
269
+ }
270
+ d = create_driver(conf)
271
+ text = '28/Feb/2013:12:00:00 +0900'
272
+ d.instance.parse(text) do |_time, record|
273
+ assert_equal 1362020400, record['time']
274
+ end
108
275
  end
109
276
  end
110
277
  end
@@ -1,57 +1,59 @@
1
1
  require_relative '../helper'
2
2
  require 'fluent/test/driver/parser'
3
- require 'fluent/plugin/parser'
3
+ require 'fluent/plugin/parser_tsv'
4
4
 
5
5
  class TSVParserTest < ::Test::Unit::TestCase
6
6
  def setup
7
7
  Fluent::Test.setup
8
8
  end
9
9
 
10
+ def create_driver(conf={})
11
+ Fluent::Test::Driver::Parser.new(Fluent::TextParser::TSVParser).configure(conf)
12
+ end
13
+
10
14
  data('array param' => '["a","b"]', 'string param' => 'a,b')
11
15
  def test_config_params(param)
12
- parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::TSVParser)
16
+ d = create_driver
13
17
 
14
- assert_equal "\t", parser.instance.delimiter
18
+ assert_equal "\t", d.instance.delimiter
15
19
 
16
- parser.configure(
17
- 'keys' => param,
18
- 'delimiter' => ',',
19
- )
20
+ d = create_driver(
21
+ 'keys' => param,
22
+ 'delimiter' => ',',
23
+ )
20
24
 
21
- assert_equal ['a', 'b'], parser.instance.keys
22
- assert_equal ",", parser.instance.delimiter
25
+ assert_equal ['a', 'b'], d.instance.keys
26
+ assert_equal ",", d.instance.delimiter
23
27
  end
24
28
 
25
29
  data('array param' => '["time","a","b"]', 'string param' => 'time,a,b')
26
30
  def test_parse(param)
27
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)
28
- parser.configure('keys' => param, 'time_key' => 'time')
29
- parser.instance.parse("2013/02/28 12:00:00\t192.168.0.1\t111") { |time, record|
31
+ d = create_driver('keys' => param, 'time_key' => 'time')
32
+ d.instance.parse("2013/02/28 12:00:00\t192.168.0.1\t111") { |time, record|
30
33
  assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
31
34
  assert_equal({
32
- 'a' => '192.168.0.1',
33
- 'b' => '111',
34
- }, record)
35
+ 'a' => '192.168.0.1',
36
+ 'b' => '111',
37
+ }, record)
35
38
  }
36
39
  end
37
40
 
38
41
  def test_parse_with_time
39
42
  time_at_start = Time.now.to_i
40
43
 
41
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)
42
- parser.configure('keys' => 'a,b')
43
- parser.instance.parse("192.168.0.1\t111") { |time, record|
44
+ d = create_driver('keys' => 'a,b')
45
+ d.instance.parse("192.168.0.1\t111") { |time, record|
44
46
  assert time && time >= time_at_start, "parser puts current time without time input"
45
47
  assert_equal({
46
- 'a' => '192.168.0.1',
47
- 'b' => '111',
48
- }, record)
48
+ 'a' => '192.168.0.1',
49
+ 'b' => '111',
50
+ }, record)
49
51
  }
50
52
 
51
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)
52
- parser.instance.estimate_current_event = false
53
- parser.configure('keys' => 'a,b', 'time_key' => 'time')
54
- parser.instance.parse("192.168.0.1\t111") { |time, record|
53
+ d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)
54
+ d.instance.estimate_current_event = false
55
+ d.configure('keys' => 'a,b', 'time_key' => 'time')
56
+ d.instance.parse("192.168.0.1\t111") { |time, record|
55
57
  assert_equal({
56
58
  'a' => '192.168.0.1',
57
59
  'b' => '111',
@@ -61,45 +63,41 @@ class TSVParserTest < ::Test::Unit::TestCase
61
63
  end
62
64
 
63
65
  data(
64
- 'left blank column' => ["\t@\t@", {"1" => "","2" => "@","3" => "@"}],
65
- 'center blank column' => ["@\t\t@", {"1" => "@","2" => "","3" => "@"}],
66
- 'right blank column' => ["@\t@\t", {"1" => "@","2" => "@","3" => ""}],
67
- '2 right blank columns' => ["@\t\t", {"1" => "@","2" => "","3" => ""}],
68
- 'left blank columns' => ["\t\t@", {"1" => "","2" => "","3" => "@"}],
69
- 'all blank columns' => ["\t\t", {"1" => "","2" => "","3" => ""}])
66
+ 'left blank column' => ["\t@\t@", {"1" => "","2" => "@","3" => "@"}],
67
+ 'center blank column' => ["@\t\t@", {"1" => "@","2" => "","3" => "@"}],
68
+ 'right blank column' => ["@\t@\t", {"1" => "@","2" => "@","3" => ""}],
69
+ '2 right blank columns' => ["@\t\t", {"1" => "@","2" => "","3" => ""}],
70
+ 'left blank columns' => ["\t\t@", {"1" => "","2" => "","3" => "@"}],
71
+ 'all blank columns' => ["\t\t", {"1" => "","2" => "","3" => ""}])
70
72
  def test_black_column(data)
71
73
  line, expected = data
72
74
 
73
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)
74
- parser.configure('keys' => '1,2,3')
75
- parser.instance.parse(line) { |time, record|
75
+ d = create_driver('keys' => '1,2,3')
76
+ d.instance.parse(line) { |time, record|
76
77
  assert_equal(expected, record)
77
78
  }
78
79
  end
79
80
 
80
81
  def test_parse_with_keep_time_key
81
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)
82
- parser.configure(
83
- 'keys'=>'time',
84
- 'time_key'=>'time',
85
- 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
86
- 'keep_time_key'=>'true',
87
- )
82
+ d = create_driver(
83
+ 'keys'=>'time',
84
+ 'time_key'=>'time',
85
+ 'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
86
+ 'keep_time_key'=>'true',
87
+ )
88
88
  text = '28/Feb/2013:12:00:00 +0900'
89
- parser.instance.parse(text) do |time, record|
89
+ d.instance.parse(text) do |time, record|
90
90
  assert_equal text, record['time']
91
91
  end
92
92
  end
93
93
 
94
94
  data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
95
- def test_parse_with_null_value_pattern
96
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)
97
- parser.configure(
98
- 'keys'=>param,
99
- 'time_key'=>'time',
100
- 'null_value_pattern'=>'^(-|null|NULL)$'
101
- )
102
- parser.instance.parse("-\tnull\tNULL\t\t--\tnuLL") do |time, record|
95
+ def test_parse_with_null_value_pattern(param)
96
+ d = create_driver(
97
+ 'keys'=>param,
98
+ 'null_value_pattern'=>'^(-|null|NULL)$'
99
+ )
100
+ d.instance.parse("-\tnull\tNULL\t\t--\tnuLL") do |time, record|
103
101
  assert_nil record['a']
104
102
  assert_nil record['b']
105
103
  assert_nil record['c']
@@ -110,14 +108,12 @@ class TSVParserTest < ::Test::Unit::TestCase
110
108
  end
111
109
 
112
110
  data('array param' => '["a","b"]', 'string param' => 'a,b')
113
- def test_parse_with_null_empty_string
114
- parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)
115
- parser.configure(
116
- 'keys'=>param,
117
- 'time_key'=>'time',
118
- 'null_empty_string'=>true
119
- )
120
- parser.instance.parse("\t ") do |time, record|
111
+ def test_parse_with_null_empty_string(param)
112
+ d = create_driver(
113
+ 'keys'=>param,
114
+ 'null_empty_string'=>true
115
+ )
116
+ d.instance.parse("\t ") do |time, record|
121
117
  assert_nil record['a']
122
118
  assert_equal record['b'], ' '
123
119
  end