fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -544,7 +544,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
544
544
  test 'secondary plugin can do delayed commit even if primary does not do it, and non-committed chunks will be rollbacked by primary' do
545
545
  written = []
546
546
  chunks = []
547
- priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'delayed_commit_timeout' => 2, 'retry_randomize' => false})
547
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'delayed_commit_timeout' => 2, 'retry_randomize' => false, 'queued_chunks_limit_size' => 10})
548
548
  secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
549
549
  @i.configure(config_element('ROOT','',{},[priconf,secconf]))
550
550
  @i.register(:prefer_buffered_processing){ true }
@@ -16,6 +16,7 @@ class Apache2ParserTest < ::Test::Unit::TestCase
16
16
  'referer' => nil,
17
17
  'agent' => 'Opera/12.0'
18
18
  }
19
+ @parser.configure({})
19
20
  end
20
21
 
21
22
  def test_parse
@@ -125,4 +125,21 @@ class LabeledTSVParserTest < ::Test::Unit::TestCase
125
125
  assert_equal record['b'], ' '
126
126
  end
127
127
  end
128
+
129
+ data("single space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
130
+ "multiple space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
131
+ "reverse" => ["k2=v2 k1=v1", { "k1" => "v1", "k2" => "v2" }],
132
+ "tab" => ["k2=v2\tk1=v1", { "k1" => "v1", "k2" => "v2" }],
133
+ "tab and space" => ["k2=v2\t k1=v1", { "k1" => "v1", "k2" => "v2" }])
134
+ def test_parse_with_delimiter_pattern(data)
135
+ text, expected = data
136
+ parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
137
+ parser.configure(
138
+ 'delimiter_pattern' => '/\s+/',
139
+ 'label_delimiter' => '='
140
+ )
141
+ parser.instance.parse(text) do |_time, record|
142
+ assert_equal(expected, record)
143
+ end
144
+ end
128
145
  end
@@ -16,6 +16,30 @@ class NginxParserTest < ::Test::Unit::TestCase
16
16
  'referer' => '-',
17
17
  'agent' => 'Opera/12.0'
18
18
  }
19
+ @expected_extended = {
20
+ 'remote' => '127.0.0.1',
21
+ 'host' => '192.168.0.1',
22
+ 'user' => '-',
23
+ 'method' => 'GET',
24
+ 'path' => '/',
25
+ 'code' => '200',
26
+ 'size' => '777',
27
+ 'referer' => '-',
28
+ 'agent' => 'Opera/12.0',
29
+ 'http_x_forwarded_for' => '-'
30
+ }
31
+ @expected_extended_multiple_ip = {
32
+ 'remote' => '127.0.0.1',
33
+ 'host' => '192.168.0.1',
34
+ 'user' => '-',
35
+ 'method' => 'GET',
36
+ 'path' => '/',
37
+ 'code' => '200',
38
+ 'size' => '777',
39
+ 'referer' => '-',
40
+ 'agent' => 'Opera/12.0',
41
+ 'http_x_forwarded_for' => '127.0.0.1, 192.168.0.1'
42
+ }
19
43
  end
20
44
 
21
45
  def create_driver
@@ -45,4 +69,20 @@ class NginxParserTest < ::Test::Unit::TestCase
45
69
  assert_equal(@expected, record)
46
70
  }
47
71
  end
72
+
73
+ def test_parse_with_http_x_forwarded_for
74
+ d = create_driver
75
+ d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" -') { |time, record|
76
+ assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
77
+ assert_equal(@expected_extended, record)
78
+ }
79
+ end
80
+
81
+ def test_parse_with_http_x_forwarded_for_multiple_ip
82
+ d = create_driver
83
+ d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" "127.0.0.1, 192.168.0.1"') { |time, record|
84
+ assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
85
+ assert_equal(@expected_extended_multiple_ip, record)
86
+ }
87
+ end
48
88
  end
@@ -160,25 +160,24 @@ class RegexpParserTest < ::Test::Unit::TestCase
160
160
  'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
161
161
  }
162
162
  d = create_driver(conf)
163
- regexp = d.instance.instance_variable_get(:@regexp)
163
+ regexp = d.instance.expression
164
164
  assert_equal(0, regexp.options)
165
165
  end
166
166
 
167
167
  data(
168
- ignorecase: [{ "ignorecase" => true }, Regexp::IGNORECASE],
169
- multiline: [{ "multiline" => true }, Regexp::MULTILINE],
170
- ignorecase_multiline: [{ "ignorecase" => true, "multiline" => true }, Regexp::IGNORECASE | Regexp::MULTILINE],
168
+ ignorecase: ["i", Regexp::IGNORECASE],
169
+ multiline: ["m", Regexp::MULTILINE],
170
+ ignorecase_multiline: ["im", Regexp::IGNORECASE | Regexp::MULTILINE],
171
171
  )
172
172
  def test_options(data)
173
173
  regexp_option, expected = data
174
174
  conf = {
175
- 'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
175
+ 'expression' => %Q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/#{regexp_option}!,
176
176
  'time_format' => "%d/%b/%Y:%H:%M:%S %z",
177
177
  'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
178
178
  }
179
- conf = conf.merge(regexp_option)
180
179
  d = create_driver(conf)
181
- regexp = d.instance.instance_variable_get(:@regexp)
180
+ regexp = d.instance.expression
182
181
  assert_equal(expected, regexp.options)
183
182
  end
184
183
  end
@@ -64,11 +64,30 @@ class SyslogParserTest < ::Test::Unit::TestCase
64
64
  end
65
65
  end
66
66
 
67
+ def test_parse_various_characters_for_tag
68
+ ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
69
+ @parser.configure({})
70
+ @parser.instance.parse("Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
71
+ assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
72
+ assert_equal(@expected.merge('ident' => ident), record)
73
+ }
74
+ end
75
+
76
+ def test_parse_various_characters_for_tag_with_priority
77
+ ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
78
+ @parser.configure('with_priority' => true)
79
+ @parser.instance.parse("<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
80
+ assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
81
+ assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)
82
+ }
83
+ end
84
+
67
85
  class TestRFC5424Regexp < self
68
86
  def test_parse_with_rfc5424_message
69
87
  @parser.configure(
70
88
  'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
71
89
  'message_format' => 'rfc5424',
90
+ 'with_priority' => true,
72
91
  )
73
92
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
74
93
  @parser.instance.parse(text) do |time, record|
@@ -78,11 +97,48 @@ class SyslogParserTest < ::Test::Unit::TestCase
78
97
  assert_equal "-", record["extradata"]
79
98
  assert_equal "Hi, from Fluentd!", record["message"]
80
99
  end
100
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
101
+ @parser.instance.patterns['format'])
102
+ end
103
+
104
+ def test_parse_with_rfc5424_message_and_without_priority
105
+ @parser.configure(
106
+ 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
107
+ 'message_format' => 'rfc5424',
108
+ )
109
+ text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
110
+ @parser.instance.parse(text) do |time, record|
111
+ assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
112
+ assert_equal "-", record["pid"]
113
+ assert_equal "-", record["msgid"]
114
+ assert_equal "-", record["extradata"]
115
+ assert_equal "Hi, from Fluentd!", record["message"]
116
+ end
117
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424,
118
+ @parser.instance.patterns['format'])
119
+ end
120
+
121
+ def test_parse_with_rfc5424_empty_message_and_without_priority
122
+ @parser.configure(
123
+ 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
124
+ 'message_format' => 'rfc5424',
125
+ )
126
+ text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - -'
127
+ @parser.instance.parse(text) do |time, record|
128
+ assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
129
+ assert_equal "-", record["pid"]
130
+ assert_equal "-", record["msgid"]
131
+ assert_equal "-", record["extradata"]
132
+ assert_nil record["message"]
133
+ end
134
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424,
135
+ @parser.instance.patterns['format'])
81
136
  end
82
137
 
83
138
  def test_parse_with_rfc5424_message_without_time_format
84
139
  @parser.configure(
85
140
  'message_format' => 'rfc5424',
141
+ 'with_priority' => true,
86
142
  )
87
143
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
88
144
  @parser.instance.parse(text) do |time, record|
@@ -94,21 +150,88 @@ class SyslogParserTest < ::Test::Unit::TestCase
94
150
  end
95
151
  end
96
152
 
153
+ def test_parse_with_rfc5424_message_with_priority_and_pid
154
+ @parser.configure(
155
+ 'message_format' => 'rfc5424',
156
+ 'with_priority' => true,
157
+ )
158
+ text = '<28>1 2018-09-26T15:54:26.620412+09:00 machine minissdpd 1298 - - peer 192.168.0.5:50123 is not from a LAN'
159
+ @parser.instance.parse(text) do |time, record|
160
+ assert_equal(event_time("2018-09-26T15:54:26.620412+0900", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
161
+ assert_equal "1298", record["pid"]
162
+ assert_equal "-", record["msgid"]
163
+ assert_equal "-", record["extradata"]
164
+ assert_equal " peer 192.168.0.5:50123 is not from a LAN", record["message"]
165
+ end
166
+ end
167
+
97
168
  def test_parse_with_rfc5424_structured_message
98
169
  @parser.configure(
99
170
  'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
100
171
  'message_format' => 'rfc5424',
172
+ 'with_priority' => true,
101
173
  )
102
174
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
103
175
  @parser.instance.parse(text) do |time, record|
104
176
  assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
105
- assert_equal "11111", record["pid"]
177
+ assert_equal "11111", record["pid"]
106
178
  assert_equal "ID24224", record["msgid"]
107
179
  assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
108
180
  record["extradata"]
109
181
  assert_equal "Hi, from Fluentd!", record["message"]
110
182
  end
111
183
  end
184
+
185
+ def test_parse_with_rfc5424_multiple_structured_message
186
+ @parser.configure(
187
+ 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
188
+ 'message_format' => 'rfc5424',
189
+ 'with_priority' => true,
190
+ )
191
+ text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"][exampleSDID@20224 class="high"] Hi, from Fluentd!'
192
+ @parser.instance.parse(text) do |time, record|
193
+ assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
194
+ assert_equal "11111", record["pid"]
195
+ assert_equal "ID24224", record["msgid"]
196
+ assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"][exampleSDID@20224 class=\"high\"]",
197
+ record["extradata"]
198
+ assert_equal "Hi, from Fluentd!", record["message"]
199
+ end
200
+ end
201
+
202
+ def test_parse_with_rfc5424_message_includes_right_bracket
203
+ @parser.configure(
204
+ 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
205
+ 'message_format' => 'rfc5424',
206
+ 'with_priority' => true,
207
+ )
208
+ text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd]!'
209
+ @parser.instance.parse(text) do |time, record|
210
+ assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
211
+ assert_equal "11111", record["pid"]
212
+ assert_equal "ID24224", record["msgid"]
213
+ assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
214
+ record["extradata"]
215
+ assert_equal "Hi, from Fluentd]!", record["message"]
216
+ end
217
+ end
218
+
219
+ def test_parse_with_rfc5424_empty_message
220
+ @parser.configure(
221
+ 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
222
+ 'message_format' => 'rfc5424',
223
+ 'with_priority' => true,
224
+ )
225
+ text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"]'
226
+ @parser.instance.parse(text) do |time, record|
227
+ assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
228
+ assert_equal "11111", record["pid"]
229
+ assert_equal "ID24224", record["msgid"]
230
+ assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
231
+ record["extradata"]
232
+ assert_nil record["message"]
233
+ end
234
+ end
112
235
  end
113
236
 
114
237
  class TestAutoRegexp < self
@@ -122,6 +245,7 @@ assert_equal "11111", record["pid"]
122
245
  assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
123
246
  assert_equal(@expected, record)
124
247
  end
248
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
125
249
  end
126
250
 
127
251
  def test_auto_with_legacy_syslog_priority_message
@@ -135,12 +259,14 @@ assert_equal "11111", record["pid"]
135
259
  assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
136
260
  assert_equal(@expected.merge('pri' => 6), record)
137
261
  end
262
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
138
263
  end
139
264
 
140
265
  def test_parse_with_rfc5424_message
141
266
  @parser.configure(
142
267
  'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
143
268
  'message_format' => 'auto',
269
+ 'with_priority' => true,
144
270
  )
145
271
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
146
272
  @parser.instance.parse(text) do |time, record|
@@ -150,12 +276,15 @@ assert_equal "11111", record["pid"]
150
276
  assert_equal "-", record["extradata"]
151
277
  assert_equal "Hi, from Fluentd!", record["message"]
152
278
  end
279
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
280
+ @parser.instance.patterns['format'])
153
281
  end
154
282
 
155
283
  def test_parse_with_rfc5424_structured_message
156
284
  @parser.configure(
157
285
  'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
158
286
  'message_format' => 'auto',
287
+ 'with_priority' => true,
159
288
  )
160
289
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
161
290
  @parser.instance.parse(text) do |time, record|
@@ -166,6 +295,8 @@ assert_equal "11111", record["pid"]
166
295
  record["extradata"]
167
296
  assert_equal "Hi, from Fluentd!", record["message"]
168
297
  end
298
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
299
+ @parser.instance.patterns['format'])
169
300
  end
170
301
 
171
302
  def test_parse_with_both_message_type
@@ -173,12 +304,15 @@ assert_equal "11111", record["pid"]
173
304
  'time_format' => '%b %d %M:%S:%H',
174
305
  'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
175
306
  'message_format' => 'auto',
307
+ 'with_priority' => true,
176
308
  )
177
- text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
309
+ text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
178
310
  @parser.instance.parse(text) do |time, record|
179
311
  assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
180
- assert_equal(@expected, record)
312
+ assert_equal(@expected.merge('pri' => 1), record)
181
313
  end
314
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
315
+
182
316
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
183
317
  @parser.instance.parse(text) do |time, record|
184
318
  assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -188,11 +322,16 @@ assert_equal "11111", record["pid"]
188
322
  record["extradata"]
189
323
  assert_equal "Hi, from Fluentd!", record["message"]
190
324
  end
191
- text = 'Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
325
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
326
+ @parser.instance.patterns['format'])
327
+
328
+ text = '<1>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
192
329
  @parser.instance.parse(text) do |time, record|
193
330
  assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
194
- assert_equal(@expected, record)
331
+ assert_equal(@expected.merge('pri' => 1), record)
195
332
  end
333
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
334
+
196
335
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
197
336
  @parser.instance.parse(text) do |time, record|
198
337
  assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -201,6 +340,8 @@ assert_equal "11111", record["pid"]
201
340
  assert_equal "-", record["extradata"]
202
341
  assert_equal "Hi, from Fluentd!", record["message"]
203
342
  end
343
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
344
+ @parser.instance.patterns['format'])
204
345
  end
205
346
 
206
347
  def test_parse_with_both_message_type_and_priority
@@ -215,6 +356,8 @@ assert_equal "11111", record["pid"]
215
356
  assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
216
357
  assert_equal(@expected.merge('pri' => 6), record)
217
358
  end
359
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
360
+
218
361
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
219
362
  @parser.instance.parse(text) do |time, record|
220
363
  assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -224,11 +367,16 @@ assert_equal "11111", record["pid"]
224
367
  record["extradata"]
225
368
  assert_equal "Hi, from Fluentd!", record["message"]
226
369
  end
370
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
371
+ @parser.instance.patterns['format'])
372
+
227
373
  text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
228
374
  @parser.instance.parse(text) do |time, record|
229
375
  assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
230
376
  assert_equal(@expected.merge('pri' => 16), record)
231
377
  end
378
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
379
+
232
380
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
233
381
  @parser.instance.parse(text) do |time, record|
234
382
  assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -237,6 +385,8 @@ assert_equal "11111", record["pid"]
237
385
  assert_equal "-", record["extradata"]
238
386
  assert_equal "Hi, from Fluentd!", record["message"]
239
387
  end
388
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
389
+ @parser.instance.patterns['format'])
240
390
  end
241
391
  end
242
392
  end
@@ -510,7 +510,7 @@ class ChildProcessTest < Test::Unit::TestCase
510
510
  unless Fluent.windows?
511
511
  test 'can specify subprocess name' do
512
512
  io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
513
- process_naming_enabled = (open("|ps"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
513
+ process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.select{|line| line.include?("caaaaaaaaaaat") }.size > 0)
514
514
  Process.kill(:TERM, io.pid) rescue nil
515
515
  io.close rescue nil
516
516
 
@@ -527,15 +527,15 @@ class ChildProcessTest < Test::Unit::TestCase
527
527
  m.lock
528
528
  ran = true
529
529
  pids << @d.child_process_id
530
- proc_lines += open("|ps"){|_io| _io.readlines }
530
+ proc_lines += open("|ps opid,cmd"){|_io| _io.readlines }
531
531
  m.unlock
532
532
  readio.read
533
533
  end
534
534
  sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
535
535
  m.lock
536
536
  pid = pids.first
537
- # 51358 ttys001 0:00.00 sleeper -e sleep 10
538
- assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[3] == "sleeeeeeeeeper" }
537
+ # 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
538
+ assert{ proc_lines.select{|line| line =~ /^\s*#{pid}\s/ }.first.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
539
539
  @d.stop; @d.shutdown; @d.close; @d.terminate
540
540
  end
541
541
  end
@@ -328,4 +328,26 @@ class CompatParameterTest < Test::Unit::TestCase
328
328
  # TODO:
329
329
  end
330
330
  end
331
+
332
+ sub_test_case 'parser plugins' do
333
+ test 'syslog parser parameters' do
334
+ hash = {
335
+ 'format' => 'syslog',
336
+ 'message_format' => 'rfc5424',
337
+ 'with_priority' => 'true',
338
+ 'rfc5424_time_format' => '%Y'
339
+ }
340
+ conf = config_element('ROOT', '', hash)
341
+ @i = DummyI0.new
342
+ @i.configure(conf)
343
+ @i.start
344
+ @i.after_start
345
+
346
+ parser = @i.parser
347
+ assert_kind_of(Fluent::Plugin::SyslogParser, parser)
348
+ assert_equal :rfc5424, parser.message_format
349
+ assert_equal true, parser.with_priority
350
+ assert_equal '%Y', parser.rfc5424_time_format
351
+ end
352
+ end
331
353
  end