fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -123,6 +123,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
123
123
  'flush_interval' => 1,
124
124
  'flush_thread_burst_interval' => 0.1,
125
125
  'retry_randomize' => false,
126
+ 'queued_chunks_limit_size' => 100
126
127
  }
127
128
  @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
128
129
  @i.register(:prefer_buffered_processing){ true }
@@ -252,6 +253,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
252
253
  'flush_thread_burst_interval' => 0.1,
253
254
  'retry_randomize' => false,
254
255
  'retry_timeout' => 3600,
256
+ 'queued_chunks_limit_size' => 100
255
257
  }
256
258
  @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
257
259
  @i.register(:prefer_buffered_processing){ true }
@@ -342,6 +344,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
342
344
  'flush_thread_burst_interval' => 0.1,
343
345
  'retry_randomize' => false,
344
346
  'retry_max_times' => 10,
347
+ 'queued_chunks_limit_size' => 100
345
348
  }
346
349
  @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
347
350
  @i.register(:prefer_buffered_processing){ true }
@@ -412,6 +415,68 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
412
415
  assert{ @i.buffer.stage.size == 1 }
413
416
  assert{ chunks.all?{|c| c.empty? } }
414
417
  end
418
+
419
+ test 'output plugin limits queued chunks via queued_chunks_limit_size' do
420
+ chunk_key = 'tag'
421
+ hash = {
422
+ 'flush_interval' => 1,
423
+ 'flush_thread_burst_interval' => 0.1,
424
+ 'retry_randomize' => false,
425
+ 'retry_max_times' => 7,
426
+ 'queued_chunks_limit_size' => 2,
427
+ }
428
+ @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
429
+ @i.register(:prefer_buffered_processing) { true }
430
+ @i.register(:format) { |tag,time,record| [tag,time.to_i,record].to_json + "\n" }
431
+ @i.register(:write) { |chunk| raise "yay, your #write must fail" }
432
+ @i.start
433
+ @i.after_start
434
+
435
+ @i.interrupt_flushes
436
+
437
+ now = Time.parse('2016-04-13 18:33:30 -0700')
438
+ Timecop.freeze(now)
439
+
440
+ @i.emit_events("test.tag.1", dummy_event_stream())
441
+
442
+ now = Time.parse('2016-04-13 18:33:31 -0700')
443
+ Timecop.freeze(now)
444
+
445
+ @i.emit_events("test.tag.2", dummy_event_stream())
446
+
447
+ @i.enqueue_thread_wait
448
+ @i.flush_thread_wakeup
449
+ waiting(4) { Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
450
+
451
+ assert { @i.buffer.queue.size > 0 }
452
+ assert { @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
453
+
454
+ assert { @i.write_count > 0 }
455
+ assert { @i.num_errors > 0 }
456
+
457
+ prev_write_count = @i.write_count
458
+ prev_num_errors = @i.num_errors
459
+
460
+ chunks = @i.buffer.queue.dup
461
+
462
+ 20.times do |i| # large times enough
463
+ now = @i.next_flush_time
464
+
465
+ Timecop.freeze(now)
466
+ @i.enqueue_thread_wait
467
+ @i.flush_thread_wakeup
468
+ waiting(4) { Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors }
469
+
470
+ @i.emit_events("test.tag.1", dummy_event_stream())
471
+ assert { @i.buffer.queue.size <= 2 }
472
+ assert { @i.buffer.stage.size == 1 } # all new data is stored into staged chunk
473
+
474
+ break if @i.buffer.queue.size == 0
475
+
476
+ prev_write_count = @i.write_count
477
+ prev_num_errors = @i.num_errors
478
+ end
479
+ end
415
480
  end
416
481
 
417
482
  sub_test_case 'bufferd output for retries with periodical retry' do
@@ -423,6 +488,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
423
488
  'retry_type' => :periodic,
424
489
  'retry_wait' => 3,
425
490
  'retry_randomize' => false,
491
+ 'queued_chunks_limit_size' => 100
426
492
  }
427
493
  @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
428
494
  @i.register(:prefer_buffered_processing){ true }
@@ -469,6 +535,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
469
535
  'retry_wait' => 30,
470
536
  'retry_randomize' => false,
471
537
  'retry_timeout' => 120,
538
+ 'queued_chunks_limit_size' => 100
472
539
  }
473
540
  @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
474
541
  @i.register(:prefer_buffered_processing){ true }
@@ -564,6 +631,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
564
631
  'retry_wait' => 3,
565
632
  'retry_randomize' => false,
566
633
  'retry_max_times' => 10,
634
+ 'queued_chunks_limit_size' => 100
567
635
  }
568
636
  @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
569
637
  @i.register(:prefer_buffered_processing){ true }
@@ -662,6 +730,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
662
730
  'retry_randomize' => false,
663
731
  'retry_timeout' => 3600,
664
732
  'retry_max_times' => 10,
733
+ 'queued_chunks_limit_size' => 100
665
734
  }
666
735
  @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
667
736
  @i.register(:prefer_buffered_processing){ true }
@@ -733,6 +802,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
733
802
  'retry_wait' => 30,
734
803
  'retry_timeout' => 360,
735
804
  'retry_max_times' => 10,
805
+ 'queued_chunks_limit_size' => 100
736
806
  }
737
807
  @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
738
808
  @i.register(:prefer_buffered_processing){ true }
@@ -544,7 +544,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
544
544
  test 'secondary plugin can do delayed commit even if primary does not do it, and non-committed chunks will be rollbacked by primary' do
545
545
  written = []
546
546
  chunks = []
547
- priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'delayed_commit_timeout' => 2, 'retry_randomize' => false})
547
+ priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'delayed_commit_timeout' => 2, 'retry_randomize' => false, 'queued_chunks_limit_size' => 10})
548
548
  secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
549
549
  @i.configure(config_element('ROOT','',{},[priconf,secconf]))
550
550
  @i.register(:prefer_buffered_processing){ true }
@@ -16,6 +16,7 @@ class Apache2ParserTest < ::Test::Unit::TestCase
16
16
  'referer' => nil,
17
17
  'agent' => 'Opera/12.0'
18
18
  }
19
+ @parser.configure({})
19
20
  end
20
21
 
21
22
  def test_parse
@@ -125,4 +125,21 @@ class LabeledTSVParserTest < ::Test::Unit::TestCase
125
125
  assert_equal record['b'], ' '
126
126
  end
127
127
  end
128
+
129
+ data("single space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
130
+ "multiple space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
131
+ "reverse" => ["k2=v2 k1=v1", { "k1" => "v1", "k2" => "v2" }],
132
+ "tab" => ["k2=v2\tk1=v1", { "k1" => "v1", "k2" => "v2" }],
133
+ "tab and space" => ["k2=v2\t k1=v1", { "k1" => "v1", "k2" => "v2" }])
134
+ def test_parse_with_delimiter_pattern(data)
135
+ text, expected = data
136
+ parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
137
+ parser.configure(
138
+ 'delimiter_pattern' => '/\s+/',
139
+ 'label_delimiter' => '='
140
+ )
141
+ parser.instance.parse(text) do |_time, record|
142
+ assert_equal(expected, record)
143
+ end
144
+ end
128
145
  end
@@ -16,6 +16,30 @@ class NginxParserTest < ::Test::Unit::TestCase
16
16
  'referer' => '-',
17
17
  'agent' => 'Opera/12.0'
18
18
  }
19
+ @expected_extended = {
20
+ 'remote' => '127.0.0.1',
21
+ 'host' => '192.168.0.1',
22
+ 'user' => '-',
23
+ 'method' => 'GET',
24
+ 'path' => '/',
25
+ 'code' => '200',
26
+ 'size' => '777',
27
+ 'referer' => '-',
28
+ 'agent' => 'Opera/12.0',
29
+ 'http_x_forwarded_for' => '-'
30
+ }
31
+ @expected_extended_multiple_ip = {
32
+ 'remote' => '127.0.0.1',
33
+ 'host' => '192.168.0.1',
34
+ 'user' => '-',
35
+ 'method' => 'GET',
36
+ 'path' => '/',
37
+ 'code' => '200',
38
+ 'size' => '777',
39
+ 'referer' => '-',
40
+ 'agent' => 'Opera/12.0',
41
+ 'http_x_forwarded_for' => '127.0.0.1, 192.168.0.1'
42
+ }
19
43
  end
20
44
 
21
45
  def create_driver
@@ -45,4 +69,20 @@ class NginxParserTest < ::Test::Unit::TestCase
45
69
  assert_equal(@expected, record)
46
70
  }
47
71
  end
72
+
73
+ def test_parse_with_http_x_forwarded_for
74
+ d = create_driver
75
+ d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" -') { |time, record|
76
+ assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
77
+ assert_equal(@expected_extended, record)
78
+ }
79
+ end
80
+
81
+ def test_parse_with_http_x_forwarded_for_multiple_ip
82
+ d = create_driver
83
+ d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" "127.0.0.1, 192.168.0.1"') { |time, record|
84
+ assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
85
+ assert_equal(@expected_extended_multiple_ip, record)
86
+ }
87
+ end
48
88
  end
@@ -160,25 +160,24 @@ class RegexpParserTest < ::Test::Unit::TestCase
160
160
  'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
161
161
  }
162
162
  d = create_driver(conf)
163
- regexp = d.instance.instance_variable_get(:@regexp)
163
+ regexp = d.instance.expression
164
164
  assert_equal(0, regexp.options)
165
165
  end
166
166
 
167
167
  data(
168
- ignorecase: [{ "ignorecase" => true }, Regexp::IGNORECASE],
169
- multiline: [{ "multiline" => true }, Regexp::MULTILINE],
170
- ignorecase_multiline: [{ "ignorecase" => true, "multiline" => true }, Regexp::IGNORECASE | Regexp::MULTILINE],
168
+ ignorecase: ["i", Regexp::IGNORECASE],
169
+ multiline: ["m", Regexp::MULTILINE],
170
+ ignorecase_multiline: ["im", Regexp::IGNORECASE | Regexp::MULTILINE],
171
171
  )
172
172
  def test_options(data)
173
173
  regexp_option, expected = data
174
174
  conf = {
175
- 'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
175
+ 'expression' => %Q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/#{regexp_option}!,
176
176
  'time_format' => "%d/%b/%Y:%H:%M:%S %z",
177
177
  'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
178
178
  }
179
- conf = conf.merge(regexp_option)
180
179
  d = create_driver(conf)
181
- regexp = d.instance.instance_variable_get(:@regexp)
180
+ regexp = d.instance.expression
182
181
  assert_equal(expected, regexp.options)
183
182
  end
184
183
  end
@@ -64,11 +64,30 @@ class SyslogParserTest < ::Test::Unit::TestCase
64
64
  end
65
65
  end
66
66
 
67
+ def test_parse_various_characters_for_tag
68
+ ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
69
+ @parser.configure({})
70
+ @parser.instance.parse("Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
71
+ assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
72
+ assert_equal(@expected.merge('ident' => ident), record)
73
+ }
74
+ end
75
+
76
+ def test_parse_various_characters_for_tag_with_priority
77
+ ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
78
+ @parser.configure('with_priority' => true)
79
+ @parser.instance.parse("<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
80
+ assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
81
+ assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)
82
+ }
83
+ end
84
+
67
85
  class TestRFC5424Regexp < self
68
86
  def test_parse_with_rfc5424_message
69
87
  @parser.configure(
70
88
  'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
71
89
  'message_format' => 'rfc5424',
90
+ 'with_priority' => true,
72
91
  )
73
92
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
74
93
  @parser.instance.parse(text) do |time, record|
@@ -78,11 +97,48 @@ class SyslogParserTest < ::Test::Unit::TestCase
78
97
  assert_equal "-", record["extradata"]
79
98
  assert_equal "Hi, from Fluentd!", record["message"]
80
99
  end
100
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
101
+ @parser.instance.patterns['format'])
102
+ end
103
+
104
+ def test_parse_with_rfc5424_message_and_without_priority
105
+ @parser.configure(
106
+ 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
107
+ 'message_format' => 'rfc5424',
108
+ )
109
+ text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
110
+ @parser.instance.parse(text) do |time, record|
111
+ assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
112
+ assert_equal "-", record["pid"]
113
+ assert_equal "-", record["msgid"]
114
+ assert_equal "-", record["extradata"]
115
+ assert_equal "Hi, from Fluentd!", record["message"]
116
+ end
117
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424,
118
+ @parser.instance.patterns['format'])
119
+ end
120
+
121
+ def test_parse_with_rfc5424_empty_message_and_without_priority
122
+ @parser.configure(
123
+ 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
124
+ 'message_format' => 'rfc5424',
125
+ )
126
+ text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - -'
127
+ @parser.instance.parse(text) do |time, record|
128
+ assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
129
+ assert_equal "-", record["pid"]
130
+ assert_equal "-", record["msgid"]
131
+ assert_equal "-", record["extradata"]
132
+ assert_nil record["message"]
133
+ end
134
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424,
135
+ @parser.instance.patterns['format'])
81
136
  end
82
137
 
83
138
  def test_parse_with_rfc5424_message_without_time_format
84
139
  @parser.configure(
85
140
  'message_format' => 'rfc5424',
141
+ 'with_priority' => true,
86
142
  )
87
143
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
88
144
  @parser.instance.parse(text) do |time, record|
@@ -94,21 +150,88 @@ class SyslogParserTest < ::Test::Unit::TestCase
94
150
  end
95
151
  end
96
152
 
153
+ def test_parse_with_rfc5424_message_with_priority_and_pid
154
+ @parser.configure(
155
+ 'message_format' => 'rfc5424',
156
+ 'with_priority' => true,
157
+ )
158
+ text = '<28>1 2018-09-26T15:54:26.620412+09:00 machine minissdpd 1298 - - peer 192.168.0.5:50123 is not from a LAN'
159
+ @parser.instance.parse(text) do |time, record|
160
+ assert_equal(event_time("2018-09-26T15:54:26.620412+0900", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
161
+ assert_equal "1298", record["pid"]
162
+ assert_equal "-", record["msgid"]
163
+ assert_equal "-", record["extradata"]
164
+ assert_equal " peer 192.168.0.5:50123 is not from a LAN", record["message"]
165
+ end
166
+ end
167
+
97
168
  def test_parse_with_rfc5424_structured_message
98
169
  @parser.configure(
99
170
  'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
100
171
  'message_format' => 'rfc5424',
172
+ 'with_priority' => true,
101
173
  )
102
174
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
103
175
  @parser.instance.parse(text) do |time, record|
104
176
  assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
105
- assert_equal "11111", record["pid"]
177
+ assert_equal "11111", record["pid"]
106
178
  assert_equal "ID24224", record["msgid"]
107
179
  assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
108
180
  record["extradata"]
109
181
  assert_equal "Hi, from Fluentd!", record["message"]
110
182
  end
111
183
  end
184
+
185
+ def test_parse_with_rfc5424_multiple_structured_message
186
+ @parser.configure(
187
+ 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
188
+ 'message_format' => 'rfc5424',
189
+ 'with_priority' => true,
190
+ )
191
+ text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"][exampleSDID@20224 class="high"] Hi, from Fluentd!'
192
+ @parser.instance.parse(text) do |time, record|
193
+ assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
194
+ assert_equal "11111", record["pid"]
195
+ assert_equal "ID24224", record["msgid"]
196
+ assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"][exampleSDID@20224 class=\"high\"]",
197
+ record["extradata"]
198
+ assert_equal "Hi, from Fluentd!", record["message"]
199
+ end
200
+ end
201
+
202
+ def test_parse_with_rfc5424_message_includes_right_bracket
203
+ @parser.configure(
204
+ 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
205
+ 'message_format' => 'rfc5424',
206
+ 'with_priority' => true,
207
+ )
208
+ text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd]!'
209
+ @parser.instance.parse(text) do |time, record|
210
+ assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
211
+ assert_equal "11111", record["pid"]
212
+ assert_equal "ID24224", record["msgid"]
213
+ assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
214
+ record["extradata"]
215
+ assert_equal "Hi, from Fluentd]!", record["message"]
216
+ end
217
+ end
218
+
219
+ def test_parse_with_rfc5424_empty_message
220
+ @parser.configure(
221
+ 'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
222
+ 'message_format' => 'rfc5424',
223
+ 'with_priority' => true,
224
+ )
225
+ text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"]'
226
+ @parser.instance.parse(text) do |time, record|
227
+ assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
228
+ assert_equal "11111", record["pid"]
229
+ assert_equal "ID24224", record["msgid"]
230
+ assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
231
+ record["extradata"]
232
+ assert_nil record["message"]
233
+ end
234
+ end
112
235
  end
113
236
 
114
237
  class TestAutoRegexp < self
@@ -122,6 +245,7 @@ assert_equal "11111", record["pid"]
122
245
  assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
123
246
  assert_equal(@expected, record)
124
247
  end
248
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
125
249
  end
126
250
 
127
251
  def test_auto_with_legacy_syslog_priority_message
@@ -135,12 +259,14 @@ assert_equal "11111", record["pid"]
135
259
  assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
136
260
  assert_equal(@expected.merge('pri' => 6), record)
137
261
  end
262
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
138
263
  end
139
264
 
140
265
  def test_parse_with_rfc5424_message
141
266
  @parser.configure(
142
267
  'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
143
268
  'message_format' => 'auto',
269
+ 'with_priority' => true,
144
270
  )
145
271
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
146
272
  @parser.instance.parse(text) do |time, record|
@@ -150,12 +276,15 @@ assert_equal "11111", record["pid"]
150
276
  assert_equal "-", record["extradata"]
151
277
  assert_equal "Hi, from Fluentd!", record["message"]
152
278
  end
279
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
280
+ @parser.instance.patterns['format'])
153
281
  end
154
282
 
155
283
  def test_parse_with_rfc5424_structured_message
156
284
  @parser.configure(
157
285
  'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
158
286
  'message_format' => 'auto',
287
+ 'with_priority' => true,
159
288
  )
160
289
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
161
290
  @parser.instance.parse(text) do |time, record|
@@ -166,6 +295,8 @@ assert_equal "11111", record["pid"]
166
295
  record["extradata"]
167
296
  assert_equal "Hi, from Fluentd!", record["message"]
168
297
  end
298
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
299
+ @parser.instance.patterns['format'])
169
300
  end
170
301
 
171
302
  def test_parse_with_both_message_type
@@ -173,12 +304,15 @@ assert_equal "11111", record["pid"]
173
304
  'time_format' => '%b %d %M:%S:%H',
174
305
  'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
175
306
  'message_format' => 'auto',
307
+ 'with_priority' => true,
176
308
  )
177
- text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
309
+ text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
178
310
  @parser.instance.parse(text) do |time, record|
179
311
  assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
180
- assert_equal(@expected, record)
312
+ assert_equal(@expected.merge('pri' => 1), record)
181
313
  end
314
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
315
+
182
316
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
183
317
  @parser.instance.parse(text) do |time, record|
184
318
  assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -188,11 +322,16 @@ assert_equal "11111", record["pid"]
188
322
  record["extradata"]
189
323
  assert_equal "Hi, from Fluentd!", record["message"]
190
324
  end
191
- text = 'Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
325
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
326
+ @parser.instance.patterns['format'])
327
+
328
+ text = '<1>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
192
329
  @parser.instance.parse(text) do |time, record|
193
330
  assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
194
- assert_equal(@expected, record)
331
+ assert_equal(@expected.merge('pri' => 1), record)
195
332
  end
333
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
334
+
196
335
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
197
336
  @parser.instance.parse(text) do |time, record|
198
337
  assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -201,6 +340,8 @@ assert_equal "11111", record["pid"]
201
340
  assert_equal "-", record["extradata"]
202
341
  assert_equal "Hi, from Fluentd!", record["message"]
203
342
  end
343
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
344
+ @parser.instance.patterns['format'])
204
345
  end
205
346
 
206
347
  def test_parse_with_both_message_type_and_priority
@@ -215,6 +356,8 @@ assert_equal "11111", record["pid"]
215
356
  assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
216
357
  assert_equal(@expected.merge('pri' => 6), record)
217
358
  end
359
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
360
+
218
361
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
219
362
  @parser.instance.parse(text) do |time, record|
220
363
  assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -224,11 +367,16 @@ assert_equal "11111", record["pid"]
224
367
  record["extradata"]
225
368
  assert_equal "Hi, from Fluentd!", record["message"]
226
369
  end
370
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
371
+ @parser.instance.patterns['format'])
372
+
227
373
  text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
228
374
  @parser.instance.parse(text) do |time, record|
229
375
  assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
230
376
  assert_equal(@expected.merge('pri' => 16), record)
231
377
  end
378
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
379
+
232
380
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
233
381
  @parser.instance.parse(text) do |time, record|
234
382
  assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
@@ -237,6 +385,8 @@ assert_equal "11111", record["pid"]
237
385
  assert_equal "-", record["extradata"]
238
386
  assert_equal "Hi, from Fluentd!", record["message"]
239
387
  end
388
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
389
+ @parser.instance.patterns['format'])
240
390
  end
241
391
  end
242
392
  end