fluentd 0.12.36 → 0.12.37
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/ChangeLog +14 -0
- data/lib/fluent/parser.rb +4 -3
- data/lib/fluent/plugin/in_tail.rb +3 -1
- data/lib/fluent/version.rb +1 -1
- data/test/test_parser.rb +60 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f88c1cef594f4ca5773a92310fd32f0fbf728a89
|
4
|
+
data.tar.gz: e9fc666c618c2b9bf35d10c21466a0539bb45d10
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5652613563369d7f73bd290c5e550a9fd02bff462502bf9074a399a4fa2df89e3da69a4f63e34480751d3411938f57346aa1d1e049a26fb921b40609c7d7d311
|
7
|
+
data.tar.gz: 641ea08ba43f5f646a6444f91939c7f99fc95d5d59a562f59123e64f2d79bf120adb17e32e3b9dea3daa51bc432619db688567c74319b131743ad6c1b475606b
|
data/ChangeLog
CHANGED
@@ -1,5 +1,19 @@
|
|
1
1
|
# v0.12
|
2
2
|
|
3
|
+
## Release 0.12.37 - 2017/06/21
|
4
|
+
|
5
|
+
### New features / Enhancement
|
6
|
+
|
7
|
+
* parser: Add rfc5424 regex without priority
|
8
|
+
https://github.com/fluent/fluentd/pull/1601
|
9
|
+
|
10
|
+
### Bug fixes
|
11
|
+
|
12
|
+
* in_tail: fix timing issue that the excluded_path doesn't apply.
|
13
|
+
https://github.com/fluent/fluentd/pull/1597
|
14
|
+
* in_tail: Detach watchers before updating watcher to avoid broken pos file
|
15
|
+
https://github.com/fluent/fluentd/pull/1598
|
16
|
+
|
3
17
|
## Release 0.12.36 - 2017/05/24
|
4
18
|
|
5
19
|
### New features / Enhancement
|
data/lib/fluent/parser.rb
CHANGED
@@ -540,7 +540,8 @@ module Fluent
|
|
540
540
|
REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
|
541
541
|
# From in_syslog default pattern
|
542
542
|
REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
|
543
|
-
|
543
|
+
REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
|
544
|
+
REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
|
544
545
|
REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
|
545
546
|
|
546
547
|
config_param :time_format, :string, default: "%b %d %H:%M:%S"
|
@@ -568,7 +569,7 @@ module Fluent
|
|
568
569
|
alias_method :parse, :parse_plain
|
569
570
|
end
|
570
571
|
@time_format = @rfc5424_time_format unless conf.has_key?('time_format')
|
571
|
-
REGEXP_RFC5424
|
572
|
+
@with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
|
572
573
|
when :auto
|
573
574
|
class << self
|
574
575
|
alias_method :parse, :parse_auto
|
@@ -590,7 +591,7 @@ module Fluent
|
|
590
591
|
|
591
592
|
def parse_auto(text, &block)
|
592
593
|
if REGEXP_DETECT_RFC5424.match(text)
|
593
|
-
@regexp = REGEXP_RFC5424
|
594
|
+
@regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
|
594
595
|
@time_parser = @time_parser_rfc5424
|
595
596
|
else
|
596
597
|
@regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
|
@@ -155,7 +155,6 @@ module Fluent
|
|
155
155
|
date = Time.now
|
156
156
|
paths = []
|
157
157
|
|
158
|
-
excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
|
159
158
|
@paths.each { |path|
|
160
159
|
path = date.strftime(path)
|
161
160
|
if path.include?('*')
|
@@ -182,6 +181,7 @@ module Fluent
|
|
182
181
|
paths << path
|
183
182
|
end
|
184
183
|
}
|
184
|
+
excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
|
185
185
|
paths - excluded
|
186
186
|
end
|
187
187
|
|
@@ -508,11 +508,13 @@ module Fluent
|
|
508
508
|
io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
|
509
509
|
@io_handler = io_handler
|
510
510
|
else # file is rotated and new file found
|
511
|
+
detach
|
511
512
|
@update_watcher.call(@path, swap_state(@pe))
|
512
513
|
end
|
513
514
|
else # file is rotated and new file not found
|
514
515
|
# Clear RotateHandler to avoid duplicated file watch in same path.
|
515
516
|
@rotate_handler = nil
|
517
|
+
detach
|
516
518
|
@update_watcher.call(@path, swap_state(@pe))
|
517
519
|
end
|
518
520
|
end
|
data/lib/fluent/version.rb
CHANGED
data/test/test_parser.rb
CHANGED
@@ -379,6 +379,7 @@ module ParserTest
|
|
379
379
|
@parser.configure(
|
380
380
|
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
381
381
|
'message_format' => 'rfc5424',
|
382
|
+
'with_priority' => true,
|
382
383
|
)
|
383
384
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
384
385
|
@parser.parse(text) do |time, record|
|
@@ -388,11 +389,32 @@ module ParserTest
|
|
388
389
|
assert_equal "-", record["extradata"]
|
389
390
|
assert_equal "Hi, from Fluentd!", record["message"]
|
390
391
|
end
|
392
|
+
assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
393
|
+
@parser.instance.patterns['format'])
|
394
|
+
|
395
|
+
end
|
396
|
+
|
397
|
+
def test_parse_with_rfc5424_message_and_without_priority
|
398
|
+
@parser.configure(
|
399
|
+
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
400
|
+
'message_format' => 'rfc5424',
|
401
|
+
)
|
402
|
+
text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
403
|
+
@parser.instance.parse(text) do |time, record|
|
404
|
+
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
405
|
+
assert_equal "-", record["pid"]
|
406
|
+
assert_equal "-", record["msgid"]
|
407
|
+
assert_equal "-", record["extradata"]
|
408
|
+
assert_equal "Hi, from Fluentd!", record["message"]
|
409
|
+
end
|
410
|
+
assert_equal(TextParser::SyslogParser::REGEXP_RFC5424,
|
411
|
+
@parser.instance.patterns['format'])
|
391
412
|
end
|
392
413
|
|
393
414
|
def test_parse_with_rfc5424_message_without_time_format
|
394
415
|
@parser.configure(
|
395
416
|
'message_format' => 'rfc5424',
|
417
|
+
'with_priority' => true,
|
396
418
|
)
|
397
419
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
398
420
|
@parser.instance.parse(text) do |time, record|
|
@@ -408,6 +430,7 @@ module ParserTest
|
|
408
430
|
@parser.configure(
|
409
431
|
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
410
432
|
'message_format' => 'rfc5424',
|
433
|
+
'with_priority' => true,
|
411
434
|
)
|
412
435
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
|
413
436
|
@parser.parse(text) do |time, record|
|
@@ -445,12 +468,15 @@ module ParserTest
|
|
445
468
|
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
|
446
469
|
assert_equal(@expected.merge('pri' => 6), record)
|
447
470
|
end
|
471
|
+
assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
|
472
|
+
@parser.instance.patterns['format'])
|
448
473
|
end
|
449
474
|
|
450
475
|
def test_parse_with_rfc5424_message
|
451
476
|
@parser.configure(
|
452
477
|
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
453
478
|
'message_format' => 'auto',
|
479
|
+
'with_priority' => true,
|
454
480
|
)
|
455
481
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
456
482
|
@parser.parse(text) do |time, record|
|
@@ -460,12 +486,15 @@ module ParserTest
|
|
460
486
|
assert_equal "-", record["extradata"]
|
461
487
|
assert_equal "Hi, from Fluentd!", record["message"]
|
462
488
|
end
|
489
|
+
assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
490
|
+
@parser.instance.patterns['format'])
|
463
491
|
end
|
464
492
|
|
465
493
|
def test_parse_with_rfc5424_structured_message
|
466
494
|
@parser.configure(
|
467
495
|
'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
468
496
|
'message_format' => 'auto',
|
497
|
+
'with_priority' => true,
|
469
498
|
)
|
470
499
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
|
471
500
|
@parser.parse(text) do |time, record|
|
@@ -476,6 +505,8 @@ module ParserTest
|
|
476
505
|
record["extradata"]
|
477
506
|
assert_equal "Hi, from Fluentd!", record["message"]
|
478
507
|
end
|
508
|
+
assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
509
|
+
@parser.instance.patterns['format'])
|
479
510
|
end
|
480
511
|
|
481
512
|
def test_parse_with_both_message_type
|
@@ -483,12 +514,16 @@ module ParserTest
|
|
483
514
|
'time_format' => '%b %d %M:%S:%H',
|
484
515
|
'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
|
485
516
|
'message_format' => 'auto',
|
517
|
+
'with_priority' => true,
|
486
518
|
)
|
487
|
-
text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
519
|
+
text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
488
520
|
@parser.parse(text) do |time, record|
|
489
521
|
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
|
490
|
-
assert_equal(@expected, record)
|
522
|
+
assert_equal(@expected.merge('pri' => 1), record)
|
491
523
|
end
|
524
|
+
assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
|
525
|
+
@parser.instance.patterns['format'])
|
526
|
+
|
492
527
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
|
493
528
|
@parser.parse(text) do |time, record|
|
494
529
|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
@@ -498,11 +533,17 @@ module ParserTest
|
|
498
533
|
record["extradata"]
|
499
534
|
assert_equal "Hi, from Fluentd!", record["message"]
|
500
535
|
end
|
501
|
-
|
536
|
+
assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
537
|
+
@parser.instance.patterns['format'])
|
538
|
+
|
539
|
+
text = '<1>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
502
540
|
@parser.parse(text) do |time, record|
|
503
541
|
assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
|
504
|
-
assert_equal(@expected, record)
|
542
|
+
assert_equal(@expected.merge('pri' => 1), record)
|
505
543
|
end
|
544
|
+
assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
|
545
|
+
@parser.instance.patterns['format'])
|
546
|
+
|
506
547
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
507
548
|
@parser.parse(text) do |time, record|
|
508
549
|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
@@ -511,6 +552,9 @@ module ParserTest
|
|
511
552
|
assert_equal "-", record["extradata"]
|
512
553
|
assert_equal "Hi, from Fluentd!", record["message"]
|
513
554
|
end
|
555
|
+
assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
556
|
+
@parser.instance.patterns['format'])
|
557
|
+
|
514
558
|
end
|
515
559
|
|
516
560
|
def test_parse_with_both_message_type_and_priority
|
@@ -525,6 +569,9 @@ module ParserTest
|
|
525
569
|
assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
|
526
570
|
assert_equal(@expected.merge('pri' => 6), record)
|
527
571
|
end
|
572
|
+
assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
|
573
|
+
@parser.instance.patterns['format'])
|
574
|
+
|
528
575
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
|
529
576
|
@parser.parse(text) do |time, record|
|
530
577
|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
@@ -534,11 +581,17 @@ module ParserTest
|
|
534
581
|
record["extradata"]
|
535
582
|
assert_equal "Hi, from Fluentd!", record["message"]
|
536
583
|
end
|
584
|
+
assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
585
|
+
@parser.instance.patterns['format'])
|
586
|
+
|
537
587
|
text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
|
538
588
|
@parser.parse(text) do |time, record|
|
539
589
|
assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
|
540
590
|
assert_equal(@expected.merge('pri' => 16), record)
|
541
591
|
end
|
592
|
+
assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
|
593
|
+
@parser.instance.patterns['format'])
|
594
|
+
|
542
595
|
text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
|
543
596
|
@parser.parse(text) do |time, record|
|
544
597
|
assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
|
@@ -547,6 +600,9 @@ module ParserTest
|
|
547
600
|
assert_equal "-", record["extradata"]
|
548
601
|
assert_equal "Hi, from Fluentd!", record["message"]
|
549
602
|
end
|
603
|
+
assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
|
604
|
+
@parser.instance.patterns['format'])
|
605
|
+
|
550
606
|
end
|
551
607
|
end
|
552
608
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluentd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.12.
|
4
|
+
version: 0.12.37
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|