fluentd 1.9.1-x64-mingw32 → 1.9.2-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6f21baedc030fb756b14e72fabb65059322c86f07537c3328175eb982061ce80
4
- data.tar.gz: b4a596b4cf6704dd752cff7d19af6618b3ab40c627d052a067174514b21157be
3
+ metadata.gz: 976b5918a9761704f4c808761a5988c6f95f1b86e6fd7f7fa68f68f5de4b64ce
4
+ data.tar.gz: 37a6608a19b50fc7d4815df34fb9321edfd48547d32513a0c631d297580ca71b
5
5
  SHA512:
6
- metadata.gz: ce0465ee776348e12636d3b003f9a54a2da7c1d643117bae037af882c4511cfe036c0145c314dcfa542ed47715c891db268acd35dab9c99053f6ebc8d32a476e
7
- data.tar.gz: '009d91ce4bd655b67e51fc93f7df5de59d82fc880bf9b5930dbcd904ada392d35aa833abf08a8211a35289994871d733ea6d5862232884abe6676a23b83ff4f4'
6
+ metadata.gz: ce2d17a96d496d47848082e7ed980b3c8bd111d071ff256f9788db0d2c13fb41da543eeee39cd3764b6b18d16a53039a49968f5797264be7dcf3b8d472a24aa2
7
+ data.tar.gz: 0b86f77306d5b5b79d75d56d013eabf81e0714446a9b8cdf814b66893e4baea1004866060c45d0b2ef1914bf852f7592cde72eba15446d2e881f839e989c7039
data/.travis.yml CHANGED
@@ -45,6 +45,9 @@ branches:
45
45
  only:
46
46
  - master
47
47
 
48
+ before_install:
49
+ - gem update --system=3.1.2
50
+
48
51
  sudo: false
49
52
  dist: trusty # for TLSv1.2 support
50
53
 
data/CHANGELOG.md CHANGED
@@ -1,5 +1,27 @@
1
1
  # v1.9
2
2
 
3
+ ## Release v1.9.2 - 2020/02/13
4
+
5
+ ### Enhancement
6
+
7
+ * in_tail: Add `pos_file_compaction_interval` parameter for auto compaction
8
+ https://github.com/fluent/fluentd/pull/2805
9
+ * command: Use given encoding when RUBYOPT has `-E`
10
+ https://github.com/fluent/fluentd/pull/2814
11
+
12
+ ### Bug fix
13
+
14
+ * command: Accept RUBYOPT with two or more options
15
+ https://github.com/fluent/fluentd/pull/2807
16
+ * command: Fix infinite loop bug when RUBYOPT is invalid
17
+ https://github.com/fluent/fluentd/pull/2813
18
+ * log: serverengine's log should be formatted with the same format of fluentd
19
+ https://github.com/fluent/fluentd/pull/2812
20
+ * in_http: Fix `NoMethodError` when `OPTIONS` request doesn't have 'Origin' header
21
+ https://github.com/fluent/fluentd/pull/2823
22
+ * parser_syslog: Improved for parsing RFC5424 structured data in `parser_syslog`
23
+ https://github.com/fluent/fluentd/pull/2816
24
+
3
25
  ## Release v1.9.1 - 2020/01/31
4
26
 
5
27
  ### Enhancement
@@ -526,14 +526,20 @@ module Fluent::Plugin
526
526
  end
527
527
 
528
528
  def include_cors_allow_origin
529
+ if @origin.nil?
530
+ return false
531
+ end
532
+
529
533
  if @cors_allow_origins.include?(@origin)
530
534
  return true
531
535
  end
532
536
  filtered_cors_allow_origins = @cors_allow_origins.select {|origin| origin != ""}
533
- return filtered_cors_allow_origins.find do |origin|
534
- (start_str,end_str) = origin.split("*",2)
535
- @origin.start_with?(start_str) and @origin.end_with?(end_str)
536
- end != nil
537
+ r = filtered_cors_allow_origins.find do |origin|
538
+ (start_str, end_str) = origin.split("*", 2)
539
+ @origin.start_with?(start_str) && @origin.end_with?(end_str)
540
+ end
541
+
542
+ !r.nil?
537
543
  end
538
544
  end
539
545
  end
@@ -71,6 +71,8 @@ module Fluent::Plugin
71
71
  config_param :rotate_wait, :time, default: 5
72
72
  desc 'Fluentd will record the position it last read into this file.'
73
73
  config_param :pos_file, :string, default: nil
74
+ desc 'The cleanup interval of pos file'
75
+ config_param :pos_file_compaction_interval, :integer, default: nil
74
76
  desc 'Start to read the logs from the head of file, not bottom.'
75
77
  config_param :read_from_head, :bool, default: false
76
78
  # When the program deletes log file and re-creates log file with same filename after passed refresh_interval,
@@ -211,7 +213,14 @@ module Fluent::Plugin
211
213
  FileUtils.mkdir_p(pos_file_dir) unless Dir.exist?(pos_file_dir)
212
214
  @pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY, @file_perm)
213
215
  @pf_file.sync = true
214
- @pf = PositionFile.parse(@pf_file)
216
+ @pf = PositionFile.load(@pf_file, logger: log)
217
+
218
+ if @pos_file_compaction_interval
219
+ timer_execute(:in_tail_refresh_compact_pos_file, @pos_file_compaction_interval) do
220
+ log.info('Clean up the pos file')
221
+ @pf.try_compact
222
+ end
223
+ end
215
224
  end
216
225
 
217
226
  refresh_watchers unless @skip_refresh_on_startup
@@ -14,7 +14,7 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
- require 'fluent/plugin/in_tail'
17
+ require 'fluent/plugin/input'
18
18
 
19
19
  module Fluent::Plugin
20
20
  class TailInput < Fluent::Plugin::Input
@@ -23,10 +23,17 @@ module Fluent::Plugin
23
23
  POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
24
24
  POSITION_FILE_ENTRY_FORMAT = "%s\t%016x\t%016x\n".freeze
25
25
 
26
- def initialize(file, file_mutex, map)
26
+ def self.load(file, logger:)
27
+ pf = new(file, logger: logger)
28
+ pf.load
29
+ pf
30
+ end
31
+
32
+ def initialize(file, logger: nil)
27
33
  @file = file
28
- @file_mutex = file_mutex
29
- @map = map
34
+ @logger = logger
35
+ @file_mutex = Mutex.new
36
+ @map = {}
30
37
  end
31
38
 
32
39
  def [](path)
@@ -48,56 +55,89 @@ module Fluent::Plugin
48
55
  end
49
56
  end
50
57
 
51
- def self.parse(file)
52
- compact(file)
58
+ def load
59
+ compact
53
60
 
54
- file_mutex = Mutex.new
55
61
  map = {}
56
- file.pos = 0
57
- file.each_line {|line|
58
- m = POSITION_FILE_ENTRY_REGEX.match(line)
59
- unless m
60
- $log.warn "Unparsable line in pos_file: #{line}"
61
- next
62
+ @file_mutex.synchronize do
63
+ @file.pos = 0
64
+
65
+ @file.each_line do |line|
66
+ m = POSITION_FILE_ENTRY_REGEX.match(line)
67
+ next if m.nil?
68
+
69
+ path = m[1]
70
+ pos = m[2].to_i(16)
71
+ ino = m[3].to_i(16)
72
+ seek = @file.pos - line.bytesize + path.bytesize + 1
73
+ map[path] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
62
74
  end
63
- path = m[1]
64
- pos = m[2].to_i(16)
65
- ino = m[3].to_i(16)
66
- seek = file.pos - line.bytesize + path.bytesize + 1
67
- map[path] = FilePositionEntry.new(file, file_mutex, seek, pos, ino)
68
- }
69
- new(file, file_mutex, map)
75
+ end
76
+
77
+ @map = map
78
+ end
79
+
80
+ # This method is similer to #compact but it tries to get less lock to avoid a lock contention
81
+ def try_compact
82
+ last_modified = nil
83
+ size = nil
84
+
85
+ @file_mutex.synchronize do
86
+ last_modified = @file.mtime
87
+ size = @file.size
88
+ end
89
+
90
+ entries = fetch_compacted_entries
91
+
92
+ @file_mutex.synchronize do
93
+ if last_modified == @file.mtime && size == @file.size
94
+ @file.pos = 0
95
+ @file.truncate(0)
96
+ @file.write(entries.join)
97
+ else
98
+ # skip
99
+ end
100
+ end
101
+ end
102
+
103
+ private
104
+
105
+ def compact
106
+ @file_mutex.synchronize do
107
+ entries = fetch_compacted_entries
108
+
109
+ @file.pos = 0
110
+ @file.truncate(0)
111
+ @file.write(entries.join)
112
+ end
70
113
  end
71
114
 
72
- # Clean up unwatched file entries
73
- def self.compact(file)
74
- existent_entries = {}
75
- file.pos = 0
76
- file.each_line do |line|
115
+ def fetch_compacted_entries
116
+ entries = {}
117
+
118
+ @file.pos = 0
119
+ @file.each_line do |line|
77
120
  m = POSITION_FILE_ENTRY_REGEX.match(line)
78
- unless m
79
- $log.warn "Unparsable line in pos_file: #{line}"
121
+ if m.nil?
122
+ @logger.warn "Unparsable line in pos_file: #{line}" if @logger
80
123
  next
81
124
  end
125
+
82
126
  path = m[1]
83
127
  pos = m[2].to_i(16)
84
128
  ino = m[3].to_i(16)
85
-
86
129
  if pos == UNWATCHED_POSITION
87
- next
88
- end
130
+ @logger.debug "Remove unwatched line from pos_file: #{line}" if @logger
131
+ else
132
+ if entries.include?(path)
133
+ @logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
134
+ end
89
135
 
90
- if existent_entries.include?(path)
91
- $log.warn("#{path} already exists. use latest one: deleted #{existent_entries[path]}")
136
+ entries[path] = (POSITION_FILE_ENTRY_FORMAT % [path, pos, ino])
92
137
  end
93
-
94
- # 32bit inode converted to 64bit at this phase
95
- existent_entries[path] = (POSITION_FILE_ENTRY_FORMAT % [path, pos, ino])
96
138
  end
97
139
 
98
- file.pos = 0
99
- file.truncate(0)
100
- file.write(existent_entries.values.join)
140
+ entries.values
101
141
  end
102
142
  end
103
143
 
@@ -27,8 +27,15 @@ module Fluent
27
27
  REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
28
28
  # From in_syslog default pattern
29
29
  REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
30
- REGEXP_RFC5424 = /\A(?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?\z/m
31
- REGEXP_RFC5424_WITH_PRI = /\A\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?\z/m
30
+ REGEXP_RFC5424 = <<~'EOS'.chomp
31
+ (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|(?:\[.*?(?<!\\)\])+))(?: (?<message>.+))?
32
+ EOS
33
+ REGEXP_RFC5424_NO_PRI = Regexp.new(<<~'EOS'.chomp % REGEXP_RFC5424, Regexp::MULTILINE)
34
+ \A%s\z
35
+ EOS
36
+ REGEXP_RFC5424_WITH_PRI = Regexp.new(<<~'EOS'.chomp % REGEXP_RFC5424, Regexp::MULTILINE)
37
+ \A<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} %s\z
38
+ EOS
32
39
  REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
33
40
 
34
41
  config_set_default :time_format, "%b %d %H:%M:%S"
@@ -73,7 +80,7 @@ module Fluent
73
80
  end
74
81
  @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
75
82
  @support_rfc5424_without_subseconds = true
76
- @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
83
+ @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424_NO_PRI
77
84
  when :auto
78
85
  class << self
79
86
  alias_method :parse, :parse_auto
@@ -96,7 +103,7 @@ module Fluent
96
103
 
97
104
  def parse_auto(text, &block)
98
105
  if REGEXP_DETECT_RFC5424.match(text)
99
- @regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
106
+ @regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424_NO_PRI
100
107
  @time_parser = @time_parser_rfc5424
101
108
  @support_rfc5424_without_subseconds = true
102
109
  parse_plain(text, &block)
@@ -15,6 +15,7 @@
15
15
  #
16
16
 
17
17
  require 'fileutils'
18
+ require 'open3'
18
19
 
19
20
  require 'fluent/config'
20
21
  require 'fluent/counter'
@@ -315,6 +316,7 @@ module Fluent
315
316
  )
316
317
  # this #init sets initialized logger to $log
317
318
  logger_initializer.init(:supervisor, 0)
319
+ logger_initializer.apply_options(format: params['log_format'], time_format: params['log_time_format'])
318
320
  logger = $log
319
321
 
320
322
  command_sender = Fluent.windows? ? "pipe" : "signal"
@@ -659,13 +661,7 @@ module Fluent
659
661
  Process.setproctitle("supervisor:#{@system_config.process_name}") if @system_config.process_name
660
662
  $log.info "starting fluentd-#{Fluent::VERSION}", pid: Process.pid, ruby: RUBY_VERSION
661
663
 
662
- rubyopt = ENV["RUBYOPT"]
663
- fluentd_spawn_cmd = [ServerEngine.ruby_bin_path, "-Eascii-8bit:ascii-8bit"]
664
- fluentd_spawn_cmd << rubyopt if rubyopt
665
- fluentd_spawn_cmd << $0
666
- fluentd_spawn_cmd += $fluentdargv
667
- fluentd_spawn_cmd << "--under-supervisor"
668
-
664
+ fluentd_spawn_cmd = build_spawn_command
669
665
  $log.info "spawn command to main: ", cmdline: fluentd_spawn_cmd
670
666
 
671
667
  params = {
@@ -689,6 +685,8 @@ module Fluent
689
685
  'rpc_endpoint' => @system_config.rpc_endpoint,
690
686
  'enable_get_dump' => @system_config.enable_get_dump,
691
687
  'counter_server' => @system_config.counter_server,
688
+ 'log_format' => @system_config.log.format,
689
+ 'log_time_format' => @system_config.log.time_format,
692
690
  }
693
691
 
694
692
  se = ServerEngine.create(ServerModule, WorkerModule){
@@ -872,5 +870,37 @@ module Fluent
872
870
  system_config.overwrite_variables(**opt)
873
871
  system_config
874
872
  end
873
+
874
+ RUBY_ENCODING_OPTIONS_REGEX = %r{\A(-E|--encoding=|--internal-encoding=|--external-encoding=)}.freeze
875
+
876
+ def build_spawn_command
877
+ fluentd_spawn_cmd = [ServerEngine.ruby_bin_path]
878
+
879
+ rubyopt = ENV['RUBYOPT']
880
+ if rubyopt
881
+ encodes, others = rubyopt.split(' ').partition { |e| e.match?(RUBY_ENCODING_OPTIONS_REGEX) }
882
+ fluentd_spawn_cmd.concat(others)
883
+
884
+ adopted_encodes = encodes.empty? ? ['-Eascii-8bit:ascii-8bit'] : encodes
885
+ fluentd_spawn_cmd.concat(adopted_encodes)
886
+ else
887
+ fluentd_spawn_cmd << '-Eascii-8bit:ascii-8bit'
888
+ end
889
+
890
+ # Adding `-h` so that it can avoid ruby's command blocking
891
+ # e.g. `ruby -Eascii-8bit:ascii-8bit` will block. but `ruby -Eascii-8bit:ascii-8bit -h` won't.
892
+ cmd = fluentd_spawn_cmd.join(' ')
893
+ _, e, s = Open3.capture3("#{cmd} -h")
894
+ if s.exitstatus != 0
895
+ $log.error('Invalid option is passed to RUBYOPT', command: cmd, error: e)
896
+ exit s.exitstatus
897
+ end
898
+
899
+ fluentd_spawn_cmd << $0
900
+ fluentd_spawn_cmd += $fluentdargv
901
+ fluentd_spawn_cmd << '--under-supervisor'
902
+
903
+ fluentd_spawn_cmd
904
+ end
875
905
  end
876
906
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.9.1'
19
+ VERSION = '1.9.2'
20
20
 
21
21
  end
@@ -47,16 +47,14 @@ class TestFluentdCommand < ::Test::Unit::TestCase
47
47
 
48
48
  def create_cmdline(conf_path, *fluentd_options)
49
49
  cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd")
50
- ["bundle", "exec", "ruby", cmd_path, "-c", conf_path, *fluentd_options]
50
+ ["bundle", "exec", cmd_path, "-c", conf_path, *fluentd_options]
51
51
  end
52
52
 
53
- def execute_command(cmdline, chdir=TMP_DIR)
53
+ def execute_command(cmdline, chdir=TMP_DIR, env = {})
54
54
  null_stream = File.open(File::NULL, 'w')
55
55
  gemfile_path = File.expand_path(File.dirname(__FILE__) + "../../../Gemfile")
56
56
 
57
- env = {
58
- "BUNDLE_GEMFILE" => gemfile_path,
59
- }
57
+ env = { "BUNDLE_GEMFILE" => gemfile_path }.merge(env)
60
58
  cmdname = cmdline.shift
61
59
  arg0 = "testing-fluentd"
62
60
  # p(here: "executing process", env: env, cmdname: cmdname, arg0: arg0, args: cmdline)
@@ -81,12 +79,12 @@ class TestFluentdCommand < ::Test::Unit::TestCase
81
79
  null_stream.close rescue nil
82
80
  end
83
81
 
84
- def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 10)
82
+ def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 10, env: {})
85
83
  matched = false
86
84
  assert_error_msg = "matched correctly"
87
85
  stdio_buf = ""
88
86
  begin
89
- execute_command(cmdline) do |pid, stdout|
87
+ execute_command(cmdline, TMP_DIR, env) do |pid, stdout|
90
88
  begin
91
89
  waiting(timeout) do
92
90
  while process_exist?(pid) && !matched
@@ -285,7 +283,7 @@ CONF
285
283
 
286
284
  assert_fluentd_fails_to_start(
287
285
  create_cmdline(conf_path),
288
- "non directory entry exists:#{@root_path} (Fluent::InvalidRootDirectory)",
286
+ "non directory entry exists:#{@root_path}",
289
287
  )
290
288
  end
291
289
  end
@@ -780,6 +778,82 @@ CONF
780
778
  )
781
779
  end
782
780
 
781
+ test "multiple values are set to RUBYOPT" do
782
+ conf = <<CONF
783
+ <source>
784
+ @type dummy
785
+ tag dummy
786
+ </source>
787
+ <match>
788
+ @type null
789
+ </match>
790
+ CONF
791
+ conf_path = create_conf_file('rubyopt_test.conf', conf)
792
+ assert_log_matches(
793
+ create_cmdline(conf_path),
794
+ '#0 fluentd worker is now running worker=0',
795
+ patterns_not_match: ['(LoadError)'],
796
+ env: { 'RUBYOPT' => '-rtest-unit -rbundler/setup' },
797
+ )
798
+ end
799
+
800
+ data(
801
+ '-E' => '-Eutf-8',
802
+ '-encoding' => '--encoding=utf-8',
803
+ '-external-encoding' => '--external-encoding=utf-8',
804
+ '-internal-encoding' => '--internal-encoding=utf-8',
805
+ )
806
+ test "-E option is set to RUBYOPT3" do |opt|
807
+ conf = <<CONF
808
+ <source>
809
+ @type dummy
810
+ tag dummy
811
+ </source>
812
+ <match>
813
+ @type null
814
+ </match>
815
+ CONF
816
+ conf_path = create_conf_file('rubyopt_test.conf', conf)
817
+ assert_log_matches(
818
+ create_cmdline(conf_path),
819
+ *opt.split(' '),
820
+ patterns_not_match: ['-Eascii-8bit:ascii-8bit'],
821
+ env: { 'RUBYOPT' => opt },
822
+ )
823
+ end
824
+
825
+ test "without RUBYOPT" do
826
+ conf = <<CONF
827
+ <source>
828
+ @type dummy
829
+ tag dummy
830
+ </source>
831
+ <match>
832
+ @type null
833
+ </match>
834
+ CONF
835
+ conf_path = create_conf_file('rubyopt_test.conf', conf)
836
+ assert_log_matches(create_cmdline(conf_path), '-Eascii-8bit:ascii-8bit')
837
+ end
838
+
839
+ test 'invalid values are set to RUBYOPT' do
840
+ conf = <<CONF
841
+ <source>
842
+ @type dummy
843
+ tag dummy
844
+ </source>
845
+ <match>
846
+ @type null
847
+ </match>
848
+ CONF
849
+ conf_path = create_conf_file('rubyopt_invalid_test.conf', conf)
850
+ assert_log_matches(
851
+ create_cmdline(conf_path),
852
+ 'Invalid option is passed to RUBYOPT',
853
+ env: { 'RUBYOPT' => 'a' },
854
+ )
855
+ end
856
+
783
857
  test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
784
858
  conf = <<CONF
785
859
  <system>
@@ -1,9 +1,17 @@
1
1
  require_relative '../../helper'
2
2
  require 'fluent/plugin/in_tail/position_file'
3
3
 
4
+ require 'fileutils'
5
+ require 'tempfile'
6
+
4
7
  class IntailPositionFileTest < Test::Unit::TestCase
5
8
  setup do
6
- @file = StringIO.new(+'')
9
+ @file = Tempfile.new('intail_position_file_test')
10
+ end
11
+
12
+ teardown do
13
+ @file.close rescue nil
14
+ @file.unlink rescue nil
7
15
  end
8
16
 
9
17
  UNWATCHED_STR = '%016x' % Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
@@ -19,10 +27,21 @@ class IntailPositionFileTest < Test::Unit::TestCase
19
27
  f.seek(0)
20
28
  end
21
29
 
22
- sub_test_case '.compact' do
30
+ test '.load' do
31
+ write_data(@file, TEST_CONTENT)
32
+ Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
33
+
34
+ @file.seek(0)
35
+ lines = @file.readlines
36
+ assert_equal 2, lines.size
37
+ assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0]
38
+ assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
39
+ end
40
+
41
+ sub_test_case '#try_compact' do
23
42
  test 'compact invalid and convert 32 bit inode value' do
24
43
  write_data(@file, TEST_CONTENT)
25
- Fluent::Plugin::TailInput::PositionFile.compact(@file)
44
+ Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
26
45
 
27
46
  @file.seek(0)
28
47
  lines = @file.readlines
@@ -36,29 +55,59 @@ class IntailPositionFileTest < Test::Unit::TestCase
36
55
  valid_path\t0000000000000002\t0000000000000001
37
56
  valid_path\t0000000000000003\t0000000000000004
38
57
  EOF
39
- Fluent::Plugin::TailInput::PositionFile.compact(@file)
58
+ Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
40
59
 
41
60
  @file.seek(0)
42
61
  lines = @file.readlines
43
62
  assert_equal "valid_path\t0000000000000003\t0000000000000004\n", lines[0]
44
63
  end
64
+
65
+ test 'does not change when the file is changed' do
66
+ write_data(@file, TEST_CONTENT)
67
+ pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
68
+
69
+ mock.proxy(pf).fetch_compacted_entries do |r|
70
+ @file.write("unwatched\t#{UNWATCHED_STR}\t0000000000000000\n")
71
+ r
72
+ end
73
+
74
+ pf.try_compact
75
+
76
+ @file.seek(0)
77
+ lines = @file.readlines
78
+ assert_equal 5, lines.size
79
+ end
45
80
  end
46
81
 
47
- test '.parse' do
48
- write_data(@file, TEST_CONTENT)
49
- Fluent::Plugin::TailInput::PositionFile.parse(@file)
82
+ sub_test_case '#load' do
83
+ test 'compact invalid and convert 32 bit inode value' do
84
+ write_data(@file, TEST_CONTENT)
85
+ Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
50
86
 
51
- @file.seek(0)
52
- lines = @file.readlines
53
- assert_equal 2, lines.size
54
- assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0]
55
- assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
87
+ @file.seek(0)
88
+ lines = @file.readlines
89
+ assert_equal 2, lines.size
90
+ assert_equal "valid_path\t0000000000000002\t0000000000000001\n", lines[0]
91
+ assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
92
+ end
93
+
94
+ test 'compact data if duplicated line' do
95
+ write_data(@file, <<~EOF)
96
+ valid_path\t0000000000000002\t0000000000000001
97
+ valid_path\t0000000000000003\t0000000000000004
98
+ EOF
99
+ Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).load
100
+
101
+ @file.seek(0)
102
+ lines = @file.readlines
103
+ assert_equal "valid_path\t0000000000000003\t0000000000000004\n", lines[0]
104
+ end
56
105
  end
57
106
 
58
107
  sub_test_case '#[]' do
59
108
  test 'return entry' do
60
109
  write_data(@file, TEST_CONTENT)
61
- pf = Fluent::Plugin::TailInput::PositionFile.parse(@file)
110
+ pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
62
111
 
63
112
  f = pf['valid_path']
64
113
  assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
@@ -82,7 +131,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
82
131
 
83
132
  test 'does not change other value position if other entry try to write' do
84
133
  write_data(@file, TEST_CONTENT)
85
- pf = Fluent::Plugin::TailInput::PositionFile.parse(@file)
134
+ pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
86
135
 
87
136
  f = pf['nonexist_path']
88
137
  assert_equal 0, f.read_inode
@@ -103,7 +152,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
103
152
  sub_test_case '#unwatch' do
104
153
  test 'deletes entry by path' do
105
154
  write_data(@file, TEST_CONTENT)
106
- pf = Fluent::Plugin::TailInput::PositionFile.parse(@file)
155
+ pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
107
156
  p1 = pf['valid_path']
108
157
  assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
109
158
 
@@ -1088,7 +1088,7 @@ class TailInputTest < Test::Unit::TestCase
1088
1088
  plugin = create_driver(EX_CONFIG, false).instance
1089
1089
  sio = StringIO.new
1090
1090
  plugin.instance_eval do
1091
- @pf = Fluent::Plugin::TailInput::PositionFile.parse(sio)
1091
+ @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, logger: $log)
1092
1092
  @loop = Coolio::Loop.new
1093
1093
  end
1094
1094
 
@@ -237,7 +237,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
237
237
  assert_equal "-", record["extradata"]
238
238
  assert_equal "Hi, from Fluentd!", record["message"]
239
239
  end
240
- assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424,
240
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_NO_PRI,
241
241
  @parser.instance.patterns['format'])
242
242
  end
243
243
 
@@ -254,7 +254,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
254
254
  assert_equal "-", record["extradata"]
255
255
  assert_nil record["message"]
256
256
  end
257
- assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424,
257
+ assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_NO_PRI,
258
258
  @parser.instance.patterns['format'])
259
259
  end
260
260
 
@@ -294,14 +294,14 @@ class SyslogParserTest < ::Test::Unit::TestCase
294
294
  'message_format' => 'rfc5424',
295
295
  'with_priority' => true,
296
296
  )
297
- text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
297
+ text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] [Hi] from Fluentd!'
298
298
  @parser.instance.parse(text) do |time, record|
299
299
  assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
300
300
  assert_equal "11111", record["pid"]
301
301
  assert_equal "ID24224", record["msgid"]
302
302
  assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
303
303
  record["extradata"]
304
- assert_equal "Hi, from Fluentd!", record["message"]
304
+ assert_equal "[Hi] from Fluentd!", record["message"]
305
305
  end
306
306
  end
307
307
 
@@ -328,14 +328,14 @@ class SyslogParserTest < ::Test::Unit::TestCase
328
328
  'message_format' => 'rfc5424',
329
329
  'with_priority' => true,
330
330
  )
331
- text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd]!'
331
+ text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] [Hi] from Fluentd]!'
332
332
  @parser.instance.parse(text) do |time, record|
333
333
  assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
334
334
  assert_equal "11111", record["pid"]
335
335
  assert_equal "ID24224", record["msgid"]
336
336
  assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
337
337
  record["extradata"]
338
- assert_equal "Hi, from Fluentd]!", record["message"]
338
+ assert_equal "[Hi] from Fluentd]!", record["message"]
339
339
  end
340
340
  end
341
341
 
@@ -235,6 +235,33 @@ class SupervisorTest < ::Test::Unit::TestCase
235
235
  Timecop.return
236
236
  end
237
237
 
238
+ def test_load_config_for_logger
239
+ tmp_dir = "#{TMP_DIR}/dir/test_load_config_log.conf"
240
+ conf_info_str = %[
241
+ <system>
242
+ <log>
243
+ format json
244
+ time_format %FT%T.%L%z
245
+ </log>
246
+ </system>
247
+ ]
248
+ write_config tmp_dir, conf_info_str
249
+ params = {
250
+ 'use_v1_config' => true,
251
+ 'conf_encoding' => 'utf8',
252
+ 'log_level' => Fluent::Log::LEVEL_INFO,
253
+ 'log_path' => 'test/tmp/supervisor/log',
254
+
255
+ 'workers' => 1,
256
+ 'log_format' => :json,
257
+ 'log_time_format' => '%FT%T.%L%z',
258
+ }
259
+
260
+ r = Fluent::Supervisor.load_config(tmp_dir, params)
261
+ assert_equal :json, r[:logger].format
262
+ assert_equal '%FT%T.%L%z', r[:logger].time_format
263
+ end
264
+
238
265
  def test_load_config_for_daemonize
239
266
  tmp_dir = "#{TMP_DIR}/dir/test_load_config.conf"
240
267
  conf_info_str = %[
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.9.1
4
+ version: 1.9.2
5
5
  platform: x64-mingw32
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-01-31 00:00:00.000000000 Z
11
+ date: 2020-02-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack