fluentd 1.7.0 → 1.7.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: aa26c04f10203245130ee2457e56f1ea064df188d91c3610aa338f5f66273663
4
- data.tar.gz: 48031fcf5dc26c8d1e3a6f6fc4a5f629451c6dc9a06c4a95043c3ba56da031b1
3
+ metadata.gz: 5bb8d41e15ede72bfbb90c06171c0eb4aa77d479cda1a8320f60e6166fbc27fc
4
+ data.tar.gz: d781e152f408a74e1cfed6b2c37f593db814d551d9f9179d1c5d15581218bf43
5
5
  SHA512:
6
- metadata.gz: ac003c0a88bfd623a9e1caf2989db9e45b38841827dc312e4e266ff38cc1ac25001a58fda0fc5b7eebb9b4efa4bfc9cbf0266ad36314aa5ed518303040e09727
7
- data.tar.gz: b84fb3fe1b06ee99fcdad4cbc37d897b507b5b916d5ad8b539863827e6d1875ca6c026ae54b6b05898750e84357db51583e59872ee76bc814e3bc7e29e2e3fc2
6
+ metadata.gz: 1f04e656acafae78bf4e2a770a5df16d549aa59ecd0a5883e97884b1de2dccc612108b11a1302b0655a2d56e3fb0b5f077da345441f34ce94a6bc5d31b3fac61
7
+ data.tar.gz: d16a3635f5c9607de95f66f3b836c44786abd7db9a4bb369b51f41862df0279bc3812962c633fe8ac8ad9f4439fe22b8aaf18ff7aa099dafe531956c0a33985e
@@ -1,11 +1,36 @@
1
1
  # v1.7
2
2
 
3
+ ## Release v1.7.1 - 2019/09/08
4
+
5
+ ### Enhancement
6
+
7
+ * socket helper/out_forward: Support Windows certstore to load certificates
8
+ https://github.com/fluent/fluentd/pull/2601
9
+ * parser_syslog: Add faster parser for rfc3164 message
10
+ https://github.com/fluent/fluentd/pull/2599
11
+
12
+ ### Bug fixes
13
+
14
+ * buf_file/buf_file_single: fix to ignore placeholder based path.
15
+ https://github.com/fluent/fluentd/pull/2594
16
+ * server helper: Ignore ETIMEDOUT error in SSL_accept
17
+ https://github.com/fluent/fluentd/pull/2595
18
+ * buf_file: ensure to remove metadata after buffer creation failure
19
+ https://github.com/fluent/fluentd/pull/2598
20
+ * buf_file_single: fix duplicated path setting check
21
+ https://github.com/fluent/fluentd/pull/2600
22
+ * fix msgpack-ruby depedency to use recent feature
23
+ https://github.com/fluent/fluentd/pull/2606
24
+
25
+
3
26
  ## Release v1.7.0 - 2019/08/20
4
27
 
5
28
  ### New feature
6
29
 
7
- * buffer: Add file_single buffer
8
- https://github.com/fluent/fluentd/pull/2479
30
+ * buffer: Add file_single buffer plugin
31
+ https://github.com/fluent/fluentd/pull/2579
32
+ * output: Add http output plugin
33
+ https://github.com/fluent/fluentd/pull/2488
9
34
 
10
35
  ### Enhancement
11
36
 
@@ -13,9 +38,9 @@
13
38
  https://github.com/fluent/fluentd/pull/2560
14
39
  https://github.com/fluent/fluentd/pull/2563
15
40
  https://github.com/fluent/fluentd/pull/2564
16
- * output: Use Mutext instead of Monitor
41
+ * output: Use Mutex instead of Monitor
17
42
  https://github.com/fluent/fluentd/pull/2561
18
- * event: Add `OneEvent#empty?` method
43
+ * event: Add `OneEventStrea#empty?` method
19
44
  https://github.com/fluent/fluentd/pull/2565
20
45
  * thread: Set thread name for ruby 2.3 or later
21
46
  https://github.com/fluent/fluentd/pull/2574
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
18
18
 
19
19
  gem.required_ruby_version = '>= 2.1'
20
20
 
21
- gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
21
+ gem.add_runtime_dependency("msgpack", [">= 1.2.0", "< 2.0.0"])
22
22
  gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
23
23
  gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
24
24
  gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
@@ -37,6 +37,7 @@ Gem::Specification.new do |gem|
37
37
  gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
38
38
  gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
39
39
  gem.add_runtime_dependency("windows-pr", ["~> 1.2.5"])
40
+ gem.add_runtime_dependency("certstore_c", ["~> 0.1.2"])
40
41
  end
41
42
 
42
43
  gem.add_development_dependency("rake", ["~> 11.0"])
@@ -132,7 +132,7 @@ module Fluent
132
132
 
133
133
  patterns = [@path]
134
134
  patterns.unshift @additional_resume_path if @additional_resume_path
135
- Dir.glob(patterns) do |path|
135
+ Dir.glob(escaped_patterns(patterns)) do |path|
136
136
  next unless File.file?(path)
137
137
 
138
138
  log.debug { "restoring buffer file: path = #{path}" }
@@ -183,6 +183,15 @@ module Fluent
183
183
  # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
184
184
  File.unlink(path, path + '.meta') rescue nil
185
185
  end
186
+
187
+ private
188
+
189
+ def escaped_patterns(patterns)
190
+ patterns.map { |pattern|
191
+ # '{' '}' are special character in Dir.glob
192
+ pattern.gsub(/[\{\}]/) { |c| "\\#{c}" }
193
+ }
194
+ end
186
195
  end
187
196
  end
188
197
  end
@@ -88,14 +88,6 @@ module Fluent
88
88
  end
89
89
  end
90
90
 
91
- type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
92
- if @@buffer_paths.has_key?(@path) && !called_in_test?
93
- type_using_this_path = @@buffer_paths[@path]
94
- raise Fluent::ConfigError, "Other '#{type_using_this_path}' plugin already uses same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
95
- end
96
-
97
- @@buffer_paths[@path] = type_of_owner
98
-
99
91
  specified_directory_exists = File.exist?(@path) && File.directory?(@path)
100
92
  unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')
101
93
 
@@ -124,6 +116,13 @@ module Fluent
124
116
  @multi_workers_available = false
125
117
  end
126
118
 
119
+ type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
120
+ if @@buffer_paths.has_key?(@path) && !called_in_test?
121
+ type_using_this_path = @@buffer_paths[@path]
122
+ raise Fluent::ConfigError, "Other '#{type_using_this_path}' plugin already uses same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
123
+ end
124
+
125
+ @@buffer_paths[@path] = type_of_owner
127
126
  @dir_permission = if @dir_permission
128
127
  @dir_permission.to_i(8)
129
128
  else
@@ -155,7 +154,7 @@ module Fluent
155
154
 
156
155
  patterns = [@path]
157
156
  patterns.unshift @additional_resume_path if @additional_resume_path
158
- Dir.glob(patterns) do |path|
157
+ Dir.glob(escaped_patterns(patterns)) do |path|
159
158
  next unless File.file?(path)
160
159
 
161
160
  log.debug { "restoring buffer file: path = #{path}" }
@@ -206,6 +205,15 @@ module Fluent
206
205
  # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
207
206
  File.unlink(path) rescue nil
208
207
  end
208
+
209
+ private
210
+
211
+ def escaped_patterns(patterns)
212
+ patterns.map { |pattern|
213
+ # '{' '}' are special character in Dir.glob
214
+ pattern.gsub(/[\{\}]/) { |c| "\\#{c}" }
215
+ }
216
+ end
209
217
  end
210
218
  end
211
219
  end
@@ -300,6 +300,13 @@ module Fluent
300
300
  # This case is easier than enqueued!. Just removing pre-create buffer file
301
301
  @chunk.close rescue nil
302
302
  File.unlink(@path) rescue nil
303
+
304
+ if @meta
305
+ # ensure to unlink when #write_metadata fails
306
+ @meta.close rescue nil
307
+ File.unlink(@meta_path) rescue nil
308
+ end
309
+
303
310
  # Same as @chunk case. See above
304
311
  raise BufferOverflowError, "can't create buffer metadata for #{path}. Stop creating buffer files: error = #{e}"
305
312
  end
@@ -107,6 +107,12 @@ module Fluent::Plugin
107
107
  config_param :tls_client_private_key_path, :string, default: nil
108
108
  desc 'The client private key passphrase for TLS.'
109
109
  config_param :tls_client_private_key_passphrase, :string, default: nil, secret: true
110
+ desc 'The certificate thumbprint for searching from Windows system certstore.'
111
+ config_param :tls_cert_thumbprint, :string, default: nil, secret: true
112
+ desc 'The certificate logical store name on Windows system certstore.'
113
+ config_param :tls_cert_logical_store_name, :string, default: nil
114
+ desc 'Enable to use certificate enterprise store on Windows system certstore.'
115
+ config_param :tls_cert_use_enterprise_store, :bool, default: true
110
116
  desc "Enable keepalive connection."
111
117
  config_param :keepalive, :bool, default: false
112
118
  desc "Expired time of keepalive. Default value is nil, which means to keep connection as long as possible"
@@ -198,6 +204,15 @@ module Fluent::Plugin
198
204
  @tls_verify_hostname = false
199
205
  @tls_allow_self_signed_cert = true
200
206
  end
207
+
208
+ if Fluent.windows?
209
+ if (@tls_cert_path || @tls_ca_cert_path) && @tls_cert_logical_store_name
210
+ raise Fluent::ConfigError, "specified both cert path and tls_cert_logical_store_name is not permitted"
211
+ end
212
+ else
213
+ raise Fluent::ConfigError, "This parameter is for only Windows" if @tls_cert_logical_store_name
214
+ raise Fluent::ConfigError, "This parameter is for only Windows" if @tls_cert_thumbprint
215
+ end
201
216
  end
202
217
 
203
218
  @ack_handler = @require_ack_response ? AckHandler.new(timeout: @ack_response_timeout, log: @log, read_length: @read_length) : nil
@@ -346,6 +361,9 @@ module Fluent::Plugin
346
361
  cert_path: @tls_client_cert_path,
347
362
  private_key_path: @tls_client_private_key_path,
348
363
  private_key_passphrase: @tls_client_private_key_passphrase,
364
+ cert_thumbprint: @tls_cert_thumbprint,
365
+ cert_logical_store_name: @tls_cert_logical_store_name,
366
+ cert_use_enterprise_store: @tls_cert_use_enterprise_store,
349
367
 
350
368
  # Enabling SO_LINGER causes data loss on Windows
351
369
  # https://github.com/fluent/fluentd/issues/1968
@@ -38,6 +38,10 @@ module Fluent
38
38
  config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
39
39
  desc 'Specify time format for event time for rfc5424 protocol'
40
40
  config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
41
+ desc 'The parser type used to parse syslog message'
42
+ config_param :parser_type, :enum, list: [:regexp, :string], default: :regexp
43
+ desc 'support colonless ident in string parser'
44
+ config_param :support_colonless_ident, :bool, default: true
41
45
 
42
46
  def initialize
43
47
  super
@@ -50,10 +54,17 @@ module Fluent
50
54
  @time_parser_rfc3164 = @time_parser_rfc5424 = nil
51
55
  @time_parser_rfc5424_without_subseconds = nil
52
56
  @support_rfc5424_without_subseconds = false
57
+ @regexp_parser = @parser_type == :regexp
53
58
  @regexp = case @message_format
54
59
  when :rfc3164
55
- class << self
56
- alias_method :parse, :parse_plain
60
+ if @regexp_parser
61
+ class << self
62
+ alias_method :parse, :parse_plain
63
+ end
64
+ else
65
+ class << self
66
+ alias_method :parse, :parse_rfc3164
67
+ end
57
68
  end
58
69
  @with_priority ? REGEXP_WITH_PRI : REGEXP
59
70
  when :rfc5424
@@ -88,11 +99,16 @@ module Fluent
88
99
  @regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
89
100
  @time_parser = @time_parser_rfc5424
90
101
  @support_rfc5424_without_subseconds = true
102
+ parse_plain(text, &block)
91
103
  else
92
104
  @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
93
105
  @time_parser = @time_parser_rfc3164
106
+ if @regexp_parser
107
+ parse_plain(text, &block)
108
+ else
109
+ parse_rfc3164(text, &block)
110
+ end
94
111
  end
95
- parse_plain(text, &block)
96
112
  end
97
113
 
98
114
  def parse_plain(text, &block)
@@ -137,6 +153,93 @@ module Fluent
137
153
 
138
154
  yield time, record
139
155
  end
156
+
157
+ SPLIT_CHAR = ' '.freeze
158
+
159
+ def parse_rfc3164(text, &block)
160
+ pri = nil
161
+ cursor = 0
162
+ if @with_priority
163
+ if text.start_with?('<'.freeze)
164
+ i = text.index('>'.freeze, 1)
165
+ if i < 2
166
+ yield nil, nil
167
+ return
168
+ end
169
+ pri = text.slice(1, i - 1).to_i
170
+ cursor = i + 1
171
+ else
172
+ yield nil, nil
173
+ return
174
+ end
175
+ end
176
+
177
+ # header part
178
+ time_size = 15 # skip Mmm dd hh:mm:ss
179
+ time_end = text[cursor + time_size]
180
+ if time_end == SPLIT_CHAR
181
+ time_str = text.slice(cursor, time_size)
182
+ cursor += 16 # time + ' '
183
+ elsif time_end == '.'.freeze
184
+ # support subsecond time
185
+ i = text.index(SPLIT_CHAR, time_size)
186
+ time_str = text.slice(cursor, i - cursor)
187
+ cursor = i + 1
188
+ else
189
+ yield nil, nil
190
+ return
191
+ end
192
+
193
+ i = text.index(SPLIT_CHAR, cursor)
194
+ if i.nil?
195
+ yield nil, nil
196
+ return
197
+ end
198
+ host_size = i - cursor
199
+ host = text.slice(cursor, host_size)
200
+ cursor += host_size + 1
201
+
202
+ record = {'host' => host}
203
+ record['pri'] = pri if pri
204
+
205
+ i = text.index(SPLIT_CHAR, cursor)
206
+
207
+ # message part
208
+ msg = if i.nil? # for 'only non-space content case'
209
+ text.slice(cursor, text.bytesize)
210
+ else
211
+ if text[i - 1] == ':'.freeze
212
+ if text[i - 2] == ']'.freeze
213
+ left_braket_pos = text.index('['.freeze, cursor)
214
+ record['ident'] = text.slice(cursor, left_braket_pos - cursor)
215
+ record['pid'] = text.slice(left_braket_pos + 1, i - left_braket_pos - 3) # remove '[' / ']:'
216
+ else
217
+ record['ident'] = text.slice(cursor, i - cursor - 1)
218
+ end
219
+ text.slice(i + 1, text.bytesize)
220
+ else
221
+ if @support_colonless_ident
222
+ if text[i - 1] == ']'.freeze
223
+ left_braket_pos = text.index('['.freeze, cursor)
224
+ record['ident'] = text.slice(cursor, left_braket_pos - cursor)
225
+ record['pid'] = text.slice(left_braket_pos + 1, i - left_braket_pos - 2) # remove '[' / ']'
226
+ else
227
+ record['ident'] = text.slice(cursor, i - cursor)
228
+ end
229
+ text.slice(i + 1, text.bytesize)
230
+ else
231
+ text.slice(cursor, text.bytesize)
232
+ end
233
+ end
234
+ end
235
+ msg.chomp!
236
+ record['message'] = msg
237
+
238
+ time = @time_parser.parse(time_str)
239
+ record['time'] = time_str if @keep_time_key
240
+
241
+ yield time, record
242
+ end
140
243
  end
141
244
  end
142
245
  end
@@ -722,7 +722,7 @@ module Fluent
722
722
 
723
723
  return true
724
724
  end
725
- rescue Errno::EPIPE, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
725
+ rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, OpenSSL::SSL::SSLError => e
726
726
  @log.trace "unexpected error before accepting TLS connection", error: e
727
727
  close rescue nil
728
728
  end
@@ -17,6 +17,9 @@
17
17
  require 'socket'
18
18
  require 'ipaddr'
19
19
  require 'openssl'
20
+ if Fluent.windows?
21
+ require 'certstore'
22
+ end
20
23
 
21
24
  require_relative 'socket_option'
22
25
 
@@ -96,7 +99,9 @@ module Fluent
96
99
  host, port,
97
100
  version: TLS_DEFAULT_VERSION, ciphers: CIPHERS_DEFAULT, insecure: false, verify_fqdn: true, fqdn: nil,
98
101
  enable_system_cert_store: true, allow_self_signed_cert: false, cert_paths: nil,
99
- cert_path: nil, private_key_path: nil, private_key_passphrase: nil, **kwargs, &block)
102
+ cert_path: nil, private_key_path: nil, private_key_passphrase: nil,
103
+ cert_thumbprint: nil, cert_logical_store_name: nil, cert_use_enterprise_store: true,
104
+ **kwargs, &block)
100
105
 
101
106
  host_is_ipaddress = IPAddr.new(host) rescue false
102
107
  fqdn ||= host unless host_is_ipaddress
@@ -113,6 +118,14 @@ module Fluent
113
118
  end
114
119
  begin
115
120
  if enable_system_cert_store
121
+ if Fluent.windows? && cert_logical_store_name
122
+ log.trace "loading Windows system certificate store"
123
+ loader = Certstore::OpenSSL::Loader.new(log, cert_store, cert_logical_store_name,
124
+ enterprise: cert_use_enterprise_store)
125
+ loader.load_cert_store
126
+ cert_store = loader.cert_store
127
+ context.cert = loader.get_certificate(cert_thumbprint) if cert_thumbprint
128
+ end
116
129
  log.trace "loading system default certificate store"
117
130
  cert_store.set_default_paths
118
131
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.7.0'
19
+ VERSION = '1.7.1'
20
20
 
21
21
  end
@@ -537,6 +537,60 @@ class FileBufferTest < Test::Unit::TestCase
537
537
  end
538
538
  end
539
539
 
540
+ sub_test_case 'there are some existing file chunks with placeholders path' do
541
+ setup do
542
+ @bufdir = File.expand_path('../../tmp/buffer_${test}_file', __FILE__)
543
+ FileUtils.rm_rf(@bufdir)
544
+ FileUtils.mkdir_p(@bufdir)
545
+
546
+ @c1id = Fluent::UniqueId.generate
547
+ p1 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c1id)}.log")
548
+ File.open(p1, 'wb') do |f|
549
+ f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
550
+ end
551
+ write_metadata(
552
+ p1 + '.meta', @c1id, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),
553
+ 1, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i
554
+ )
555
+
556
+ @c2id = Fluent::UniqueId.generate
557
+ p2 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c2id)}.log")
558
+ File.open(p2, 'wb') do |f|
559
+ f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
560
+ end
561
+ write_metadata(
562
+ p2 + '.meta', @c2id, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),
563
+ 1, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i
564
+ )
565
+
566
+ @bufpath = File.join(@bufdir, 'etest.*.log')
567
+
568
+ Fluent::Test.setup
569
+ @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
570
+ @p = Fluent::Plugin::FileBuffer.new
571
+ @p.owner = @d
572
+ @p.configure(config_element('buffer', '', {'path' => @bufpath}))
573
+ @p.start
574
+ end
575
+
576
+ teardown do
577
+ if @p
578
+ @p.stop unless @p.stopped?
579
+ @p.before_shutdown unless @p.before_shutdown?
580
+ @p.shutdown unless @p.shutdown?
581
+ @p.after_shutdown unless @p.after_shutdown?
582
+ @p.close unless @p.closed?
583
+ @p.terminate unless @p.terminated?
584
+ end
585
+ FileUtils.rm_rf(@bufdir)
586
+ end
587
+
588
+ test '#resume returns staged/queued chunks with metadata' do
589
+ assert_equal 1, @p.stage.size
590
+ assert_equal 1, @p.queue.size
591
+ end
592
+ end
593
+
540
594
  sub_test_case 'there are some existing file chunks, both in specified path and per-worker directory under specified path, configured as multi workers' do
541
595
  setup do
542
596
  @bufdir = File.expand_path('../../tmp/buffer_file/path', __FILE__)
@@ -138,6 +138,25 @@ class FileSingleBufferTest < Test::Unit::TestCase
138
138
  end
139
139
  end
140
140
 
141
+ test 'raise config error when using same file path' do
142
+ d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
143
+ d2 = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
144
+ Fluent::SystemConfig.overwrite_system_config({}) do
145
+ d.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => File.join(PATH, 'foo.*.bar') })]))
146
+ end
147
+
148
+ any_instance_of(Fluent::Plugin::FileSingleBuffer) do |klass|
149
+ stub(klass).called_in_test? { false }
150
+ end
151
+
152
+ err = assert_raise(Fluent::ConfigError) do
153
+ Fluent::SystemConfig.overwrite_system_config({}) do
154
+ d2.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => PATH })]))
155
+ end
156
+ end
157
+ assert_match(/plugin already uses same buffer path/, err.message)
158
+ end
159
+
141
160
  sub_test_case 'buffer plugin configured only with path' do
142
161
  setup do
143
162
  @bufpath = File.join(@bufdir, 'testbuf.*.buf')
@@ -502,6 +521,54 @@ class FileSingleBufferTest < Test::Unit::TestCase
502
521
  end
503
522
  end
504
523
 
524
+ sub_test_case 'there are some existing file chunks with placeholders path' do
525
+ setup do
526
+ @buf_ph_dir = File.expand_path('../../tmp/buffer_${test}_file_single_dir', __FILE__)
527
+ FileUtils.rm_rf(@buf_ph_dir)
528
+ FileUtils.mkdir_p(@buf_ph_dir)
529
+
530
+ @c1id = Fluent::UniqueId.generate
531
+ p1 = File.join(@buf_ph_dir, "fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf")
532
+ File.open(p1, 'wb') do |f|
533
+ f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
534
+ end
535
+ t = Time.now - 50000
536
+ File.utime(t, t, p1)
537
+
538
+ @c2id = Fluent::UniqueId.generate
539
+ p2 = File.join(@buf_ph_dir, "fsb.testing.b#{Fluent::UniqueId.hex(@c2id)}.buf")
540
+ File.open(p2, 'wb') do |f|
541
+ f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
542
+ end
543
+ end
544
+
545
+ teardown do
546
+ if @p
547
+ @p.stop unless @p.stopped?
548
+ @p.before_shutdown unless @p.before_shutdown?
549
+ @p.shutdown unless @p.shutdown?
550
+ @p.after_shutdown unless @p.after_shutdown?
551
+ @p.close unless @p.closed?
552
+ @p.terminate unless @p.terminated?
553
+ end
554
+ FileUtils.rm_rf(@buf_ph_dir)
555
+ end
556
+
557
+ test '#resume returns staged/queued chunks with metadata' do
558
+ @d = create_driver(%[
559
+ <buffer tag>
560
+ @type file_single
561
+ path #{@buf_ph_dir}
562
+ </buffer>
563
+ ])
564
+ @p = @d.instance.buffer
565
+ @p.start
566
+
567
+ assert_equal 1, @p.stage.size
568
+ assert_equal 1, @p.queue.size
569
+ end
570
+ end
571
+
505
572
  sub_test_case 'there are some existing msgpack file chunks' do
506
573
  setup do
507
574
  packer = Fluent::MessagePackFactory.packer
@@ -495,6 +495,24 @@ class BufferFileChunkTest < Test::Unit::TestCase
495
495
  end
496
496
  end
497
497
 
498
+ test 'ensure to remove metadata file if #write_metadata raise an error becuase of disk full' do
499
+ chunk_path = File.join(@chunkdir, 'test.*.log')
500
+ stub(Fluent::UniqueId).hex(anything) { 'id' } # to fix chunk id
501
+
502
+ any_instance_of(Fluent::Plugin::Buffer::FileChunk) do |klass|
503
+ stub(klass).write_metadata(anything) do |v|
504
+ raise 'disk full'
505
+ end
506
+ end
507
+
508
+ err = assert_raise(Fluent::Plugin::Buffer::BufferOverflowError) do
509
+ Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, chunk_path, :create)
510
+ end
511
+
512
+ assert_false File.exist?(File.join(@chunkdir, 'test.bid.log.meta'))
513
+ assert_match(/create buffer metadata/, err.message)
514
+ end
515
+
498
516
  sub_test_case 'chunk with file for staged chunk' do
499
517
  setup do
500
518
  @chunk_id = gen_test_chunk_id
@@ -855,6 +873,7 @@ class BufferFileChunkTest < Test::Unit::TestCase
855
873
  assert_equal @gzipped_src, c.read(compressed: :gzip)
856
874
 
857
875
  io = StringIO.new
876
+ io.set_encoding(Encoding::ASCII_8BIT)
858
877
  c.write_to(io, compressed: :gzip)
859
878
  assert_equal @gzipped_src, io.string
860
879
  end
@@ -613,6 +613,7 @@ class BufferFileSingleChunkTest < Test::Unit::TestCase
613
613
  assert_equal @gzipped_src, c.read(compressed: :gzip)
614
614
 
615
615
  io = StringIO.new
616
+ io.set_encoding(Encoding::ASCII_8BIT)
616
617
  c.write_to(io, compressed: :gzip)
617
618
  assert_equal @gzipped_src, io.string
618
619
  end
@@ -331,6 +331,7 @@ class BufferMemoryChunkTest < Test::Unit::TestCase
331
331
  assert_equal @gzipped_src, c.read(compressed: :gzip)
332
332
 
333
333
  io = StringIO.new
334
+ io.set_encoding(Encoding::ASCII_8BIT)
334
335
  c.write_to(io, compressed: :gzip)
335
336
  assert_equal @gzipped_src, io.string
336
337
  end
@@ -185,6 +185,81 @@ EOL
185
185
  assert_equal([dummy_cert_path], d.instance.tls_ca_cert_path)
186
186
  end
187
187
 
188
+ sub_test_case "certstore loading parameters for Windows" do
189
+ test 'certstore related config parameters' do
190
+ omit "certstore related values raise error on not Windows" if Fluent.windows?
191
+ conf = %[
192
+ send_timeout 5
193
+ transport tls
194
+ tls_cert_logical_store_name Root
195
+ tls_cert_thumbprint a909502dd82ae41433e6f83886b00d4277a32a7b
196
+ <server>
197
+ host #{TARGET_HOST}
198
+ port #{TARGET_PORT}
199
+ </server>
200
+ ]
201
+
202
+ assert_raise(Fluent::ConfigError) do
203
+ create_driver(conf)
204
+ end
205
+ end
206
+
207
+ test 'cert_logical_store_name and tls_cert_thumbprint default values' do
208
+ conf = %[
209
+ send_timeout 5
210
+ transport tls
211
+ <server>
212
+ host #{TARGET_HOST}
213
+ port #{TARGET_PORT}
214
+ </server>
215
+ ]
216
+
217
+ @d = d = create_driver(conf)
218
+ assert_nil d.instance.tls_cert_logical_store_name
219
+ assert_nil d.instance.tls_cert_thumbprint
220
+ end
221
+
222
+ data('CA cert' => 'tls_ca_cert_path',
223
+ 'non CA cert' => 'tls_cert_path')
224
+ test 'specify tls_cert_logical_store_name and tls_cert_path should raise error' do |param|
225
+ omit "Loading CertStore feature works only Windows" unless Fluent.windows?
226
+ dummy_cert_path = File.join(TMP_DIR, "dummy_cert.pem")
227
+ FileUtils.touch(dummy_cert_path)
228
+ conf = %[
229
+ send_timeout 5
230
+ transport tls
231
+ #{param} #{dummy_cert_path}
232
+ tls_cert_logical_store_name Root
233
+ <server>
234
+ host #{TARGET_HOST}
235
+ port #{TARGET_PORT}
236
+ </server>
237
+ ]
238
+
239
+ assert_raise(Fluent::ConfigError) do
240
+ create_driver(conf)
241
+ end
242
+ end
243
+
244
+ test 'configure cert_logical_store_name and tls_cert_thumbprint' do
245
+ omit "Loading CertStore feature works only Windows" unless Fluent.windows?
246
+ conf = %[
247
+ send_timeout 5
248
+ transport tls
249
+ tls_cert_logical_store_name Root
250
+ tls_cert_thumbprint a909502dd82ae41433e6f83886b00d4277a32a7b
251
+ <server>
252
+ host #{TARGET_HOST}
253
+ port #{TARGET_PORT}
254
+ </server>
255
+ ]
256
+
257
+ @d = d = create_driver(conf)
258
+ assert_equal "Root", d.instance.tls_cert_logical_store_name
259
+ assert_equal "a909502dd82ae41433e6f83886b00d4277a32a7b", d.instance.tls_cert_thumbprint
260
+ end
261
+ end
262
+
188
263
  test 'compress_default_value' do
189
264
  @d = d = create_driver
190
265
  assert_equal :text, d.instance.compress
@@ -14,8 +14,9 @@ class SyslogParserTest < ::Test::Unit::TestCase
14
14
  }
15
15
  end
16
16
 
17
- def test_parse
18
- @parser.configure({})
17
+ data('regexp' => 'regexp', 'string' => 'string')
18
+ def test_parse(param)
19
+ @parser.configure({'parser_type' => param})
19
20
  @parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
20
21
  assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
21
22
  assert_equal(@expected, record)
@@ -24,8 +25,9 @@ class SyslogParserTest < ::Test::Unit::TestCase
24
25
  assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
25
26
  end
26
27
 
27
- def test_parse_with_time_format
28
- @parser.configure('time_format' => '%b %d %M:%S:%H')
28
+ data('regexp' => 'regexp', 'string' => 'string')
29
+ def test_parse_with_time_format(param)
30
+ @parser.configure('time_format' => '%b %d %M:%S:%H', 'parser_type' => param)
29
31
  @parser.instance.parse('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
30
32
  assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
31
33
  assert_equal(@expected, record)
@@ -33,8 +35,18 @@ class SyslogParserTest < ::Test::Unit::TestCase
33
35
  assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
34
36
  end
35
37
 
36
- def test_parse_with_priority
37
- @parser.configure('with_priority' => true)
38
+ data('regexp' => 'regexp', 'string' => 'string')
39
+ def test_parse_with_subsecond_time(param)
40
+ @parser.configure('time_format' => '%b %d %H:%M:%S.%N', 'parser_type' => param)
41
+ @parser.instance.parse('Feb 28 12:00:00.456 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
42
+ assert_equal(event_time('Feb 28 12:00:00.456', format: '%b %d %H:%M:%S.%N'), time)
43
+ assert_equal(@expected, record)
44
+ }
45
+ end
46
+
47
+ data('regexp' => 'regexp', 'string' => 'string')
48
+ def test_parse_with_priority(param)
49
+ @parser.configure('with_priority' => true, 'parser_type' => param)
38
50
  @parser.instance.parse('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
39
51
  assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
40
52
  assert_equal(@expected.merge('pri' => 6), record)
@@ -43,8 +55,18 @@ class SyslogParserTest < ::Test::Unit::TestCase
43
55
  assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
44
56
  end
45
57
 
46
- def test_parse_without_colon
47
- @parser.configure({})
58
+ data('regexp' => 'regexp', 'string' => 'string')
59
+ def test_parse_with_empty_priority(param)
60
+ @parser.configure('with_priority' => true, 'parser_type' => param)
61
+ @parser.instance.parse('<>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
62
+ assert_nil time
63
+ assert_nil record
64
+ }
65
+ end
66
+
67
+ data('regexp' => 'regexp', 'string' => 'string')
68
+ def test_parse_without_colon(param)
69
+ @parser.configure({'parser_type' => param})
48
70
  @parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
49
71
  assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
50
72
  assert_equal(@expected, record)
@@ -53,10 +75,12 @@ class SyslogParserTest < ::Test::Unit::TestCase
53
75
  assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
54
76
  end
55
77
 
56
- def test_parse_with_keep_time_key
78
+ data('regexp' => 'regexp', 'string' => 'string')
79
+ def test_parse_with_keep_time_key(param)
57
80
  @parser.configure(
58
81
  'time_format' => '%b %d %M:%S:%H',
59
82
  'keep_time_key'=>'true',
83
+ 'parser_type' => param
60
84
  )
61
85
  text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
62
86
  @parser.instance.parse(text) do |time, record|
@@ -64,24 +88,87 @@ class SyslogParserTest < ::Test::Unit::TestCase
64
88
  end
65
89
  end
66
90
 
67
- def test_parse_various_characters_for_tag
91
+ data('regexp' => 'regexp', 'string' => 'string')
92
+ def test_parse_various_characters_for_tag(param)
68
93
  ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
69
- @parser.configure({})
94
+ @parser.configure({'parser_type' => param})
70
95
  @parser.instance.parse("Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
71
96
  assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
72
97
  assert_equal(@expected.merge('ident' => ident), record)
73
98
  }
74
99
  end
75
100
 
76
- def test_parse_various_characters_for_tag_with_priority
101
+ data('regexp' => 'regexp', 'string' => 'string')
102
+ def test_parse_various_characters_for_tag_with_priority(param)
77
103
  ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
78
- @parser.configure('with_priority' => true)
104
+ @parser.configure('with_priority' => true, 'parser_type' => param)
79
105
  @parser.instance.parse("<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
80
106
  assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
81
107
  assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)
82
108
  }
83
109
  end
84
110
 
111
+ sub_test_case 'Check the difference of regexp and string parser' do
112
+ # examples from rfc3164
113
+ data('regexp' => 'regexp', 'string' => 'string')
114
+ test 'wrong result with no ident message by default' do |param|
115
+ @parser.configure('parser_type' => param)
116
+ @parser.instance.parse('Feb 5 17:32:18 10.0.0.99 Use the BFG!') { |time, record|
117
+ assert_equal({'host' => '10.0.0.99', 'ident' => 'Use', 'message' => 'the BFG!'}, record)
118
+ }
119
+ end
120
+
121
+ test "proper result with no ident message by 'support_colonless_ident false'" do
122
+ @parser.configure('parser_type' => 'string', 'support_colonless_ident' => false)
123
+ @parser.instance.parse('Feb 5 17:32:18 10.0.0.99 Use the BFG!') { |time, record|
124
+ assert_equal({'host' => '10.0.0.99', 'message' => 'Use the BFG!'}, record)
125
+ }
126
+ end
127
+
128
+ test "string parsers can't parse broken syslog message and generate wrong record" do
129
+ @parser.configure('parser_type' => 'string')
130
+ @parser.instance.parse("1990 Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.32 sched[0]: That's All Folks!") { |time, record|
131
+ expected = {'host' => 'scapegoat.dmz.example.org', 'ident' => 'sched', 'pid' => '0', 'message' => "That's All Folks!"}
132
+ assert_not_equal(expected, record)
133
+ }
134
+ end
135
+
136
+ test "regexp parsers can't parse broken syslog message and raises an error" do
137
+ @parser.configure('parser_type' => 'regexp')
138
+ assert_raise(Fluent::TimeParser::TimeParseError) {
139
+ @parser.instance.parse("1990 Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.32 sched[0]: That's All Folks!") { |time, record| }
140
+ }
141
+ end
142
+
143
+ data('regexp' => 'regexp', 'string' => 'string')
144
+ test "':' included message breaks regexp parser" do |param|
145
+ @parser.configure('parser_type' => param)
146
+ @parser.instance.parse('Aug 10 12:00:00 127.0.0.1 test foo:bar') { |time, record|
147
+ expected = {'host' => '127.0.0.1', 'ident' => 'test', 'message' => 'foo:bar'}
148
+ if param == 'string'
149
+ assert_equal(expected, record)
150
+ else
151
+ assert_not_equal(expected, record)
152
+ end
153
+ }
154
+ end
155
+
156
+ data('regexp' => 'regexp', 'string' => 'string')
157
+ test "Only no whitespace content in MSG causes different result" do |param|
158
+ @parser.configure('parser_type' => param)
159
+ @parser.instance.parse('Aug 10 12:00:00 127.0.0.1 value1,value2,value3,value4') { |time, record|
160
+ # 'message' is correct but regexp set it as 'ident'
161
+ if param == 'string'
162
+ expected = {'host' => '127.0.0.1', 'message' => 'value1,value2,value3,value4'}
163
+ assert_equal(expected, record)
164
+ else
165
+ expected = {'host' => '127.0.0.1', 'ident' => 'value1,value2,value3,value4', 'message' => ''}
166
+ assert_equal(expected, record)
167
+ end
168
+ }
169
+ end
170
+ end
171
+
85
172
  class TestRFC5424Regexp < self
86
173
  def test_parse_with_rfc5424_message
87
174
  @parser.configure(
@@ -273,10 +360,12 @@ class SyslogParserTest < ::Test::Unit::TestCase
273
360
  end
274
361
 
275
362
  class TestAutoRegexp < self
276
- def test_auto_with_legacy_syslog_message
363
+ data('regexp' => 'regexp', 'string' => 'string')
364
+ def test_auto_with_legacy_syslog_message(param)
277
365
  @parser.configure(
278
366
  'time_format' => '%b %d %M:%S:%H',
279
367
  'message_format' => 'auto',
368
+ 'parser_type' => param
280
369
  )
281
370
  text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
282
371
  @parser.instance.parse(text) do |time, record|
@@ -286,11 +375,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
286
375
  assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
287
376
  end
288
377
 
289
- def test_auto_with_legacy_syslog_priority_message
378
+ data('regexp' => 'regexp', 'string' => 'string')
379
+ def test_auto_with_legacy_syslog_priority_message(param)
290
380
  @parser.configure(
291
381
  'time_format' => '%b %d %M:%S:%H',
292
382
  'with_priority' => true,
293
383
  'message_format' => 'auto',
384
+ 'parser_type' => param
294
385
  )
295
386
  text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
296
387
  @parser.instance.parse(text) do |time, record|
@@ -300,11 +391,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
300
391
  assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
301
392
  end
302
393
 
303
- def test_parse_with_rfc5424_message
394
+ data('regexp' => 'regexp', 'string' => 'string')
395
+ def test_parse_with_rfc5424_message(param)
304
396
  @parser.configure(
305
397
  'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
306
398
  'message_format' => 'auto',
307
399
  'with_priority' => true,
400
+ 'parser_type' => param
308
401
  )
309
402
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
310
403
  @parser.instance.parse(text) do |time, record|
@@ -318,11 +411,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
318
411
  @parser.instance.patterns['format'])
319
412
  end
320
413
 
321
- def test_parse_with_rfc5424_structured_message
414
+ data('regexp' => 'regexp', 'string' => 'string')
415
+ def test_parse_with_rfc5424_structured_message(param)
322
416
  @parser.configure(
323
417
  'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
324
418
  'message_format' => 'auto',
325
419
  'with_priority' => true,
420
+ 'parser_type' => param
326
421
  )
327
422
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
328
423
  @parser.instance.parse(text) do |time, record|
@@ -337,12 +432,14 @@ class SyslogParserTest < ::Test::Unit::TestCase
337
432
  @parser.instance.patterns['format'])
338
433
  end
339
434
 
340
- def test_parse_with_both_message_type
435
+ data('regexp' => 'regexp', 'string' => 'string')
436
+ def test_parse_with_both_message_type(param)
341
437
  @parser.configure(
342
438
  'time_format' => '%b %d %M:%S:%H',
343
439
  'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
344
440
  'message_format' => 'auto',
345
441
  'with_priority' => true,
442
+ 'parser_type' => param
346
443
  )
347
444
  text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
348
445
  @parser.instance.parse(text) do |time, record|
@@ -382,12 +479,14 @@ class SyslogParserTest < ::Test::Unit::TestCase
382
479
  @parser.instance.patterns['format'])
383
480
  end
384
481
 
385
- def test_parse_with_both_message_type_and_priority
482
+ data('regexp' => 'regexp', 'string' => 'string')
483
+ def test_parse_with_both_message_type_and_priority(param)
386
484
  @parser.configure(
387
485
  'time_format' => '%b %d %M:%S:%H',
388
486
  'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
389
487
  'with_priority' => true,
390
488
  'message_format' => 'auto',
489
+ 'parser_type' => param
391
490
  )
392
491
  text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
393
492
  @parser.instance.parse(text) do |time, record|
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.7.0
4
+ version: 1.7.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-08-21 00:00:00.000000000 Z
11
+ date: 2019-09-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack
@@ -16,7 +16,7 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.7.0
19
+ version: 1.2.0
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
22
  version: 2.0.0
@@ -26,7 +26,7 @@ dependencies:
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: 0.7.0
29
+ version: 1.2.0
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: 2.0.0