fluentd 1.7.0 → 1.7.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: aa26c04f10203245130ee2457e56f1ea064df188d91c3610aa338f5f66273663
4
- data.tar.gz: 48031fcf5dc26c8d1e3a6f6fc4a5f629451c6dc9a06c4a95043c3ba56da031b1
3
+ metadata.gz: 1ce449e77af62c25fcef41773c3717738ec443550becdde0ca666710739f7eae
4
+ data.tar.gz: 00d125b1c53ac99c8d2edf8b915909b91e07642e7fdad35154147a3b5a472416
5
5
  SHA512:
6
- metadata.gz: ac003c0a88bfd623a9e1caf2989db9e45b38841827dc312e4e266ff38cc1ac25001a58fda0fc5b7eebb9b4efa4bfc9cbf0266ad36314aa5ed518303040e09727
7
- data.tar.gz: b84fb3fe1b06ee99fcdad4cbc37d897b507b5b916d5ad8b539863827e6d1875ca6c026ae54b6b05898750e84357db51583e59872ee76bc814e3bc7e29e2e3fc2
6
+ metadata.gz: d78a18ab98f55b3568b0e401f87d80444d14b09b544fc4962a14bf821180c5f97a155332888961ce096adfa9ae6d01def4a154dc4a769939b6256846cf0b14b3
7
+ data.tar.gz: cc36a5137a9753d9888f6669d04dbfe943d35d3cb030f5718fa439f957e8658632e96246b219151b73f46a2ac74191a0f6e87f049aae1fe446b2e62fc3c4a274
data/CHANGELOG.md CHANGED
@@ -1,11 +1,53 @@
1
1
  # v1.7
2
2
 
3
+ ## Release v1.7.2 - 2019/09/19
4
+
5
+ ### Enhancement
6
+
7
+ * in_tcp: Add security/client to restrict access
8
+ https://github.com/fluent/fluentd/pull/2622
9
+
10
+ ### Bug fixes
11
+
12
+ * buf_file/buf_file_single: fix to handle compress data during restart
13
+ https://github.com/fluent/fluentd/pull/2620
14
+ * plugin: Use `__send__` to avoid conflict with user defined `send`
15
+ https://github.com/fluent/fluentd/pull/2614
16
+ * buffer: reject invalid timekey at configure phase
17
+ https://github.com/fluent/fluentd/pull/2615
18
+
19
+
20
+ ## Release v1.7.1 - 2019/09/08
21
+
22
+ ### Enhancement
23
+
24
+ * socket helper/out_forward: Support Windows certstore to load certificates
25
+ https://github.com/fluent/fluentd/pull/2601
26
+ * parser_syslog: Add faster parser for rfc3164 message
27
+ https://github.com/fluent/fluentd/pull/2599
28
+
29
+ ### Bug fixes
30
+
31
+ * buf_file/buf_file_single: fix to ignore placeholder based path.
32
+ https://github.com/fluent/fluentd/pull/2594
33
+ * server helper: Ignore ETIMEDOUT error in SSL_accept
34
+ https://github.com/fluent/fluentd/pull/2595
35
+ * buf_file: ensure to remove metadata after buffer creation failure
36
+ https://github.com/fluent/fluentd/pull/2598
37
+ * buf_file_single: fix duplicated path setting check
38
+ https://github.com/fluent/fluentd/pull/2600
39
+ * fix msgpack-ruby depedency to use recent feature
40
+ https://github.com/fluent/fluentd/pull/2606
41
+
42
+
3
43
  ## Release v1.7.0 - 2019/08/20
4
44
 
5
45
  ### New feature
6
46
 
7
- * buffer: Add file_single buffer
8
- https://github.com/fluent/fluentd/pull/2479
47
+ * buffer: Add file_single buffer plugin
48
+ https://github.com/fluent/fluentd/pull/2579
49
+ * output: Add http output plugin
50
+ https://github.com/fluent/fluentd/pull/2488
9
51
 
10
52
  ### Enhancement
11
53
 
@@ -13,9 +55,9 @@
13
55
  https://github.com/fluent/fluentd/pull/2560
14
56
  https://github.com/fluent/fluentd/pull/2563
15
57
  https://github.com/fluent/fluentd/pull/2564
16
- * output: Use Mutext instead of Monitor
58
+ * output: Use Mutex instead of Monitor
17
59
  https://github.com/fluent/fluentd/pull/2561
18
- * event: Add `OneEvent#empty?` method
60
+ * event: Add `OneEventStrea#empty?` method
19
61
  https://github.com/fluent/fluentd/pull/2565
20
62
  * thread: Set thread name for ruby 2.3 or later
21
63
  https://github.com/fluent/fluentd/pull/2574
data/fluentd.gemspec CHANGED
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
18
18
 
19
19
  gem.required_ruby_version = '>= 2.1'
20
20
 
21
- gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
21
+ gem.add_runtime_dependency("msgpack", [">= 1.2.0", "< 2.0.0"])
22
22
  gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
23
23
  gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
24
24
  gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
@@ -37,6 +37,7 @@ Gem::Specification.new do |gem|
37
37
  gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
38
38
  gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
39
39
  gem.add_runtime_dependency("windows-pr", ["~> 1.2.5"])
40
+ gem.add_runtime_dependency("certstore_c", ["~> 0.1.2"])
40
41
  end
41
42
 
42
43
  gem.add_development_dependency("rake", ["~> 11.0"])
@@ -132,7 +132,7 @@ module Fluent
132
132
 
133
133
  patterns = [@path]
134
134
  patterns.unshift @additional_resume_path if @additional_resume_path
135
- Dir.glob(patterns) do |path|
135
+ Dir.glob(escaped_patterns(patterns)) do |path|
136
136
  next unless File.file?(path)
137
137
 
138
138
  log.debug { "restoring buffer file: path = #{path}" }
@@ -146,7 +146,7 @@ module Fluent
146
146
  end
147
147
 
148
148
  begin
149
- chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode) # file chunk resumes contents of metadata
149
+ chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
150
150
  rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
151
151
  handle_broken_files(path, mode, e)
152
152
  next
@@ -183,6 +183,15 @@ module Fluent
183
183
  # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
184
184
  File.unlink(path, path + '.meta') rescue nil
185
185
  end
186
+
187
+ private
188
+
189
+ def escaped_patterns(patterns)
190
+ patterns.map { |pattern|
191
+ # '{' '}' are special character in Dir.glob
192
+ pattern.gsub(/[\{\}]/) { |c| "\\#{c}" }
193
+ }
194
+ end
186
195
  end
187
196
  end
188
197
  end
@@ -88,14 +88,6 @@ module Fluent
88
88
  end
89
89
  end
90
90
 
91
- type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
92
- if @@buffer_paths.has_key?(@path) && !called_in_test?
93
- type_using_this_path = @@buffer_paths[@path]
94
- raise Fluent::ConfigError, "Other '#{type_using_this_path}' plugin already uses same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
95
- end
96
-
97
- @@buffer_paths[@path] = type_of_owner
98
-
99
91
  specified_directory_exists = File.exist?(@path) && File.directory?(@path)
100
92
  unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')
101
93
 
@@ -124,6 +116,13 @@ module Fluent
124
116
  @multi_workers_available = false
125
117
  end
126
118
 
119
+ type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
120
+ if @@buffer_paths.has_key?(@path) && !called_in_test?
121
+ type_using_this_path = @@buffer_paths[@path]
122
+ raise Fluent::ConfigError, "Other '#{type_using_this_path}' plugin already uses same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
123
+ end
124
+
125
+ @@buffer_paths[@path] = type_of_owner
127
126
  @dir_permission = if @dir_permission
128
127
  @dir_permission.to_i(8)
129
128
  else
@@ -155,7 +154,7 @@ module Fluent
155
154
 
156
155
  patterns = [@path]
157
156
  patterns.unshift @additional_resume_path if @additional_resume_path
158
- Dir.glob(patterns) do |path|
157
+ Dir.glob(escaped_patterns(patterns)) do |path|
159
158
  next unless File.file?(path)
160
159
 
161
160
  log.debug { "restoring buffer file: path = #{path}" }
@@ -168,7 +167,7 @@ module Fluent
168
167
  end
169
168
 
170
169
  begin
171
- chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path)
170
+ chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path, compress: @compress)
172
171
  chunk.restore_size(@chunk_format) if @calc_num_records
173
172
  rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e
174
173
  handle_broken_files(path, mode, e)
@@ -206,6 +205,15 @@ module Fluent
206
205
  # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
207
206
  File.unlink(path) rescue nil
208
207
  end
208
+
209
+ private
210
+
211
+ def escaped_patterns(patterns)
212
+ patterns.map { |pattern|
213
+ # '{' '}' are special character in Dir.glob
214
+ pattern.gsub(/[\{\}]/) { |c| "\\#{c}" }
215
+ }
216
+ end
209
217
  end
210
218
  end
211
219
  end
@@ -300,6 +300,13 @@ module Fluent
300
300
  # This case is easier than enqueued!. Just removing pre-create buffer file
301
301
  @chunk.close rescue nil
302
302
  File.unlink(@path) rescue nil
303
+
304
+ if @meta
305
+ # ensure to unlink when #write_metadata fails
306
+ @meta.close rescue nil
307
+ File.unlink(@meta_path) rescue nil
308
+ end
309
+
303
310
  # Same as @chunk case. See above
304
311
  raise BufferOverflowError, "can't create buffer metadata for #{path}. Stop creating buffer files: error = #{e}"
305
312
  end
@@ -41,6 +41,16 @@ module Fluent::Plugin
41
41
  desc 'The payload is read up to this character.'
42
42
  config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
43
43
 
44
+ # in_forward like host/network restriction
45
+ config_section :security, required: false, multi: false do
46
+ config_section :client, param_name: :clients, required: true, multi: true do
47
+ desc 'The IP address or host name of the client'
48
+ config_param :host, :string, default: nil
49
+ desc 'Network address specification'
50
+ config_param :network, :string, default: nil
51
+ end
52
+ end
53
+
44
54
  def configure(conf)
45
55
  compat_parameters_convert(conf, :parser)
46
56
  parser_config = conf.elements('parse').first
@@ -51,6 +61,33 @@ module Fluent::Plugin
51
61
  @_event_loop_blocking_timeout = @blocking_timeout
52
62
  @source_hostname_key ||= @source_host_key if @source_host_key
53
63
 
64
+ @nodes = nil
65
+ if @security
66
+ @nodes = []
67
+ @security.clients.each do |client|
68
+ if client.host && client.network
69
+ raise Fluent::ConfigError, "both of 'host' and 'network' are specified for client"
70
+ end
71
+ if !client.host && !client.network
72
+ raise Fluent::ConfigError, "Either of 'host' and 'network' must be specified for client"
73
+ end
74
+ source = nil
75
+ if client.host
76
+ begin
77
+ source = IPSocket.getaddress(client.host)
78
+ rescue SocketError
79
+ raise Fluent::ConfigError, "host '#{client.host}' cannot be resolved"
80
+ end
81
+ end
82
+ source_addr = begin
83
+ IPAddr.new(source || client.network)
84
+ rescue ArgumentError
85
+ raise Fluent::ConfigError, "network '#{client.network}' address format is invalid"
86
+ end
87
+ @nodes.push(source_addr)
88
+ end
89
+ end
90
+
54
91
  @parser = parser_create(conf: parser_config)
55
92
  end
56
93
 
@@ -64,6 +101,11 @@ module Fluent::Plugin
64
101
  del_size = @delimiter.length
65
102
  if @_extract_enabled && @_extract_tag_key
66
103
  server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
104
+ unless check_client(conn)
105
+ conn.close
106
+ next
107
+ end
108
+
67
109
  conn.buffer << data
68
110
  buf = conn.buffer
69
111
  pos = 0
@@ -89,6 +131,11 @@ module Fluent::Plugin
89
131
  end
90
132
  else
91
133
  server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
134
+ unless check_client(conn)
135
+ conn.close
136
+ next
137
+ end
138
+
92
139
  conn.buffer << data
93
140
  buf = conn.buffer
94
141
  pos = 0
@@ -114,5 +161,20 @@ module Fluent::Plugin
114
161
  end
115
162
  end
116
163
  end
164
+
165
+ private
166
+
167
+ def check_client(conn)
168
+ if @nodes
169
+ remote_addr = conn.remote_addr
170
+ node = @nodes.find { |n| n.include?(remote_addr) rescue false }
171
+ unless node
172
+ log.warn "anonymous client '#{remote_addr}' denied"
173
+ return false
174
+ end
175
+ end
176
+
177
+ true
178
+ end
117
179
  end
118
180
  end
@@ -94,7 +94,7 @@ module Fluent
94
94
  @outputs.each do |o|
95
95
  begin
96
96
  log.debug "calling #{method_name} on output plugin dynamically created", type: Fluent::Plugin.lookup_type_from_class(o.class), plugin_id: o.plugin_id
97
- o.send(method_name) unless o.send(checker_name)
97
+ o.__send__(method_name) unless o.__send__(checker_name)
98
98
  rescue Exception => e
99
99
  log.warn "unexpected error while calling #{method_name} on output plugin dynamically created", plugin: o.class, plugin_id: o.plugin_id, error: e
100
100
  log.warn_backtrace
@@ -107,6 +107,12 @@ module Fluent::Plugin
107
107
  config_param :tls_client_private_key_path, :string, default: nil
108
108
  desc 'The client private key passphrase for TLS.'
109
109
  config_param :tls_client_private_key_passphrase, :string, default: nil, secret: true
110
+ desc 'The certificate thumbprint for searching from Windows system certstore.'
111
+ config_param :tls_cert_thumbprint, :string, default: nil, secret: true
112
+ desc 'The certificate logical store name on Windows system certstore.'
113
+ config_param :tls_cert_logical_store_name, :string, default: nil
114
+ desc 'Enable to use certificate enterprise store on Windows system certstore.'
115
+ config_param :tls_cert_use_enterprise_store, :bool, default: true
110
116
  desc "Enable keepalive connection."
111
117
  config_param :keepalive, :bool, default: false
112
118
  desc "Expired time of keepalive. Default value is nil, which means to keep connection as long as possible"
@@ -198,6 +204,15 @@ module Fluent::Plugin
198
204
  @tls_verify_hostname = false
199
205
  @tls_allow_self_signed_cert = true
200
206
  end
207
+
208
+ if Fluent.windows?
209
+ if (@tls_cert_path || @tls_ca_cert_path) && @tls_cert_logical_store_name
210
+ raise Fluent::ConfigError, "specified both cert path and tls_cert_logical_store_name is not permitted"
211
+ end
212
+ else
213
+ raise Fluent::ConfigError, "This parameter is for only Windows" if @tls_cert_logical_store_name
214
+ raise Fluent::ConfigError, "This parameter is for only Windows" if @tls_cert_thumbprint
215
+ end
201
216
  end
202
217
 
203
218
  @ack_handler = @require_ack_response ? AckHandler.new(timeout: @ack_response_timeout, log: @log, read_length: @read_length) : nil
@@ -346,6 +361,9 @@ module Fluent::Plugin
346
361
  cert_path: @tls_client_cert_path,
347
362
  private_key_path: @tls_client_private_key_path,
348
363
  private_key_passphrase: @tls_client_private_key_passphrase,
364
+ cert_thumbprint: @tls_cert_thumbprint,
365
+ cert_logical_store_name: @tls_cert_logical_store_name,
366
+ cert_use_enterprise_store: @tls_cert_use_enterprise_store,
349
367
 
350
368
  # Enabling SO_LINGER causes data loss on Windows
351
369
  # https://github.com/fluent/fluentd/issues/1968
@@ -310,6 +310,9 @@ module Fluent
310
310
  Fluent::Timezone.validate!(@buffer_config.timekey_zone)
311
311
  @timekey_zone = @buffer_config.timekey_use_utc ? '+0000' : @buffer_config.timekey_zone
312
312
  @timekey = @buffer_config.timekey
313
+ if @timekey <= 0
314
+ raise Fluent::ConfigError, "timekey should be greater than 0. current timekey: #{@timekey}"
315
+ end
313
316
  @timekey_use_utc = @buffer_config.timekey_use_utc
314
317
  @offset = Fluent::Timezone.utc_offset(@timekey_zone)
315
318
  @calculate_offset = @offset.respond_to?(:call) ? @offset : nil
@@ -38,6 +38,10 @@ module Fluent
38
38
  config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
39
39
  desc 'Specify time format for event time for rfc5424 protocol'
40
40
  config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
41
+ desc 'The parser type used to parse syslog message'
42
+ config_param :parser_type, :enum, list: [:regexp, :string], default: :regexp
43
+ desc 'support colonless ident in string parser'
44
+ config_param :support_colonless_ident, :bool, default: true
41
45
 
42
46
  def initialize
43
47
  super
@@ -50,10 +54,17 @@ module Fluent
50
54
  @time_parser_rfc3164 = @time_parser_rfc5424 = nil
51
55
  @time_parser_rfc5424_without_subseconds = nil
52
56
  @support_rfc5424_without_subseconds = false
57
+ @regexp_parser = @parser_type == :regexp
53
58
  @regexp = case @message_format
54
59
  when :rfc3164
55
- class << self
56
- alias_method :parse, :parse_plain
60
+ if @regexp_parser
61
+ class << self
62
+ alias_method :parse, :parse_plain
63
+ end
64
+ else
65
+ class << self
66
+ alias_method :parse, :parse_rfc3164
67
+ end
57
68
  end
58
69
  @with_priority ? REGEXP_WITH_PRI : REGEXP
59
70
  when :rfc5424
@@ -88,11 +99,16 @@ module Fluent
88
99
  @regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
89
100
  @time_parser = @time_parser_rfc5424
90
101
  @support_rfc5424_without_subseconds = true
102
+ parse_plain(text, &block)
91
103
  else
92
104
  @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
93
105
  @time_parser = @time_parser_rfc3164
106
+ if @regexp_parser
107
+ parse_plain(text, &block)
108
+ else
109
+ parse_rfc3164(text, &block)
110
+ end
94
111
  end
95
- parse_plain(text, &block)
96
112
  end
97
113
 
98
114
  def parse_plain(text, &block)
@@ -137,6 +153,93 @@ module Fluent
137
153
 
138
154
  yield time, record
139
155
  end
156
+
157
+ SPLIT_CHAR = ' '.freeze
158
+
159
+ def parse_rfc3164(text, &block)
160
+ pri = nil
161
+ cursor = 0
162
+ if @with_priority
163
+ if text.start_with?('<'.freeze)
164
+ i = text.index('>'.freeze, 1)
165
+ if i < 2
166
+ yield nil, nil
167
+ return
168
+ end
169
+ pri = text.slice(1, i - 1).to_i
170
+ cursor = i + 1
171
+ else
172
+ yield nil, nil
173
+ return
174
+ end
175
+ end
176
+
177
+ # header part
178
+ time_size = 15 # skip Mmm dd hh:mm:ss
179
+ time_end = text[cursor + time_size]
180
+ if time_end == SPLIT_CHAR
181
+ time_str = text.slice(cursor, time_size)
182
+ cursor += 16 # time + ' '
183
+ elsif time_end == '.'.freeze
184
+ # support subsecond time
185
+ i = text.index(SPLIT_CHAR, time_size)
186
+ time_str = text.slice(cursor, i - cursor)
187
+ cursor = i + 1
188
+ else
189
+ yield nil, nil
190
+ return
191
+ end
192
+
193
+ i = text.index(SPLIT_CHAR, cursor)
194
+ if i.nil?
195
+ yield nil, nil
196
+ return
197
+ end
198
+ host_size = i - cursor
199
+ host = text.slice(cursor, host_size)
200
+ cursor += host_size + 1
201
+
202
+ record = {'host' => host}
203
+ record['pri'] = pri if pri
204
+
205
+ i = text.index(SPLIT_CHAR, cursor)
206
+
207
+ # message part
208
+ msg = if i.nil? # for 'only non-space content case'
209
+ text.slice(cursor, text.bytesize)
210
+ else
211
+ if text[i - 1] == ':'.freeze
212
+ if text[i - 2] == ']'.freeze
213
+ left_braket_pos = text.index('['.freeze, cursor)
214
+ record['ident'] = text.slice(cursor, left_braket_pos - cursor)
215
+ record['pid'] = text.slice(left_braket_pos + 1, i - left_braket_pos - 3) # remove '[' / ']:'
216
+ else
217
+ record['ident'] = text.slice(cursor, i - cursor - 1)
218
+ end
219
+ text.slice(i + 1, text.bytesize)
220
+ else
221
+ if @support_colonless_ident
222
+ if text[i - 1] == ']'.freeze
223
+ left_braket_pos = text.index('['.freeze, cursor)
224
+ record['ident'] = text.slice(cursor, left_braket_pos - cursor)
225
+ record['pid'] = text.slice(left_braket_pos + 1, i - left_braket_pos - 2) # remove '[' / ']'
226
+ else
227
+ record['ident'] = text.slice(cursor, i - cursor)
228
+ end
229
+ text.slice(i + 1, text.bytesize)
230
+ else
231
+ text.slice(cursor, text.bytesize)
232
+ end
233
+ end
234
+ end
235
+ msg.chomp!
236
+ record['message'] = msg
237
+
238
+ time = @time_parser.parse(time_str)
239
+ record['time'] = time_str if @keep_time_key
240
+
241
+ yield time, record
242
+ end
140
243
  end
141
244
  end
142
245
  end
@@ -103,7 +103,7 @@ module Fluent
103
103
  def formatter_operate(method_name, &block)
104
104
  @_formatters.each_pair do |usage, formatter|
105
105
  begin
106
- formatter.send(method_name)
106
+ formatter.__send__(method_name)
107
107
  block.call(formatter) if block_given?
108
108
  rescue => e
109
109
  log.error "unexpected error while #{method_name}", usage: usage, formatter: formatter, error: e
@@ -103,7 +103,7 @@ module Fluent
103
103
  def parser_operate(method_name, &block)
104
104
  @_parsers.each_pair do |usage, parser|
105
105
  begin
106
- parser.send(method_name)
106
+ parser.__send__(method_name)
107
107
  block.call(parser) if block_given?
108
108
  rescue => e
109
109
  log.error "unexpected error while #{method_name}", usage: usage, parser: parser, error: e
@@ -722,7 +722,7 @@ module Fluent
722
722
 
723
723
  return true
724
724
  end
725
- rescue Errno::EPIPE, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
725
+ rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, OpenSSL::SSL::SSLError => e
726
726
  @log.trace "unexpected error before accepting TLS connection", error: e
727
727
  close rescue nil
728
728
  end
@@ -17,6 +17,9 @@
17
17
  require 'socket'
18
18
  require 'ipaddr'
19
19
  require 'openssl'
20
+ if Fluent.windows?
21
+ require 'certstore'
22
+ end
20
23
 
21
24
  require_relative 'socket_option'
22
25
 
@@ -96,7 +99,9 @@ module Fluent
96
99
  host, port,
97
100
  version: TLS_DEFAULT_VERSION, ciphers: CIPHERS_DEFAULT, insecure: false, verify_fqdn: true, fqdn: nil,
98
101
  enable_system_cert_store: true, allow_self_signed_cert: false, cert_paths: nil,
99
- cert_path: nil, private_key_path: nil, private_key_passphrase: nil, **kwargs, &block)
102
+ cert_path: nil, private_key_path: nil, private_key_passphrase: nil,
103
+ cert_thumbprint: nil, cert_logical_store_name: nil, cert_use_enterprise_store: true,
104
+ **kwargs, &block)
100
105
 
101
106
  host_is_ipaddress = IPAddr.new(host) rescue false
102
107
  fqdn ||= host unless host_is_ipaddress
@@ -113,6 +118,14 @@ module Fluent
113
118
  end
114
119
  begin
115
120
  if enable_system_cert_store
121
+ if Fluent.windows? && cert_logical_store_name
122
+ log.trace "loading Windows system certificate store"
123
+ loader = Certstore::OpenSSL::Loader.new(log, cert_store, cert_logical_store_name,
124
+ enterprise: cert_use_enterprise_store)
125
+ loader.load_cert_store
126
+ cert_store = loader.cert_store
127
+ context.cert = loader.get_certificate(cert_thumbprint) if cert_thumbprint
128
+ end
116
129
  log.trace "loading system default certificate store"
117
130
  cert_store.set_default_paths
118
131
  end
@@ -138,7 +138,7 @@ module Fluent
138
138
  @_storages.each_pair do |usage, s|
139
139
  begin
140
140
  block.call(s) if block_given?
141
- s.storage.send(method_name)
141
+ s.storage.__send__(method_name)
142
142
  rescue => e
143
143
  log.error "unexpected error while #{method_name}", usage: usage, storage: s.storage, error: e
144
144
  end
@@ -240,7 +240,7 @@ module Fluent
240
240
  lifecycle do |instance, kind|
241
241
  begin
242
242
  log.debug "calling #{method} on #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
243
- instance.send(method) unless instance.send(checker)
243
+ instance.__send__(method) unless instance.__send__(checker)
244
244
  rescue Exception => e
245
245
  log.warn "unexpected error while calling #{method} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
246
246
  log.warn_backtrace
@@ -270,17 +270,17 @@ module Fluent
270
270
  operation = "preparing shutdown" # for logging
271
271
  log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
272
272
  begin
273
- instance.send(:before_shutdown) unless instance.send(:before_shutdown?)
273
+ instance.__send__(:before_shutdown) unless instance.__send__(:before_shutdown?)
274
274
  rescue Exception => e
275
275
  log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
276
276
  log.warn_backtrace
277
277
  end
278
278
  operation = "shutting down"
279
279
  log.info "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
280
- instance.send(:shutdown) unless instance.send(:shutdown?)
280
+ instance.__send__(:shutdown) unless instance.__send__(:shutdown?)
281
281
  else
282
282
  log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
283
- instance.send(method) unless instance.send(checker)
283
+ instance.__send__(method) unless instance.__send__(checker)
284
284
  end
285
285
  rescue Exception => e
286
286
  log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
@@ -136,7 +136,7 @@ module Fluent
136
136
  supervisor_value = instance_variable_get("@#{param}")
137
137
  next if supervisor_value.nil? # it's not configured by command line options
138
138
 
139
- system.send("#{param}=", supervisor_value)
139
+ system.__send__("#{param}=", supervisor_value)
140
140
  end
141
141
  end
142
142
  }
@@ -179,7 +179,7 @@ module Fluent
179
179
  @_system_config = (defined?($_system_config) && $_system_config ? $_system_config : Fluent::Engine.system_config).dup
180
180
  end
181
181
  opts.each_pair do |key, value|
182
- @_system_config.send(:"#{key.to_s}=", value)
182
+ @_system_config.__send__(:"#{key.to_s}=", value)
183
183
  end
184
184
  end
185
185
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.7.0'
19
+ VERSION = '1.7.2'
20
20
 
21
21
  end
@@ -537,6 +537,60 @@ class FileBufferTest < Test::Unit::TestCase
537
537
  end
538
538
  end
539
539
 
540
+ sub_test_case 'there are some existing file chunks with placeholders path' do
541
+ setup do
542
+ @bufdir = File.expand_path('../../tmp/buffer_${test}_file', __FILE__)
543
+ FileUtils.rm_rf(@bufdir)
544
+ FileUtils.mkdir_p(@bufdir)
545
+
546
+ @c1id = Fluent::UniqueId.generate
547
+ p1 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c1id)}.log")
548
+ File.open(p1, 'wb') do |f|
549
+ f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
550
+ end
551
+ write_metadata(
552
+ p1 + '.meta', @c1id, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),
553
+ 1, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i
554
+ )
555
+
556
+ @c2id = Fluent::UniqueId.generate
557
+ p2 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c2id)}.log")
558
+ File.open(p2, 'wb') do |f|
559
+ f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
560
+ end
561
+ write_metadata(
562
+ p2 + '.meta', @c2id, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),
563
+ 1, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i
564
+ )
565
+
566
+ @bufpath = File.join(@bufdir, 'etest.*.log')
567
+
568
+ Fluent::Test.setup
569
+ @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
570
+ @p = Fluent::Plugin::FileBuffer.new
571
+ @p.owner = @d
572
+ @p.configure(config_element('buffer', '', {'path' => @bufpath}))
573
+ @p.start
574
+ end
575
+
576
+ teardown do
577
+ if @p
578
+ @p.stop unless @p.stopped?
579
+ @p.before_shutdown unless @p.before_shutdown?
580
+ @p.shutdown unless @p.shutdown?
581
+ @p.after_shutdown unless @p.after_shutdown?
582
+ @p.close unless @p.closed?
583
+ @p.terminate unless @p.terminated?
584
+ end
585
+ FileUtils.rm_rf(@bufdir)
586
+ end
587
+
588
+ test '#resume returns staged/queued chunks with metadata' do
589
+ assert_equal 1, @p.stage.size
590
+ assert_equal 1, @p.queue.size
591
+ end
592
+ end
593
+
540
594
  sub_test_case 'there are some existing file chunks, both in specified path and per-worker directory under specified path, configured as multi workers' do
541
595
  setup do
542
596
  @bufdir = File.expand_path('../../tmp/buffer_file/path', __FILE__)
@@ -138,6 +138,25 @@ class FileSingleBufferTest < Test::Unit::TestCase
138
138
  end
139
139
  end
140
140
 
141
+ test 'raise config error when using same file path' do
142
+ d = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
143
+ d2 = FluentPluginFileSingleBufferTest::DummyOutputPlugin.new
144
+ Fluent::SystemConfig.overwrite_system_config({}) do
145
+ d.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => File.join(PATH, 'foo.*.bar') })]))
146
+ end
147
+
148
+ any_instance_of(Fluent::Plugin::FileSingleBuffer) do |klass|
149
+ stub(klass).called_in_test? { false }
150
+ end
151
+
152
+ err = assert_raise(Fluent::ConfigError) do
153
+ Fluent::SystemConfig.overwrite_system_config({}) do
154
+ d2.configure(config_element('ROOT', '', {}, [config_element('buffer', '', { 'path' => PATH })]))
155
+ end
156
+ end
157
+ assert_match(/plugin already uses same buffer path/, err.message)
158
+ end
159
+
141
160
  sub_test_case 'buffer plugin configured only with path' do
142
161
  setup do
143
162
  @bufpath = File.join(@bufdir, 'testbuf.*.buf')
@@ -502,6 +521,54 @@ class FileSingleBufferTest < Test::Unit::TestCase
502
521
  end
503
522
  end
504
523
 
524
+ sub_test_case 'there are some existing file chunks with placeholders path' do
525
+ setup do
526
+ @buf_ph_dir = File.expand_path('../../tmp/buffer_${test}_file_single_dir', __FILE__)
527
+ FileUtils.rm_rf(@buf_ph_dir)
528
+ FileUtils.mkdir_p(@buf_ph_dir)
529
+
530
+ @c1id = Fluent::UniqueId.generate
531
+ p1 = File.join(@buf_ph_dir, "fsb.testing.q#{Fluent::UniqueId.hex(@c1id)}.buf")
532
+ File.open(p1, 'wb') do |f|
533
+ f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
534
+ end
535
+ t = Time.now - 50000
536
+ File.utime(t, t, p1)
537
+
538
+ @c2id = Fluent::UniqueId.generate
539
+ p2 = File.join(@buf_ph_dir, "fsb.testing.b#{Fluent::UniqueId.hex(@c2id)}.buf")
540
+ File.open(p2, 'wb') do |f|
541
+ f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
542
+ end
543
+ end
544
+
545
+ teardown do
546
+ if @p
547
+ @p.stop unless @p.stopped?
548
+ @p.before_shutdown unless @p.before_shutdown?
549
+ @p.shutdown unless @p.shutdown?
550
+ @p.after_shutdown unless @p.after_shutdown?
551
+ @p.close unless @p.closed?
552
+ @p.terminate unless @p.terminated?
553
+ end
554
+ FileUtils.rm_rf(@buf_ph_dir)
555
+ end
556
+
557
+ test '#resume returns staged/queued chunks with metadata' do
558
+ @d = create_driver(%[
559
+ <buffer tag>
560
+ @type file_single
561
+ path #{@buf_ph_dir}
562
+ </buffer>
563
+ ])
564
+ @p = @d.instance.buffer
565
+ @p.start
566
+
567
+ assert_equal 1, @p.stage.size
568
+ assert_equal 1, @p.queue.size
569
+ end
570
+ end
571
+
505
572
  sub_test_case 'there are some existing msgpack file chunks' do
506
573
  setup do
507
574
  packer = Fluent::MessagePackFactory.packer
@@ -495,6 +495,24 @@ class BufferFileChunkTest < Test::Unit::TestCase
495
495
  end
496
496
  end
497
497
 
498
+ test 'ensure to remove metadata file if #write_metadata raise an error becuase of disk full' do
499
+ chunk_path = File.join(@chunkdir, 'test.*.log')
500
+ stub(Fluent::UniqueId).hex(anything) { 'id' } # to fix chunk id
501
+
502
+ any_instance_of(Fluent::Plugin::Buffer::FileChunk) do |klass|
503
+ stub(klass).write_metadata(anything) do |v|
504
+ raise 'disk full'
505
+ end
506
+ end
507
+
508
+ err = assert_raise(Fluent::Plugin::Buffer::BufferOverflowError) do
509
+ Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, chunk_path, :create)
510
+ end
511
+
512
+ assert_false File.exist?(File.join(@chunkdir, 'test.bid.log.meta'))
513
+ assert_match(/create buffer metadata/, err.message)
514
+ end
515
+
498
516
  sub_test_case 'chunk with file for staged chunk' do
499
517
  setup do
500
518
  @chunk_id = gen_test_chunk_id
@@ -855,6 +873,7 @@ class BufferFileChunkTest < Test::Unit::TestCase
855
873
  assert_equal @gzipped_src, c.read(compressed: :gzip)
856
874
 
857
875
  io = StringIO.new
876
+ io.set_encoding(Encoding::ASCII_8BIT)
858
877
  c.write_to(io, compressed: :gzip)
859
878
  assert_equal @gzipped_src, io.string
860
879
  end
@@ -613,6 +613,7 @@ class BufferFileSingleChunkTest < Test::Unit::TestCase
613
613
  assert_equal @gzipped_src, c.read(compressed: :gzip)
614
614
 
615
615
  io = StringIO.new
616
+ io.set_encoding(Encoding::ASCII_8BIT)
616
617
  c.write_to(io, compressed: :gzip)
617
618
  assert_equal @gzipped_src, io.string
618
619
  end
@@ -331,6 +331,7 @@ class BufferMemoryChunkTest < Test::Unit::TestCase
331
331
  assert_equal @gzipped_src, c.read(compressed: :gzip)
332
332
 
333
333
  io = StringIO.new
334
+ io.set_encoding(Encoding::ASCII_8BIT)
334
335
  c.write_to(io, compressed: :gzip)
335
336
  assert_equal @gzipped_src, io.string
336
337
  end
@@ -163,6 +163,46 @@ class TcpInputTest < Test::Unit::TestCase
163
163
  assert_equal address, event[2]['addr']
164
164
  end
165
165
 
166
+ sub_test_case '<security>' do
167
+ test 'accept from allowed client' do
168
+ d = create_driver(CONFIG + %!
169
+ <security>
170
+ <client>
171
+ network 127.0.0.1
172
+ </client>
173
+ </security>
174
+ !)
175
+ d.run(expect_records: 1) do
176
+ create_tcp_socket('127.0.0.1', PORT) do |sock|
177
+ sock.send("hello\n", 0)
178
+ end
179
+ end
180
+
181
+ assert_equal 1, d.events.size
182
+ event = d.events[0]
183
+ assert_equal 'tcp', event[0]
184
+ assert_equal 'hello', event[2]['message']
185
+ end
186
+
187
+ test 'deny from disallowed client' do
188
+ d = create_driver(CONFIG + %!
189
+ <security>
190
+ <client>
191
+ network 200.0.0.0
192
+ </client>
193
+ </security>
194
+ !)
195
+ d.run(shutdown: false, expect_records: 1, timeout: 2) do
196
+ create_tcp_socket('127.0.0.1', PORT) do |sock|
197
+ sock.send("hello\n", 0)
198
+ end
199
+ end
200
+
201
+ assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
202
+ assert_equal 0, d.events.size
203
+ end
204
+ end
205
+
166
206
  sub_test_case '<extract>' do
167
207
  test 'extract tag from record field' do
168
208
  d = create_driver(BASE_CONFIG + %!
@@ -185,6 +185,81 @@ EOL
185
185
  assert_equal([dummy_cert_path], d.instance.tls_ca_cert_path)
186
186
  end
187
187
 
188
+ sub_test_case "certstore loading parameters for Windows" do
189
+ test 'certstore related config parameters' do
190
+ omit "certstore related values raise error on not Windows" if Fluent.windows?
191
+ conf = %[
192
+ send_timeout 5
193
+ transport tls
194
+ tls_cert_logical_store_name Root
195
+ tls_cert_thumbprint a909502dd82ae41433e6f83886b00d4277a32a7b
196
+ <server>
197
+ host #{TARGET_HOST}
198
+ port #{TARGET_PORT}
199
+ </server>
200
+ ]
201
+
202
+ assert_raise(Fluent::ConfigError) do
203
+ create_driver(conf)
204
+ end
205
+ end
206
+
207
+ test 'cert_logical_store_name and tls_cert_thumbprint default values' do
208
+ conf = %[
209
+ send_timeout 5
210
+ transport tls
211
+ <server>
212
+ host #{TARGET_HOST}
213
+ port #{TARGET_PORT}
214
+ </server>
215
+ ]
216
+
217
+ @d = d = create_driver(conf)
218
+ assert_nil d.instance.tls_cert_logical_store_name
219
+ assert_nil d.instance.tls_cert_thumbprint
220
+ end
221
+
222
+ data('CA cert' => 'tls_ca_cert_path',
223
+ 'non CA cert' => 'tls_cert_path')
224
+ test 'specify tls_cert_logical_store_name and tls_cert_path should raise error' do |param|
225
+ omit "Loading CertStore feature works only Windows" unless Fluent.windows?
226
+ dummy_cert_path = File.join(TMP_DIR, "dummy_cert.pem")
227
+ FileUtils.touch(dummy_cert_path)
228
+ conf = %[
229
+ send_timeout 5
230
+ transport tls
231
+ #{param} #{dummy_cert_path}
232
+ tls_cert_logical_store_name Root
233
+ <server>
234
+ host #{TARGET_HOST}
235
+ port #{TARGET_PORT}
236
+ </server>
237
+ ]
238
+
239
+ assert_raise(Fluent::ConfigError) do
240
+ create_driver(conf)
241
+ end
242
+ end
243
+
244
+ test 'configure cert_logical_store_name and tls_cert_thumbprint' do
245
+ omit "Loading CertStore feature works only Windows" unless Fluent.windows?
246
+ conf = %[
247
+ send_timeout 5
248
+ transport tls
249
+ tls_cert_logical_store_name Root
250
+ tls_cert_thumbprint a909502dd82ae41433e6f83886b00d4277a32a7b
251
+ <server>
252
+ host #{TARGET_HOST}
253
+ port #{TARGET_PORT}
254
+ </server>
255
+ ]
256
+
257
+ @d = d = create_driver(conf)
258
+ assert_equal "Root", d.instance.tls_cert_logical_store_name
259
+ assert_equal "a909502dd82ae41433e6f83886b00d4277a32a7b", d.instance.tls_cert_thumbprint
260
+ end
261
+ end
262
+
188
263
  test 'compress_default_value' do
189
264
  @d = d = create_driver
190
265
  assert_equal :text, d.instance.compress
@@ -868,6 +868,23 @@ class OutputTest < Test::Unit::TestCase
868
868
  end
869
869
  end
870
870
 
871
+ test 'raises an error if timekey is less than equal 0' do
872
+ i = create_output(:delayed)
873
+ assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: 0.0') do
874
+ i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => nil })]))
875
+ end
876
+
877
+ i = create_output(:delayed)
878
+ assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: 0.0') do
879
+ i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => 0 })]))
880
+ end
881
+
882
+ i = create_output(:delayed)
883
+ assert_raise Fluent::ConfigError.new('timekey should be greater than 0. current timekey: -1.0') do
884
+ i.configure(config_element('ROOT','',{},[config_element('buffer', 'time', { "timekey" => -1 })]))
885
+ end
886
+ end
887
+
871
888
  sub_test_case 'sync output feature' do
872
889
  setup do
873
890
  @i = create_output(:sync)
@@ -14,8 +14,9 @@ class SyslogParserTest < ::Test::Unit::TestCase
14
14
  }
15
15
  end
16
16
 
17
- def test_parse
18
- @parser.configure({})
17
+ data('regexp' => 'regexp', 'string' => 'string')
18
+ def test_parse(param)
19
+ @parser.configure({'parser_type' => param})
19
20
  @parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
20
21
  assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
21
22
  assert_equal(@expected, record)
@@ -24,8 +25,9 @@ class SyslogParserTest < ::Test::Unit::TestCase
24
25
  assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
25
26
  end
26
27
 
27
- def test_parse_with_time_format
28
- @parser.configure('time_format' => '%b %d %M:%S:%H')
28
+ data('regexp' => 'regexp', 'string' => 'string')
29
+ def test_parse_with_time_format(param)
30
+ @parser.configure('time_format' => '%b %d %M:%S:%H', 'parser_type' => param)
29
31
  @parser.instance.parse('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
30
32
  assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
31
33
  assert_equal(@expected, record)
@@ -33,8 +35,18 @@ class SyslogParserTest < ::Test::Unit::TestCase
33
35
  assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
34
36
  end
35
37
 
36
- def test_parse_with_priority
37
- @parser.configure('with_priority' => true)
38
+ data('regexp' => 'regexp', 'string' => 'string')
39
+ def test_parse_with_subsecond_time(param)
40
+ @parser.configure('time_format' => '%b %d %H:%M:%S.%N', 'parser_type' => param)
41
+ @parser.instance.parse('Feb 28 12:00:00.456 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
42
+ assert_equal(event_time('Feb 28 12:00:00.456', format: '%b %d %H:%M:%S.%N'), time)
43
+ assert_equal(@expected, record)
44
+ }
45
+ end
46
+
47
+ data('regexp' => 'regexp', 'string' => 'string')
48
+ def test_parse_with_priority(param)
49
+ @parser.configure('with_priority' => true, 'parser_type' => param)
38
50
  @parser.instance.parse('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
39
51
  assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
40
52
  assert_equal(@expected.merge('pri' => 6), record)
@@ -43,8 +55,18 @@ class SyslogParserTest < ::Test::Unit::TestCase
43
55
  assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
44
56
  end
45
57
 
46
- def test_parse_without_colon
47
- @parser.configure({})
58
+ data('regexp' => 'regexp', 'string' => 'string')
59
+ def test_parse_with_empty_priority(param)
60
+ @parser.configure('with_priority' => true, 'parser_type' => param)
61
+ @parser.instance.parse('<>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
62
+ assert_nil time
63
+ assert_nil record
64
+ }
65
+ end
66
+
67
+ data('regexp' => 'regexp', 'string' => 'string')
68
+ def test_parse_without_colon(param)
69
+ @parser.configure({'parser_type' => param})
48
70
  @parser.instance.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
49
71
  assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
50
72
  assert_equal(@expected, record)
@@ -53,10 +75,12 @@ class SyslogParserTest < ::Test::Unit::TestCase
53
75
  assert_equal("%b %d %H:%M:%S", @parser.instance.patterns['time_format'])
54
76
  end
55
77
 
56
- def test_parse_with_keep_time_key
78
+ data('regexp' => 'regexp', 'string' => 'string')
79
+ def test_parse_with_keep_time_key(param)
57
80
  @parser.configure(
58
81
  'time_format' => '%b %d %M:%S:%H',
59
82
  'keep_time_key'=>'true',
83
+ 'parser_type' => param
60
84
  )
61
85
  text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
62
86
  @parser.instance.parse(text) do |time, record|
@@ -64,24 +88,87 @@ class SyslogParserTest < ::Test::Unit::TestCase
64
88
  end
65
89
  end
66
90
 
67
- def test_parse_various_characters_for_tag
91
+ data('regexp' => 'regexp', 'string' => 'string')
92
+ def test_parse_various_characters_for_tag(param)
68
93
  ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
69
- @parser.configure({})
94
+ @parser.configure({'parser_type' => param})
70
95
  @parser.instance.parse("Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
71
96
  assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
72
97
  assert_equal(@expected.merge('ident' => ident), record)
73
98
  }
74
99
  end
75
100
 
76
- def test_parse_various_characters_for_tag_with_priority
101
+ data('regexp' => 'regexp', 'string' => 'string')
102
+ def test_parse_various_characters_for_tag_with_priority(param)
77
103
  ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
78
- @parser.configure('with_priority' => true)
104
+ @parser.configure('with_priority' => true, 'parser_type' => param)
79
105
  @parser.instance.parse("<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
80
106
  assert_equal(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
81
107
  assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)
82
108
  }
83
109
  end
84
110
 
111
+ sub_test_case 'Check the difference of regexp and string parser' do
112
+ # examples from rfc3164
113
+ data('regexp' => 'regexp', 'string' => 'string')
114
+ test 'wrong result with no ident message by default' do |param|
115
+ @parser.configure('parser_type' => param)
116
+ @parser.instance.parse('Feb 5 17:32:18 10.0.0.99 Use the BFG!') { |time, record|
117
+ assert_equal({'host' => '10.0.0.99', 'ident' => 'Use', 'message' => 'the BFG!'}, record)
118
+ }
119
+ end
120
+
121
+ test "proper result with no ident message by 'support_colonless_ident false'" do
122
+ @parser.configure('parser_type' => 'string', 'support_colonless_ident' => false)
123
+ @parser.instance.parse('Feb 5 17:32:18 10.0.0.99 Use the BFG!') { |time, record|
124
+ assert_equal({'host' => '10.0.0.99', 'message' => 'Use the BFG!'}, record)
125
+ }
126
+ end
127
+
128
+ test "string parsers can't parse broken syslog message and generate wrong record" do
129
+ @parser.configure('parser_type' => 'string')
130
+ @parser.instance.parse("1990 Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.32 sched[0]: That's All Folks!") { |time, record|
131
+ expected = {'host' => 'scapegoat.dmz.example.org', 'ident' => 'sched', 'pid' => '0', 'message' => "That's All Folks!"}
132
+ assert_not_equal(expected, record)
133
+ }
134
+ end
135
+
136
+ test "regexp parsers can't parse broken syslog message and raises an error" do
137
+ @parser.configure('parser_type' => 'regexp')
138
+ assert_raise(Fluent::TimeParser::TimeParseError) {
139
+ @parser.instance.parse("1990 Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.32 sched[0]: That's All Folks!") { |time, record| }
140
+ }
141
+ end
142
+
143
+ data('regexp' => 'regexp', 'string' => 'string')
144
+ test "':' included message breaks regexp parser" do |param|
145
+ @parser.configure('parser_type' => param)
146
+ @parser.instance.parse('Aug 10 12:00:00 127.0.0.1 test foo:bar') { |time, record|
147
+ expected = {'host' => '127.0.0.1', 'ident' => 'test', 'message' => 'foo:bar'}
148
+ if param == 'string'
149
+ assert_equal(expected, record)
150
+ else
151
+ assert_not_equal(expected, record)
152
+ end
153
+ }
154
+ end
155
+
156
+ data('regexp' => 'regexp', 'string' => 'string')
157
+ test "Only no whitespace content in MSG causes different result" do |param|
158
+ @parser.configure('parser_type' => param)
159
+ @parser.instance.parse('Aug 10 12:00:00 127.0.0.1 value1,value2,value3,value4') { |time, record|
160
+ # 'message' is correct but regexp set it as 'ident'
161
+ if param == 'string'
162
+ expected = {'host' => '127.0.0.1', 'message' => 'value1,value2,value3,value4'}
163
+ assert_equal(expected, record)
164
+ else
165
+ expected = {'host' => '127.0.0.1', 'ident' => 'value1,value2,value3,value4', 'message' => ''}
166
+ assert_equal(expected, record)
167
+ end
168
+ }
169
+ end
170
+ end
171
+
85
172
  class TestRFC5424Regexp < self
86
173
  def test_parse_with_rfc5424_message
87
174
  @parser.configure(
@@ -273,10 +360,12 @@ class SyslogParserTest < ::Test::Unit::TestCase
273
360
  end
274
361
 
275
362
  class TestAutoRegexp < self
276
- def test_auto_with_legacy_syslog_message
363
+ data('regexp' => 'regexp', 'string' => 'string')
364
+ def test_auto_with_legacy_syslog_message(param)
277
365
  @parser.configure(
278
366
  'time_format' => '%b %d %M:%S:%H',
279
367
  'message_format' => 'auto',
368
+ 'parser_type' => param
280
369
  )
281
370
  text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
282
371
  @parser.instance.parse(text) do |time, record|
@@ -286,11 +375,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
286
375
  assert_equal(Fluent::Plugin::SyslogParser::REGEXP, @parser.instance.patterns['format'])
287
376
  end
288
377
 
289
- def test_auto_with_legacy_syslog_priority_message
378
+ data('regexp' => 'regexp', 'string' => 'string')
379
+ def test_auto_with_legacy_syslog_priority_message(param)
290
380
  @parser.configure(
291
381
  'time_format' => '%b %d %M:%S:%H',
292
382
  'with_priority' => true,
293
383
  'message_format' => 'auto',
384
+ 'parser_type' => param
294
385
  )
295
386
  text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
296
387
  @parser.instance.parse(text) do |time, record|
@@ -300,11 +391,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
300
391
  assert_equal(Fluent::Plugin::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
301
392
  end
302
393
 
303
- def test_parse_with_rfc5424_message
394
+ data('regexp' => 'regexp', 'string' => 'string')
395
+ def test_parse_with_rfc5424_message(param)
304
396
  @parser.configure(
305
397
  'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
306
398
  'message_format' => 'auto',
307
399
  'with_priority' => true,
400
+ 'parser_type' => param
308
401
  )
309
402
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
310
403
  @parser.instance.parse(text) do |time, record|
@@ -318,11 +411,13 @@ class SyslogParserTest < ::Test::Unit::TestCase
318
411
  @parser.instance.patterns['format'])
319
412
  end
320
413
 
321
- def test_parse_with_rfc5424_structured_message
414
+ data('regexp' => 'regexp', 'string' => 'string')
415
+ def test_parse_with_rfc5424_structured_message(param)
322
416
  @parser.configure(
323
417
  'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
324
418
  'message_format' => 'auto',
325
419
  'with_priority' => true,
420
+ 'parser_type' => param
326
421
  )
327
422
  text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
328
423
  @parser.instance.parse(text) do |time, record|
@@ -337,12 +432,14 @@ class SyslogParserTest < ::Test::Unit::TestCase
337
432
  @parser.instance.patterns['format'])
338
433
  end
339
434
 
340
- def test_parse_with_both_message_type
435
+ data('regexp' => 'regexp', 'string' => 'string')
436
+ def test_parse_with_both_message_type(param)
341
437
  @parser.configure(
342
438
  'time_format' => '%b %d %M:%S:%H',
343
439
  'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
344
440
  'message_format' => 'auto',
345
441
  'with_priority' => true,
442
+ 'parser_type' => param
346
443
  )
347
444
  text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
348
445
  @parser.instance.parse(text) do |time, record|
@@ -382,12 +479,14 @@ class SyslogParserTest < ::Test::Unit::TestCase
382
479
  @parser.instance.patterns['format'])
383
480
  end
384
481
 
385
- def test_parse_with_both_message_type_and_priority
482
+ data('regexp' => 'regexp', 'string' => 'string')
483
+ def test_parse_with_both_message_type_and_priority(param)
386
484
  @parser.configure(
387
485
  'time_format' => '%b %d %M:%S:%H',
388
486
  'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
389
487
  'with_priority' => true,
390
488
  'message_format' => 'auto',
489
+ 'parser_type' => param
391
490
  )
392
491
  text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
393
492
  @parser.instance.parse(text) do |time, record|
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.7.0
4
+ version: 1.7.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-08-21 00:00:00.000000000 Z
11
+ date: 2019-09-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack
@@ -16,7 +16,7 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.7.0
19
+ version: 1.2.0
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
22
  version: 2.0.0
@@ -26,7 +26,7 @@ dependencies:
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: 0.7.0
29
+ version: 1.2.0
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: 2.0.0