fluentd 0.10.51 → 0.10.52

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c0d7335ca94183ce7a33d7578b88addeb64b1c8f
4
- data.tar.gz: 77807fdc964ab88d09ecc41787339c0825dba08c
3
+ metadata.gz: 6871e2cbaea0e7068a4671988446b2cbbf8d5efb
4
+ data.tar.gz: 7ec21b7f21fa8787b7a577894849df0ea5acf22b
5
5
  SHA512:
6
- metadata.gz: d3d90bcc098a60fc5525960445b25e6e5aea09f4221be6cbb943b8d656a34bb89ffdbfdad89a645394c85d953fd9577f2ec359f37f90c66520e658cb5c4f8d6a
7
- data.tar.gz: a5759301599ea3243f1cb0021910ad411145cb524312d9d653be7bf94aeba6866eab42515c9dc77cb7279743f73290652c312feba6682113e57720b1951eca40
6
+ metadata.gz: 924081ce1193ab72e08f05bc9ab758716d2aea3d988b1c2f735781eda2432acc78c5fea0bf40032aedc54f72067a5a39784e23cb9fa3a15ac4064792aff45dc7
7
+ data.tar.gz: 726dbda087652aa3c88f17f3a7f6dcd34d4dc7dab5a576efbf08883f10256f3e2ee3fa0bae375a69a3f769fd47deb36f22bb256f6defbd607f1bd6596649dc1c
data/.gitignore CHANGED
@@ -17,4 +17,5 @@ fluent-gem
17
17
  fluentd
18
18
  pkg/*
19
19
  test/tmp/*
20
+ spec/config/tmp/*
20
21
  make_dist.sh
data/ChangeLog CHANGED
@@ -1,3 +1,14 @@
1
+ Release 0.10.52 - 2014/07/15
2
+
3
+ * in_tail: Fix typo of warning argument. lb to line
4
+ * config: Fix 'uninitialized constant Fluent::Config::V1Parser::URI' error
5
+ * config: Reject '@' prefix of parameter name in V1 configuration. @ is reserved prefix.
6
+ * config; Accept '#' comment in array and hash types
7
+ * parser: Add time_format option to SyslogParser
8
+ * parser: SyslogParser now makes ':' optional after pid
9
+ * process: Fix 'mutex can't lock' issue of DetachMultiProcessMixin on Ruby 2.x
10
+ * Add --without-source option to ignore <source> sections
11
+
1
12
  Release 0.10.51 - 2014/07/04
2
13
 
3
14
  * in_tail: Fix rotation handling when new file not found
@@ -6,7 +6,7 @@ Gem::Specification.new do |gem|
6
6
 
7
7
  gem.authors = ["Sadayuki Furuhashi"]
8
8
  gem.email = ["frsyuki@gmail.com"]
9
- gem.description = %q{Fluentd is an event collector system. It is a generalized version of syslogd, which handles JSON objects for its log messages}
9
+ gem.description = %q{Fluentd is an open source data collector designed to scale and simplify log management. It can collect, process and ship many kinds of data in near real-time.}
10
10
  gem.summary = %q{Fluentd event collector}
11
11
  gem.homepage = "http://fluentd.org/"
12
12
 
@@ -17,10 +17,7 @@
17
17
  #
18
18
 
19
19
  require 'optparse'
20
- require 'fluent/log'
21
- require 'fluent/env'
22
20
  require 'fluent/supervisor'
23
- require 'fluent/version'
24
21
 
25
22
  op = OptionParser.new
26
23
  op.version = Fluent::VERSION
@@ -79,6 +76,10 @@ op.on('--suppress-repeated-stacktrace', "suppress repeated stacktrace", TrueClas
79
76
  opts[:suppress_repeated_stacktrace] = b
80
77
  }
81
78
 
79
+ op.on('--without-source', "invoke a fluentd without input plugins", TrueClass) {|b|
80
+ opts[:without_source] = b
81
+ }
82
+
82
83
  op.on('--use-v1-config', "Use v1 configuration format", TrueClass) {|b|
83
84
  opts[:use_v1_config] = b
84
85
  }
@@ -156,6 +157,4 @@ if setup_path = opts[:setup_path]
156
157
  exit 0
157
158
  end
158
159
 
159
- require 'fluent/supervisor'
160
160
  Fluent::Supervisor.new(opts).start
161
-
@@ -22,6 +22,19 @@ module Fluent
22
22
  e
23
23
  end
24
24
 
25
+ def inspect
26
+ attrs = super
27
+ "name:#{@name}, arg:#{@arg}, " + attrs + ", " + @elements.inspect
28
+ end
29
+
30
+ def ==(o)
31
+ self.name == o.name && self.arg == o.arg &&
32
+ self.keys.size == o.keys.size &&
33
+ self.keys.reduce(true){|r, k| r && self[k] == o[k] } &&
34
+ self.elements.size == o.elements.size &&
35
+ [self.elements, o.elements].transpose.reduce(true){|r, e| r && e[0] == e[1] }
36
+ end
37
+
25
38
  def +(o)
26
39
  Element.new(@name.dup, @arg.dup, o.merge(self), @elements + o.elements, (@unused + o.unused).uniq)
27
40
  end
@@ -139,19 +139,61 @@ module Fluent
139
139
 
140
140
  def scan_json(is_array)
141
141
  result = nil
142
+ # Yajl does not raise ParseError for imcomplete json string, like '[1', '{"h"', '{"h":' or '{"h1":1'
143
+ # This is the reason to use JSON module.
144
+
145
+ buffer = (is_array ? "[" : "{")
146
+ line_buffer = ""
142
147
 
143
- y = Yajl::Parser.new(:allow_comments => true)
144
- y.on_parse_complete = Proc.new { |obj| result = obj }
145
- y << (is_array ? '[' : '{')
146
148
  until result
147
- ch = getch
148
- parse_error! "got incomplete #{is_array ? 'array' : 'hash'} configuration" if ch.nil?
149
- y << ch
149
+ char = getch
150
+
151
+ break if char.nil?
152
+
153
+ if char == "#"
154
+ # If this is out of json string literals, this object can be parsed correctly
155
+ # '{"foo":"bar", #' -> '{"foo":"bar"}' (to check)
156
+ parsed = nil
157
+ begin
158
+ parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? "]" : "}"))
159
+ rescue JSON::ParserError => e
160
+ # This '#' is in json string literals
161
+ end
162
+
163
+ if parsed
164
+ # ignore chars as comment before newline
165
+ while (char = getch) != "\n"
166
+ # ignore comment char
167
+ end
168
+ buffer << line_buffer + "\n"
169
+ line_buffer = ""
170
+ else
171
+ # '#' is a char in json string
172
+ line_buffer << char
173
+ end
174
+
175
+ next # This char '#' MUST NOT terminate json object.
176
+ end
177
+
178
+ if char == "\n"
179
+ buffer << line_buffer + "\n"
180
+ line_buffer = ""
181
+ next
182
+ end
183
+
184
+ line_buffer << char
185
+ begin
186
+ result = JSON.parse(buffer + line_buffer)
187
+ rescue JSON::ParserError => e
188
+ # Incomplete json string yet
189
+ end
190
+ end
191
+
192
+ unless result
193
+ parse_error! "got incomplete #{is_array ? 'array' : 'hash'} configuration"
150
194
  end
151
195
 
152
- Yajl.dump(result) # Convert json to string for config_param
153
- rescue Yajl::ParseError => e
154
- parse_error! "unexpected #{is_array ? 'array' : 'hash'} parse error"
196
+ JSON.dump(result)
155
197
  end
156
198
  end
157
199
  end
@@ -103,6 +103,10 @@ module Fluent
103
103
  if k == '@include'
104
104
  parse_include(attrs, elems)
105
105
  else
106
+ if k.start_with?('@')
107
+ parse_error! "'@' is reserved prefix. Don't use '@' in parameter name"
108
+ end
109
+
106
110
  v = parse_literal
107
111
  unless line_end
108
112
  parse_error! "expected end of line"
@@ -35,6 +35,7 @@ module Fluent
35
35
  @next_emit_error_log_time = nil
36
36
 
37
37
  @suppress_config_dump = false
38
+ @without_source = false
38
39
  end
39
40
 
40
41
  MATCH_CACHE_SIZE = 1024
@@ -43,13 +44,18 @@ module Fluent
43
44
 
44
45
  attr_reader :matches, :sources
45
46
 
46
- def init
47
+ def init(opts = {})
47
48
  BasicSocket.do_not_reverse_lookup = true
48
49
  Plugin.load_plugins
49
50
  if defined?(Encoding)
50
51
  Encoding.default_internal = 'ASCII-8BIT' if Encoding.respond_to?(:default_internal)
51
52
  Encoding.default_external = 'ASCII-8BIT' if Encoding.respond_to?(:default_external)
52
53
  end
54
+
55
+ suppress_interval(opts[:suppress_interval]) if opts[:suppress_interval]
56
+ @suppress_config_dump = opts[:suppress_config_dump] if opts[:suppress_config_dump]
57
+ @without_source = opts[:without_source] if opts[:without_source]
58
+
53
59
  self
54
60
  end
55
61
 
@@ -58,10 +64,6 @@ module Fluent
58
64
  @next_emit_error_log_time = Time.now.to_i
59
65
  end
60
66
 
61
- def suppress_config_dump=(flag)
62
- @suppress_config_dump = flag
63
- end
64
-
65
67
  def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
66
68
  if fname =~ /\.rb$/
67
69
  require 'fluent/config/dsl'
@@ -74,7 +76,11 @@ module Fluent
74
76
  def run_configure(conf)
75
77
  configure(conf)
76
78
  conf.check_not_fetched { |key, e|
77
- $log.warn "parameter '#{key}' in #{e.to_s.strip} is not used." unless e.name == 'system'
79
+ unless e.name == 'system'
80
+ unless @without_source && e.name == 'source'
81
+ $log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."
82
+ end
83
+ end
78
84
  }
79
85
  end
80
86
 
@@ -88,20 +94,24 @@ module Fluent
88
94
  $log.info "using configuration file: #{conf.to_s.rstrip}"
89
95
  end
90
96
 
91
- conf.elements.select {|e|
92
- e.name == 'source'
93
- }.each {|e|
94
- type = e['type']
95
- unless type
96
- raise ConfigError, "Missing 'type' parameter on <source> directive"
97
- end
98
- $log.info "adding source type=#{type.dump}"
97
+ if @without_source
98
+ $log.info "'--without-source' is applied. Ignore <source> sections"
99
+ else
100
+ conf.elements.select {|e|
101
+ e.name == 'source'
102
+ }.each {|e|
103
+ type = e['type']
104
+ unless type
105
+ raise ConfigError, "Missing 'type' parameter on <source> directive"
106
+ end
107
+ $log.info "adding source type=#{type.dump}"
99
108
 
100
- input = Plugin.new_input(type)
101
- input.configure(e)
109
+ input = Plugin.new_input(type)
110
+ input.configure(e)
102
111
 
103
- @sources << input
104
- }
112
+ @sources << input
113
+ }
114
+ end
105
115
 
106
116
  conf.elements.select {|e|
107
117
  e.name == 'match'
@@ -471,11 +471,11 @@ module Fluent
471
471
  include Configurable
472
472
 
473
473
  # From existence TextParser pattern
474
- REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/
474
+ REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
475
475
  # From in_syslog default pattern
476
- REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/
477
- TIME_FORMAT = "%b %d %H:%M:%S"
476
+ REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
478
477
 
478
+ config_param :time_format, :string, :default => "%b %d %H:%M:%S"
479
479
  config_param :with_priority, :bool, :default => false
480
480
 
481
481
  attr_accessor :estimate_current_event
@@ -483,7 +483,6 @@ module Fluent
483
483
  def initialize
484
484
  super
485
485
  @estimate_current_event = true
486
- @time_parser = TextParser::TimeParser.new(TIME_FORMAT)
487
486
  @mutex = Mutex.new
488
487
  end
489
488
 
@@ -491,10 +490,11 @@ module Fluent
491
490
  super
492
491
 
493
492
  @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
493
+ @time_parser = TextParser::TimeParser.new(@time_format)
494
494
  end
495
495
 
496
496
  def patterns
497
- {'format' => @regexp, 'time_format' => TIME_FORMAT}
497
+ {'format' => @regexp, 'time_format' => @time_format}
498
498
  end
499
499
 
500
500
  def call(text)
@@ -267,7 +267,7 @@ module Fluent
267
267
  lb = line
268
268
  else
269
269
  if lb.nil?
270
- log.warn "got incomplete line before first line from #{tail_watcher.path}: #{lb.inspect}"
270
+ log.warn "got incomplete line before first line from #{tail_watcher.path}: #{line.inspect}"
271
271
  else
272
272
  lb << line
273
273
  end
@@ -378,9 +378,12 @@ module Fluent
378
378
  def stop
379
379
  return if @finished
380
380
  @finished = true
381
- @mutex.synchronize do
382
- @cond.broadcast
383
- end
381
+ # Creating new thread due to mutex can't lock in main thread during trap context
382
+ Thread.new {
383
+ @mutex.synchronize do
384
+ @cond.broadcast
385
+ end
386
+ }.run
384
387
  end
385
388
 
386
389
  def finished?
@@ -16,9 +16,7 @@
16
16
  # limitations under the License.
17
17
  #
18
18
 
19
- require 'fluent/env'
20
- require 'fluent/log'
21
- require 'fluent/config'
19
+ require 'fluent/load'
22
20
  require 'etc'
23
21
 
24
22
  module Fluent
@@ -61,7 +59,6 @@ module Fluent
61
59
  end
62
60
 
63
61
  $log = Fluent::Log.new(@io, @level, @opts)
64
-
65
62
  $log.enable_color(false) if @path
66
63
  $log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
67
64
  end
@@ -91,6 +88,7 @@ module Fluent
91
88
  :chgroup => nil,
92
89
  :suppress_interval => 0,
93
90
  :suppress_repeated_stacktrace => false,
91
+ :without_source => false,
94
92
  :use_v1_config => false,
95
93
  }
96
94
  end
@@ -112,6 +110,7 @@ module Fluent
112
110
  @log_level = opt[:log_level]
113
111
  @suppress_interval = opt[:suppress_interval]
114
112
  @suppress_config_dump = opt[:suppress_config_dump]
113
+ @without_source = opt[:without_source]
115
114
 
116
115
  log_opts = {:suppress_repeated_stacktrace => opt[:suppress_repeated_stacktrace]}
117
116
  @log = LoggerInitializer.new(@log_path, @log_level, @chuser, @chgroup, log_opts)
@@ -120,7 +119,6 @@ module Fluent
120
119
  end
121
120
 
122
121
  def start
123
- require 'fluent/load'
124
122
  @log.init
125
123
 
126
124
  dry_run if @dry_run
@@ -321,9 +319,8 @@ module Fluent
321
319
  end
322
320
  end
323
321
 
324
- # with_log is for disabling logging before Log#init is called
325
- def read_config(with_log = true)
326
- $log.info "reading config file", :path => @config_path if with_log
322
+ def read_config
323
+ $log.info "reading config file", :path => @config_path
327
324
  @config_fname = File.basename(@config_path)
328
325
  @config_basedir = File.dirname(@config_path)
329
326
  @config_data = File.read(@config_path)
@@ -343,6 +340,7 @@ module Fluent
343
340
  config_param :suppress_repeated_stacktrace, :bool, :default => nil
344
341
  config_param :emit_error_log_interval, :time, :default => nil
345
342
  config_param :suppress_config_dump, :bool, :default => nil
343
+ config_param :without_source, :bool, :default => nil
346
344
 
347
345
  def initialize(conf)
348
346
  super()
@@ -355,12 +353,16 @@ module Fluent
355
353
  opt[:suppress_interval] = @emit_error_log_interval unless @emit_error_log_interval.nil?
356
354
  opt[:suppress_config_dump] = @suppress_config_dump unless @suppress_config_dump.nil?
357
355
  opt[:suppress_repeated_stacktrace] = @suppress_repeated_stacktrace unless @suppress_repeated_stacktrace.nil?
356
+ opt[:without_source] = @without_source unless @without_source.nil?
358
357
  opt
359
358
  end
360
359
  end
361
360
 
362
361
  def apply_system_config(opt)
363
- read_config(false)
362
+ # Create NULL file to avoid $log uninitialized problem before call @log.init
363
+ file = File.open(File::NULL)
364
+ $log = Fluent::Log.new(file, Log::LEVEL_INFO)
365
+ read_config
364
366
  systems = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config).elements.select { |e|
365
367
  e.name == 'system'
366
368
  }
@@ -368,6 +370,8 @@ module Fluent
368
370
  raise ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
369
371
 
370
372
  opt.merge!(SystemConfig.new(systems.first).to_opt)
373
+ ensure
374
+ file.close
371
375
  end
372
376
 
373
377
  def run_configure
@@ -393,13 +397,8 @@ module Fluent
393
397
  end
394
398
 
395
399
  def init_engine
396
- require 'fluent/load'
397
- Fluent::Engine.init
398
- if @suppress_interval
399
- Fluent::Engine.suppress_interval(@suppress_interval)
400
- end
401
-
402
- Fluent::Engine.suppress_config_dump = @suppress_config_dump
400
+ init_opts = {:suppress_interval => @suppress_interval, :suppress_config_dump => @suppress_config_dump, :without_source => @without_source}
401
+ Fluent::Engine.init(init_opts)
403
402
 
404
403
  @libs.each {|lib|
405
404
  require lib
@@ -1,5 +1,5 @@
1
1
  module Fluent
2
2
 
3
- VERSION = '0.10.51'
3
+ VERSION = '0.10.52'
4
4
 
5
5
  end
@@ -37,49 +37,53 @@ describe Fluent::Config::V1Parser do
37
37
  expect(%[
38
38
  k1 v1
39
39
  k2 v2
40
- ]).to be_parsed_as("k1"=>"v1", "k2"=>"v2")
40
+ ]).to be_parsed_as(e('ROOT', '', {"k1"=>"v1", "k2"=>"v2"}))
41
41
  end
42
42
 
43
43
  it "allows attribute without value" do
44
44
  expect(%[
45
45
  k1
46
46
  k2 v2
47
- ]).to be_parsed_as("k1"=>"", "k2"=>"v2")
47
+ ]).to be_parsed_as(e('ROOT', '', {"k1"=>"", "k2"=>"v2"}))
48
48
  end
49
49
 
50
50
  it "parses attribute key always string" do
51
- expect("1 1").to be_parsed_as("1" => "1")
51
+ expect("1 1").to be_parsed_as(e('ROOT', '', {"1" => "1"}))
52
52
  end
53
53
 
54
54
  [
55
55
  "_.%$!,",
56
- "/=~-~@`:?",
56
+ "/=~-~@\`:?",
57
57
  "()*{}.[]",
58
58
  ].each do |v|
59
59
  it "parses a value with symbol #{v.inspect}" do
60
- expect("k #{v}").to be_parsed_as("k" => v)
60
+ expect("k #{v}").to be_parsed_as(e('ROOT', '', {"k" => v}))
61
61
  end
62
62
  end
63
63
 
64
64
  it "ignores spacing around value" do
65
- expect(" k1 a ").to be_parsed_as("k1" => "a")
65
+ expect(" k1 a ").to be_parsed_as(e('ROOT', '', {"k1" => "a"}))
66
66
  end
67
67
 
68
68
  it "allows spaces in value" do
69
- expect("k1 a b c").to be_parsed_as("k1" => "a b c")
69
+ expect("k1 a b c").to be_parsed_as(e('ROOT', '', {"k1" => "a b c"}))
70
70
  end
71
71
 
72
72
  it "ignores comments after value" do
73
- expect(" k1 a#comment").to be_parsed_as("k1" => "a")
73
+ expect(" k1 a#comment").to be_parsed_as(e('ROOT', '', {"k1" => "a"}))
74
74
  end
75
75
 
76
76
  it "allows # in value if quoted" do
77
- expect(' k1 "a#comment"').to be_parsed_as("k1" => "a#comment")
77
+ expect(' k1 "a#comment"').to be_parsed_as(e('ROOT', '', {"k1" => "a#comment"}))
78
78
  end
79
79
 
80
80
  it "rejects characters after quoted string" do
81
81
  expect(' k1 "a" 1').to be_parse_error
82
82
  end
83
+
84
+ it "rejects @ prefix in parameter name" do
85
+ expect(' @k v').to be_parse_error
86
+ end
83
87
  end
84
88
 
85
89
  describe 'element parsing' do
@@ -118,13 +122,25 @@ describe Fluent::Config::V1Parser do
118
122
  </nested2>
119
123
  </test>
120
124
  ]).to be_parsed_as(root(
121
- e("test", 'var', {'key'=>'val'}, [
125
+ e("test", 'var', {'key'=>'1'}, [
122
126
  e('nested1'),
123
127
  e('nested2')
124
128
  ])
125
129
  ))
126
130
  end
127
131
 
132
+ it "accepts multiline json values" do
133
+ expect(%[
134
+ <test var>
135
+ key ["a",
136
+ "b", "c",
137
+ "d"]
138
+ </test>
139
+ ]).to be_parsed_as(root(
140
+ e("test", 'var', {'key'=>"[\"a\",\"b\",\"c\",\"d\"]"})
141
+ ))
142
+ end
143
+
128
144
  [
129
145
  "**",
130
146
  "*.*",
@@ -148,7 +164,7 @@ describe Fluent::Config::V1Parser do
148
164
  <test >
149
165
  </test>
150
166
  ]).to be_parsed_as(root(
151
- e("test", nil)
167
+ e("test", '')
152
168
  ))
153
169
  end
154
170
 
@@ -47,4 +47,3 @@ shared_context 'config_helper' do
47
47
  end
48
48
  end
49
49
  end
50
-
@@ -172,6 +172,30 @@ describe Fluent::Config::LiteralParser do
172
172
  it { expect('[ "a" , "b" ]').to be_parsed_as_json(["a","b"]) }
173
173
  it { expect("[\n\"a\"\n,\n\"b\"\n]").to be_parsed_as_json(["a","b"]) }
174
174
  it { expect('["ab","cd"]').to be_parsed_as_json(["ab","cd"]) }
175
+ json_array_with_js_comment = <<EOA
176
+ [
177
+ "a", // this is a
178
+ "b", // this is b
179
+ "c" // this is c
180
+ ]
181
+ EOA
182
+ it { expect(json_array_with_js_comment).to be_parsed_as_json(["a","b","c"]) }
183
+ json_array_with_comment = <<EOA
184
+ [
185
+ "a", # this is a
186
+ "b", # this is b
187
+ "c" # this is c
188
+ ]
189
+ EOA
190
+ it { expect(json_array_with_comment).to be_parsed_as_json(["a","b","c"]) }
191
+ json_array_with_tailing_comma = <<EOA
192
+ [
193
+ "a", # this is a
194
+ "b", # this is b
195
+ "c", # this is c
196
+ ]
197
+ EOA
198
+ it { expect(json_array_with_tailing_comma).to be_parse_error }
175
199
  end
176
200
 
177
201
  describe 'map parsing' do
@@ -185,6 +209,14 @@ describe Fluent::Config::LiteralParser do
185
209
  it { expect('{"a":"b","c":"d"}').to be_parsed_as_json({"a"=>"b","c"=>"d"}) }
186
210
  it { expect('{ "a" : "b" , "c" : "d" }').to be_parsed_as_json({"a"=>"b","c"=>"d"}) }
187
211
  it { expect("{\n\"a\"\n:\n\"b\"\n,\n\"c\"\n:\n\"d\"\n}").to be_parsed_as_json({"a"=>"b","c"=>"d"}) }
212
+ json_hash_with_comment = <<EOH
213
+ {
214
+ "a": 1, # this is a
215
+ "b": 2, # this is b
216
+ "c": 3 # this is c
217
+ }
218
+ EOH
219
+ it { expect(json_hash_with_comment).to be_parsed_as_json({"a"=>1,"b"=>2,"c"=>3}) }
188
220
  end
189
221
  end
190
222
 
@@ -161,37 +161,51 @@ module ParserTest
161
161
 
162
162
  def setup
163
163
  @parser = TextParser::TEMPLATE_REGISTRY.lookup('syslog').call
164
+ @expected = {
165
+ 'host' => '192.168.0.1',
166
+ 'ident' => 'fluentd',
167
+ 'pid' => '11111',
168
+ 'message' => '[error] Syslog test'
169
+ }
164
170
  end
165
171
 
166
172
  def test_call
167
173
  @parser.configure({})
168
174
  @parser.call('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
169
175
  assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
170
- assert_equal({
171
- 'host' => '192.168.0.1',
172
- 'ident' => 'fluentd',
173
- 'pid' => '11111',
174
- 'message' => '[error] Syslog test'
175
- }, record)
176
+ assert_equal(@expected, record)
176
177
  }
177
178
  assert_equal(TextParser::SyslogParser::REGEXP, @parser.patterns['format'])
178
- assert_equal(TextParser::SyslogParser::TIME_FORMAT, @parser.patterns['time_format'])
179
+ assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
180
+ end
181
+
182
+ def test_call_with_time_format
183
+ @parser.configure('time_format' => '%b %d %M:%S:%H')
184
+ @parser.call('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
185
+ assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
186
+ assert_equal(@expected, record)
187
+ }
188
+ assert_equal('%b %d %M:%S:%H', @parser.patterns['time_format'])
179
189
  end
180
190
 
181
191
  def test_call_with_priority
182
192
  @parser.configure('with_priority' => true)
183
193
  @parser.call('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
184
194
  assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
185
- assert_equal({
186
- 'pri' => 6,
187
- 'host' => '192.168.0.1',
188
- 'ident' => 'fluentd',
189
- 'pid' => '11111',
190
- 'message' => '[error] Syslog test'
191
- }, record)
195
+ assert_equal(@expected.merge('pri' => 6), record)
192
196
  }
193
197
  assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI, @parser.patterns['format'])
194
- assert_equal(TextParser::SyslogParser::TIME_FORMAT, @parser.patterns['time_format'])
198
+ assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
199
+ end
200
+
201
+ def test_call_without_colon
202
+ @parser.configure({})
203
+ @parser.call('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
204
+ assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
205
+ assert_equal(@expected, record)
206
+ }
207
+ assert_equal(TextParser::SyslogParser::REGEXP, @parser.patterns['format'])
208
+ assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
195
209
  end
196
210
  end
197
211
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.10.51
4
+ version: 0.10.52
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-07-03 00:00:00.000000000 Z
11
+ date: 2014-07-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack
@@ -240,8 +240,8 @@ dependencies:
240
240
  - - ">="
241
241
  - !ruby/object:Gem::Version
242
242
  version: 0.3.0
243
- description: Fluentd is an event collector system. It is a generalized version of
244
- syslogd, which handles JSON objects for its log messages
243
+ description: Fluentd is an open source data collector designed to scale and simplify
244
+ log management. It can collect, process and ship many kinds of data in near real-time.
245
245
  email:
246
246
  - frsyuki@gmail.com
247
247
  executables: