fluentd 0.10.51 → 0.10.52
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/ChangeLog +11 -0
- data/fluentd.gemspec +1 -1
- data/lib/fluent/command/fluentd.rb +4 -5
- data/lib/fluent/config/element.rb +13 -0
- data/lib/fluent/config/literal_parser.rb +51 -9
- data/lib/fluent/config/v1_parser.rb +4 -0
- data/lib/fluent/engine.rb +28 -18
- data/lib/fluent/parser.rb +5 -5
- data/lib/fluent/plugin/in_tail.rb +1 -1
- data/lib/fluent/process.rb +6 -3
- data/lib/fluent/supervisor.rb +15 -16
- data/lib/fluent/version.rb +1 -1
- data/spec/config/config_parser_spec.rb +27 -11
- data/spec/config/helper.rb +0 -1
- data/spec/config/literal_parser_spec.rb +32 -0
- data/test/test_parser.rb +29 -15
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6871e2cbaea0e7068a4671988446b2cbbf8d5efb
|
4
|
+
data.tar.gz: 7ec21b7f21fa8787b7a577894849df0ea5acf22b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 924081ce1193ab72e08f05bc9ab758716d2aea3d988b1c2f735781eda2432acc78c5fea0bf40032aedc54f72067a5a39784e23cb9fa3a15ac4064792aff45dc7
|
7
|
+
data.tar.gz: 726dbda087652aa3c88f17f3a7f6dcd34d4dc7dab5a576efbf08883f10256f3e2ee3fa0bae375a69a3f769fd47deb36f22bb256f6defbd607f1bd6596649dc1c
|
data/.gitignore
CHANGED
data/ChangeLog
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
Release 0.10.52 - 2014/07/15
|
2
|
+
|
3
|
+
* in_tail: Fix typo of warning argument. lb to line
|
4
|
+
* config: Fix 'uninitialized constant Fluent::Config::V1Parser::URI' error
|
5
|
+
* config: Reject '@' prefix of parameter name in V1 configuration. @ is reserved prefix.
|
6
|
+
* config; Accept '#' comment in array and hash types
|
7
|
+
* parser: Add time_format option to SyslogParser
|
8
|
+
* parser: SyslogParser now makes ':' optional after pid
|
9
|
+
* process: Fix 'mutex can't lock' issue of DetachMultiProcessMixin on Ruby 2.x
|
10
|
+
* Add --without-source option to ignore <source> sections
|
11
|
+
|
1
12
|
Release 0.10.51 - 2014/07/04
|
2
13
|
|
3
14
|
* in_tail: Fix rotation handling when new file not found
|
data/fluentd.gemspec
CHANGED
@@ -6,7 +6,7 @@ Gem::Specification.new do |gem|
|
|
6
6
|
|
7
7
|
gem.authors = ["Sadayuki Furuhashi"]
|
8
8
|
gem.email = ["frsyuki@gmail.com"]
|
9
|
-
gem.description = %q{Fluentd is an
|
9
|
+
gem.description = %q{Fluentd is an open source data collector designed to scale and simplify log management. It can collect, process and ship many kinds of data in near real-time.}
|
10
10
|
gem.summary = %q{Fluentd event collector}
|
11
11
|
gem.homepage = "http://fluentd.org/"
|
12
12
|
|
@@ -17,10 +17,7 @@
|
|
17
17
|
#
|
18
18
|
|
19
19
|
require 'optparse'
|
20
|
-
require 'fluent/log'
|
21
|
-
require 'fluent/env'
|
22
20
|
require 'fluent/supervisor'
|
23
|
-
require 'fluent/version'
|
24
21
|
|
25
22
|
op = OptionParser.new
|
26
23
|
op.version = Fluent::VERSION
|
@@ -79,6 +76,10 @@ op.on('--suppress-repeated-stacktrace', "suppress repeated stacktrace", TrueClas
|
|
79
76
|
opts[:suppress_repeated_stacktrace] = b
|
80
77
|
}
|
81
78
|
|
79
|
+
op.on('--without-source', "invoke a fluentd without input plugins", TrueClass) {|b|
|
80
|
+
opts[:without_source] = b
|
81
|
+
}
|
82
|
+
|
82
83
|
op.on('--use-v1-config', "Use v1 configuration format", TrueClass) {|b|
|
83
84
|
opts[:use_v1_config] = b
|
84
85
|
}
|
@@ -156,6 +157,4 @@ if setup_path = opts[:setup_path]
|
|
156
157
|
exit 0
|
157
158
|
end
|
158
159
|
|
159
|
-
require 'fluent/supervisor'
|
160
160
|
Fluent::Supervisor.new(opts).start
|
161
|
-
|
@@ -22,6 +22,19 @@ module Fluent
|
|
22
22
|
e
|
23
23
|
end
|
24
24
|
|
25
|
+
def inspect
|
26
|
+
attrs = super
|
27
|
+
"name:#{@name}, arg:#{@arg}, " + attrs + ", " + @elements.inspect
|
28
|
+
end
|
29
|
+
|
30
|
+
def ==(o)
|
31
|
+
self.name == o.name && self.arg == o.arg &&
|
32
|
+
self.keys.size == o.keys.size &&
|
33
|
+
self.keys.reduce(true){|r, k| r && self[k] == o[k] } &&
|
34
|
+
self.elements.size == o.elements.size &&
|
35
|
+
[self.elements, o.elements].transpose.reduce(true){|r, e| r && e[0] == e[1] }
|
36
|
+
end
|
37
|
+
|
25
38
|
def +(o)
|
26
39
|
Element.new(@name.dup, @arg.dup, o.merge(self), @elements + o.elements, (@unused + o.unused).uniq)
|
27
40
|
end
|
@@ -139,19 +139,61 @@ module Fluent
|
|
139
139
|
|
140
140
|
def scan_json(is_array)
|
141
141
|
result = nil
|
142
|
+
# Yajl does not raise ParseError for imcomplete json string, like '[1', '{"h"', '{"h":' or '{"h1":1'
|
143
|
+
# This is the reason to use JSON module.
|
144
|
+
|
145
|
+
buffer = (is_array ? "[" : "{")
|
146
|
+
line_buffer = ""
|
142
147
|
|
143
|
-
y = Yajl::Parser.new(:allow_comments => true)
|
144
|
-
y.on_parse_complete = Proc.new { |obj| result = obj }
|
145
|
-
y << (is_array ? '[' : '{')
|
146
148
|
until result
|
147
|
-
|
148
|
-
|
149
|
-
|
149
|
+
char = getch
|
150
|
+
|
151
|
+
break if char.nil?
|
152
|
+
|
153
|
+
if char == "#"
|
154
|
+
# If this is out of json string literals, this object can be parsed correctly
|
155
|
+
# '{"foo":"bar", #' -> '{"foo":"bar"}' (to check)
|
156
|
+
parsed = nil
|
157
|
+
begin
|
158
|
+
parsed = JSON.parse(buffer + line_buffer.rstrip.sub(/,$/, '') + (is_array ? "]" : "}"))
|
159
|
+
rescue JSON::ParserError => e
|
160
|
+
# This '#' is in json string literals
|
161
|
+
end
|
162
|
+
|
163
|
+
if parsed
|
164
|
+
# ignore chars as comment before newline
|
165
|
+
while (char = getch) != "\n"
|
166
|
+
# ignore comment char
|
167
|
+
end
|
168
|
+
buffer << line_buffer + "\n"
|
169
|
+
line_buffer = ""
|
170
|
+
else
|
171
|
+
# '#' is a char in json string
|
172
|
+
line_buffer << char
|
173
|
+
end
|
174
|
+
|
175
|
+
next # This char '#' MUST NOT terminate json object.
|
176
|
+
end
|
177
|
+
|
178
|
+
if char == "\n"
|
179
|
+
buffer << line_buffer + "\n"
|
180
|
+
line_buffer = ""
|
181
|
+
next
|
182
|
+
end
|
183
|
+
|
184
|
+
line_buffer << char
|
185
|
+
begin
|
186
|
+
result = JSON.parse(buffer + line_buffer)
|
187
|
+
rescue JSON::ParserError => e
|
188
|
+
# Incomplete json string yet
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
unless result
|
193
|
+
parse_error! "got incomplete #{is_array ? 'array' : 'hash'} configuration"
|
150
194
|
end
|
151
195
|
|
152
|
-
|
153
|
-
rescue Yajl::ParseError => e
|
154
|
-
parse_error! "unexpected #{is_array ? 'array' : 'hash'} parse error"
|
196
|
+
JSON.dump(result)
|
155
197
|
end
|
156
198
|
end
|
157
199
|
end
|
@@ -103,6 +103,10 @@ module Fluent
|
|
103
103
|
if k == '@include'
|
104
104
|
parse_include(attrs, elems)
|
105
105
|
else
|
106
|
+
if k.start_with?('@')
|
107
|
+
parse_error! "'@' is reserved prefix. Don't use '@' in parameter name"
|
108
|
+
end
|
109
|
+
|
106
110
|
v = parse_literal
|
107
111
|
unless line_end
|
108
112
|
parse_error! "expected end of line"
|
data/lib/fluent/engine.rb
CHANGED
@@ -35,6 +35,7 @@ module Fluent
|
|
35
35
|
@next_emit_error_log_time = nil
|
36
36
|
|
37
37
|
@suppress_config_dump = false
|
38
|
+
@without_source = false
|
38
39
|
end
|
39
40
|
|
40
41
|
MATCH_CACHE_SIZE = 1024
|
@@ -43,13 +44,18 @@ module Fluent
|
|
43
44
|
|
44
45
|
attr_reader :matches, :sources
|
45
46
|
|
46
|
-
def init
|
47
|
+
def init(opts = {})
|
47
48
|
BasicSocket.do_not_reverse_lookup = true
|
48
49
|
Plugin.load_plugins
|
49
50
|
if defined?(Encoding)
|
50
51
|
Encoding.default_internal = 'ASCII-8BIT' if Encoding.respond_to?(:default_internal)
|
51
52
|
Encoding.default_external = 'ASCII-8BIT' if Encoding.respond_to?(:default_external)
|
52
53
|
end
|
54
|
+
|
55
|
+
suppress_interval(opts[:suppress_interval]) if opts[:suppress_interval]
|
56
|
+
@suppress_config_dump = opts[:suppress_config_dump] if opts[:suppress_config_dump]
|
57
|
+
@without_source = opts[:without_source] if opts[:without_source]
|
58
|
+
|
53
59
|
self
|
54
60
|
end
|
55
61
|
|
@@ -58,10 +64,6 @@ module Fluent
|
|
58
64
|
@next_emit_error_log_time = Time.now.to_i
|
59
65
|
end
|
60
66
|
|
61
|
-
def suppress_config_dump=(flag)
|
62
|
-
@suppress_config_dump = flag
|
63
|
-
end
|
64
|
-
|
65
67
|
def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
|
66
68
|
if fname =~ /\.rb$/
|
67
69
|
require 'fluent/config/dsl'
|
@@ -74,7 +76,11 @@ module Fluent
|
|
74
76
|
def run_configure(conf)
|
75
77
|
configure(conf)
|
76
78
|
conf.check_not_fetched { |key, e|
|
77
|
-
|
79
|
+
unless e.name == 'system'
|
80
|
+
unless @without_source && e.name == 'source'
|
81
|
+
$log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."
|
82
|
+
end
|
83
|
+
end
|
78
84
|
}
|
79
85
|
end
|
80
86
|
|
@@ -88,20 +94,24 @@ module Fluent
|
|
88
94
|
$log.info "using configuration file: #{conf.to_s.rstrip}"
|
89
95
|
end
|
90
96
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
97
|
+
if @without_source
|
98
|
+
$log.info "'--without-source' is applied. Ignore <source> sections"
|
99
|
+
else
|
100
|
+
conf.elements.select {|e|
|
101
|
+
e.name == 'source'
|
102
|
+
}.each {|e|
|
103
|
+
type = e['type']
|
104
|
+
unless type
|
105
|
+
raise ConfigError, "Missing 'type' parameter on <source> directive"
|
106
|
+
end
|
107
|
+
$log.info "adding source type=#{type.dump}"
|
99
108
|
|
100
|
-
|
101
|
-
|
109
|
+
input = Plugin.new_input(type)
|
110
|
+
input.configure(e)
|
102
111
|
|
103
|
-
|
104
|
-
|
112
|
+
@sources << input
|
113
|
+
}
|
114
|
+
end
|
105
115
|
|
106
116
|
conf.elements.select {|e|
|
107
117
|
e.name == 'match'
|
data/lib/fluent/parser.rb
CHANGED
@@ -471,11 +471,11 @@ module Fluent
|
|
471
471
|
include Configurable
|
472
472
|
|
473
473
|
# From existence TextParser pattern
|
474
|
-
REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/
|
474
|
+
REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
|
475
475
|
# From in_syslog default pattern
|
476
|
-
REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/
|
477
|
-
TIME_FORMAT = "%b %d %H:%M:%S"
|
476
|
+
REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
|
478
477
|
|
478
|
+
config_param :time_format, :string, :default => "%b %d %H:%M:%S"
|
479
479
|
config_param :with_priority, :bool, :default => false
|
480
480
|
|
481
481
|
attr_accessor :estimate_current_event
|
@@ -483,7 +483,6 @@ module Fluent
|
|
483
483
|
def initialize
|
484
484
|
super
|
485
485
|
@estimate_current_event = true
|
486
|
-
@time_parser = TextParser::TimeParser.new(TIME_FORMAT)
|
487
486
|
@mutex = Mutex.new
|
488
487
|
end
|
489
488
|
|
@@ -491,10 +490,11 @@ module Fluent
|
|
491
490
|
super
|
492
491
|
|
493
492
|
@regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
|
493
|
+
@time_parser = TextParser::TimeParser.new(@time_format)
|
494
494
|
end
|
495
495
|
|
496
496
|
def patterns
|
497
|
-
{'format' => @regexp, 'time_format' =>
|
497
|
+
{'format' => @regexp, 'time_format' => @time_format}
|
498
498
|
end
|
499
499
|
|
500
500
|
def call(text)
|
@@ -267,7 +267,7 @@ module Fluent
|
|
267
267
|
lb = line
|
268
268
|
else
|
269
269
|
if lb.nil?
|
270
|
-
log.warn "got incomplete line before first line from #{tail_watcher.path}: #{
|
270
|
+
log.warn "got incomplete line before first line from #{tail_watcher.path}: #{line.inspect}"
|
271
271
|
else
|
272
272
|
lb << line
|
273
273
|
end
|
data/lib/fluent/process.rb
CHANGED
@@ -378,9 +378,12 @@ module Fluent
|
|
378
378
|
def stop
|
379
379
|
return if @finished
|
380
380
|
@finished = true
|
381
|
-
|
382
|
-
|
383
|
-
|
381
|
+
# Creating new thread due to mutex can't lock in main thread during trap context
|
382
|
+
Thread.new {
|
383
|
+
@mutex.synchronize do
|
384
|
+
@cond.broadcast
|
385
|
+
end
|
386
|
+
}.run
|
384
387
|
end
|
385
388
|
|
386
389
|
def finished?
|
data/lib/fluent/supervisor.rb
CHANGED
@@ -16,9 +16,7 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
|
19
|
-
require 'fluent/
|
20
|
-
require 'fluent/log'
|
21
|
-
require 'fluent/config'
|
19
|
+
require 'fluent/load'
|
22
20
|
require 'etc'
|
23
21
|
|
24
22
|
module Fluent
|
@@ -61,7 +59,6 @@ module Fluent
|
|
61
59
|
end
|
62
60
|
|
63
61
|
$log = Fluent::Log.new(@io, @level, @opts)
|
64
|
-
|
65
62
|
$log.enable_color(false) if @path
|
66
63
|
$log.enable_debug if @level <= Fluent::Log::LEVEL_DEBUG
|
67
64
|
end
|
@@ -91,6 +88,7 @@ module Fluent
|
|
91
88
|
:chgroup => nil,
|
92
89
|
:suppress_interval => 0,
|
93
90
|
:suppress_repeated_stacktrace => false,
|
91
|
+
:without_source => false,
|
94
92
|
:use_v1_config => false,
|
95
93
|
}
|
96
94
|
end
|
@@ -112,6 +110,7 @@ module Fluent
|
|
112
110
|
@log_level = opt[:log_level]
|
113
111
|
@suppress_interval = opt[:suppress_interval]
|
114
112
|
@suppress_config_dump = opt[:suppress_config_dump]
|
113
|
+
@without_source = opt[:without_source]
|
115
114
|
|
116
115
|
log_opts = {:suppress_repeated_stacktrace => opt[:suppress_repeated_stacktrace]}
|
117
116
|
@log = LoggerInitializer.new(@log_path, @log_level, @chuser, @chgroup, log_opts)
|
@@ -120,7 +119,6 @@ module Fluent
|
|
120
119
|
end
|
121
120
|
|
122
121
|
def start
|
123
|
-
require 'fluent/load'
|
124
122
|
@log.init
|
125
123
|
|
126
124
|
dry_run if @dry_run
|
@@ -321,9 +319,8 @@ module Fluent
|
|
321
319
|
end
|
322
320
|
end
|
323
321
|
|
324
|
-
|
325
|
-
|
326
|
-
$log.info "reading config file", :path => @config_path if with_log
|
322
|
+
def read_config
|
323
|
+
$log.info "reading config file", :path => @config_path
|
327
324
|
@config_fname = File.basename(@config_path)
|
328
325
|
@config_basedir = File.dirname(@config_path)
|
329
326
|
@config_data = File.read(@config_path)
|
@@ -343,6 +340,7 @@ module Fluent
|
|
343
340
|
config_param :suppress_repeated_stacktrace, :bool, :default => nil
|
344
341
|
config_param :emit_error_log_interval, :time, :default => nil
|
345
342
|
config_param :suppress_config_dump, :bool, :default => nil
|
343
|
+
config_param :without_source, :bool, :default => nil
|
346
344
|
|
347
345
|
def initialize(conf)
|
348
346
|
super()
|
@@ -355,12 +353,16 @@ module Fluent
|
|
355
353
|
opt[:suppress_interval] = @emit_error_log_interval unless @emit_error_log_interval.nil?
|
356
354
|
opt[:suppress_config_dump] = @suppress_config_dump unless @suppress_config_dump.nil?
|
357
355
|
opt[:suppress_repeated_stacktrace] = @suppress_repeated_stacktrace unless @suppress_repeated_stacktrace.nil?
|
356
|
+
opt[:without_source] = @without_source unless @without_source.nil?
|
358
357
|
opt
|
359
358
|
end
|
360
359
|
end
|
361
360
|
|
362
361
|
def apply_system_config(opt)
|
363
|
-
|
362
|
+
# Create NULL file to avoid $log uninitialized problem before call @log.init
|
363
|
+
file = File.open(File::NULL)
|
364
|
+
$log = Fluent::Log.new(file, Log::LEVEL_INFO)
|
365
|
+
read_config
|
364
366
|
systems = Fluent::Config.parse(@config_data, @config_fname, @config_basedir, @use_v1_config).elements.select { |e|
|
365
367
|
e.name == 'system'
|
366
368
|
}
|
@@ -368,6 +370,8 @@ module Fluent
|
|
368
370
|
raise ConfigError, "<system> is duplicated. <system> should be only one" if systems.size > 1
|
369
371
|
|
370
372
|
opt.merge!(SystemConfig.new(systems.first).to_opt)
|
373
|
+
ensure
|
374
|
+
file.close
|
371
375
|
end
|
372
376
|
|
373
377
|
def run_configure
|
@@ -393,13 +397,8 @@ module Fluent
|
|
393
397
|
end
|
394
398
|
|
395
399
|
def init_engine
|
396
|
-
|
397
|
-
Fluent::Engine.init
|
398
|
-
if @suppress_interval
|
399
|
-
Fluent::Engine.suppress_interval(@suppress_interval)
|
400
|
-
end
|
401
|
-
|
402
|
-
Fluent::Engine.suppress_config_dump = @suppress_config_dump
|
400
|
+
init_opts = {:suppress_interval => @suppress_interval, :suppress_config_dump => @suppress_config_dump, :without_source => @without_source}
|
401
|
+
Fluent::Engine.init(init_opts)
|
403
402
|
|
404
403
|
@libs.each {|lib|
|
405
404
|
require lib
|
data/lib/fluent/version.rb
CHANGED
@@ -37,49 +37,53 @@ describe Fluent::Config::V1Parser do
|
|
37
37
|
expect(%[
|
38
38
|
k1 v1
|
39
39
|
k2 v2
|
40
|
-
]).to be_parsed_as("k1"=>"v1", "k2"=>"v2")
|
40
|
+
]).to be_parsed_as(e('ROOT', '', {"k1"=>"v1", "k2"=>"v2"}))
|
41
41
|
end
|
42
42
|
|
43
43
|
it "allows attribute without value" do
|
44
44
|
expect(%[
|
45
45
|
k1
|
46
46
|
k2 v2
|
47
|
-
]).to be_parsed_as("k1"=>"", "k2"=>"v2")
|
47
|
+
]).to be_parsed_as(e('ROOT', '', {"k1"=>"", "k2"=>"v2"}))
|
48
48
|
end
|
49
49
|
|
50
50
|
it "parses attribute key always string" do
|
51
|
-
expect("1 1").to be_parsed_as("1" => "1")
|
51
|
+
expect("1 1").to be_parsed_as(e('ROOT', '', {"1" => "1"}))
|
52
52
|
end
|
53
53
|
|
54
54
|
[
|
55
55
|
"_.%$!,",
|
56
|
-
"
|
56
|
+
"/=~-~@\`:?",
|
57
57
|
"()*{}.[]",
|
58
58
|
].each do |v|
|
59
59
|
it "parses a value with symbol #{v.inspect}" do
|
60
|
-
expect("k #{v}").to be_parsed_as("k" => v)
|
60
|
+
expect("k #{v}").to be_parsed_as(e('ROOT', '', {"k" => v}))
|
61
61
|
end
|
62
62
|
end
|
63
63
|
|
64
64
|
it "ignores spacing around value" do
|
65
|
-
expect(" k1 a ").to be_parsed_as("k1" => "a")
|
65
|
+
expect(" k1 a ").to be_parsed_as(e('ROOT', '', {"k1" => "a"}))
|
66
66
|
end
|
67
67
|
|
68
68
|
it "allows spaces in value" do
|
69
|
-
expect("k1 a b c").to be_parsed_as("k1" => "a b c")
|
69
|
+
expect("k1 a b c").to be_parsed_as(e('ROOT', '', {"k1" => "a b c"}))
|
70
70
|
end
|
71
71
|
|
72
72
|
it "ignores comments after value" do
|
73
|
-
expect(" k1 a#comment").to be_parsed_as("k1" => "a")
|
73
|
+
expect(" k1 a#comment").to be_parsed_as(e('ROOT', '', {"k1" => "a"}))
|
74
74
|
end
|
75
75
|
|
76
76
|
it "allows # in value if quoted" do
|
77
|
-
expect(' k1 "a#comment"').to be_parsed_as("k1" => "a#comment")
|
77
|
+
expect(' k1 "a#comment"').to be_parsed_as(e('ROOT', '', {"k1" => "a#comment"}))
|
78
78
|
end
|
79
79
|
|
80
80
|
it "rejects characters after quoted string" do
|
81
81
|
expect(' k1 "a" 1').to be_parse_error
|
82
82
|
end
|
83
|
+
|
84
|
+
it "rejects @ prefix in parameter name" do
|
85
|
+
expect(' @k v').to be_parse_error
|
86
|
+
end
|
83
87
|
end
|
84
88
|
|
85
89
|
describe 'element parsing' do
|
@@ -118,13 +122,25 @@ describe Fluent::Config::V1Parser do
|
|
118
122
|
</nested2>
|
119
123
|
</test>
|
120
124
|
]).to be_parsed_as(root(
|
121
|
-
e("test", 'var', {'key'=>'
|
125
|
+
e("test", 'var', {'key'=>'1'}, [
|
122
126
|
e('nested1'),
|
123
127
|
e('nested2')
|
124
128
|
])
|
125
129
|
))
|
126
130
|
end
|
127
131
|
|
132
|
+
it "accepts multiline json values" do
|
133
|
+
expect(%[
|
134
|
+
<test var>
|
135
|
+
key ["a",
|
136
|
+
"b", "c",
|
137
|
+
"d"]
|
138
|
+
</test>
|
139
|
+
]).to be_parsed_as(root(
|
140
|
+
e("test", 'var', {'key'=>"[\"a\",\"b\",\"c\",\"d\"]"})
|
141
|
+
))
|
142
|
+
end
|
143
|
+
|
128
144
|
[
|
129
145
|
"**",
|
130
146
|
"*.*",
|
@@ -148,7 +164,7 @@ describe Fluent::Config::V1Parser do
|
|
148
164
|
<test >
|
149
165
|
</test>
|
150
166
|
]).to be_parsed_as(root(
|
151
|
-
e("test",
|
167
|
+
e("test", '')
|
152
168
|
))
|
153
169
|
end
|
154
170
|
|
data/spec/config/helper.rb
CHANGED
@@ -172,6 +172,30 @@ describe Fluent::Config::LiteralParser do
|
|
172
172
|
it { expect('[ "a" , "b" ]').to be_parsed_as_json(["a","b"]) }
|
173
173
|
it { expect("[\n\"a\"\n,\n\"b\"\n]").to be_parsed_as_json(["a","b"]) }
|
174
174
|
it { expect('["ab","cd"]').to be_parsed_as_json(["ab","cd"]) }
|
175
|
+
json_array_with_js_comment = <<EOA
|
176
|
+
[
|
177
|
+
"a", // this is a
|
178
|
+
"b", // this is b
|
179
|
+
"c" // this is c
|
180
|
+
]
|
181
|
+
EOA
|
182
|
+
it { expect(json_array_with_js_comment).to be_parsed_as_json(["a","b","c"]) }
|
183
|
+
json_array_with_comment = <<EOA
|
184
|
+
[
|
185
|
+
"a", # this is a
|
186
|
+
"b", # this is b
|
187
|
+
"c" # this is c
|
188
|
+
]
|
189
|
+
EOA
|
190
|
+
it { expect(json_array_with_comment).to be_parsed_as_json(["a","b","c"]) }
|
191
|
+
json_array_with_tailing_comma = <<EOA
|
192
|
+
[
|
193
|
+
"a", # this is a
|
194
|
+
"b", # this is b
|
195
|
+
"c", # this is c
|
196
|
+
]
|
197
|
+
EOA
|
198
|
+
it { expect(json_array_with_tailing_comma).to be_parse_error }
|
175
199
|
end
|
176
200
|
|
177
201
|
describe 'map parsing' do
|
@@ -185,6 +209,14 @@ describe Fluent::Config::LiteralParser do
|
|
185
209
|
it { expect('{"a":"b","c":"d"}').to be_parsed_as_json({"a"=>"b","c"=>"d"}) }
|
186
210
|
it { expect('{ "a" : "b" , "c" : "d" }').to be_parsed_as_json({"a"=>"b","c"=>"d"}) }
|
187
211
|
it { expect("{\n\"a\"\n:\n\"b\"\n,\n\"c\"\n:\n\"d\"\n}").to be_parsed_as_json({"a"=>"b","c"=>"d"}) }
|
212
|
+
json_hash_with_comment = <<EOH
|
213
|
+
{
|
214
|
+
"a": 1, # this is a
|
215
|
+
"b": 2, # this is b
|
216
|
+
"c": 3 # this is c
|
217
|
+
}
|
218
|
+
EOH
|
219
|
+
it { expect(json_hash_with_comment).to be_parsed_as_json({"a"=>1,"b"=>2,"c"=>3}) }
|
188
220
|
end
|
189
221
|
end
|
190
222
|
|
data/test/test_parser.rb
CHANGED
@@ -161,37 +161,51 @@ module ParserTest
|
|
161
161
|
|
162
162
|
def setup
|
163
163
|
@parser = TextParser::TEMPLATE_REGISTRY.lookup('syslog').call
|
164
|
+
@expected = {
|
165
|
+
'host' => '192.168.0.1',
|
166
|
+
'ident' => 'fluentd',
|
167
|
+
'pid' => '11111',
|
168
|
+
'message' => '[error] Syslog test'
|
169
|
+
}
|
164
170
|
end
|
165
171
|
|
166
172
|
def test_call
|
167
173
|
@parser.configure({})
|
168
174
|
@parser.call('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
169
175
|
assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
|
170
|
-
assert_equal(
|
171
|
-
'host' => '192.168.0.1',
|
172
|
-
'ident' => 'fluentd',
|
173
|
-
'pid' => '11111',
|
174
|
-
'message' => '[error] Syslog test'
|
175
|
-
}, record)
|
176
|
+
assert_equal(@expected, record)
|
176
177
|
}
|
177
178
|
assert_equal(TextParser::SyslogParser::REGEXP, @parser.patterns['format'])
|
178
|
-
assert_equal(
|
179
|
+
assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
|
180
|
+
end
|
181
|
+
|
182
|
+
def test_call_with_time_format
|
183
|
+
@parser.configure('time_format' => '%b %d %M:%S:%H')
|
184
|
+
@parser.call('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
185
|
+
assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
|
186
|
+
assert_equal(@expected, record)
|
187
|
+
}
|
188
|
+
assert_equal('%b %d %M:%S:%H', @parser.patterns['time_format'])
|
179
189
|
end
|
180
190
|
|
181
191
|
def test_call_with_priority
|
182
192
|
@parser.configure('with_priority' => true)
|
183
193
|
@parser.call('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
|
184
194
|
assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
|
185
|
-
assert_equal(
|
186
|
-
'pri' => 6,
|
187
|
-
'host' => '192.168.0.1',
|
188
|
-
'ident' => 'fluentd',
|
189
|
-
'pid' => '11111',
|
190
|
-
'message' => '[error] Syslog test'
|
191
|
-
}, record)
|
195
|
+
assert_equal(@expected.merge('pri' => 6), record)
|
192
196
|
}
|
193
197
|
assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI, @parser.patterns['format'])
|
194
|
-
assert_equal(
|
198
|
+
assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
|
199
|
+
end
|
200
|
+
|
201
|
+
def test_call_without_colon
|
202
|
+
@parser.configure({})
|
203
|
+
@parser.call('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
|
204
|
+
assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
|
205
|
+
assert_equal(@expected, record)
|
206
|
+
}
|
207
|
+
assert_equal(TextParser::SyslogParser::REGEXP, @parser.patterns['format'])
|
208
|
+
assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
|
195
209
|
end
|
196
210
|
end
|
197
211
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluentd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.10.
|
4
|
+
version: 0.10.52
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-07-
|
11
|
+
date: 2014-07-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|
@@ -240,8 +240,8 @@ dependencies:
|
|
240
240
|
- - ">="
|
241
241
|
- !ruby/object:Gem::Version
|
242
242
|
version: 0.3.0
|
243
|
-
description: Fluentd is an
|
244
|
-
|
243
|
+
description: Fluentd is an open source data collector designed to scale and simplify
|
244
|
+
log management. It can collect, process and ship many kinds of data in near real-time.
|
245
245
|
email:
|
246
246
|
- frsyuki@gmail.com
|
247
247
|
executables:
|