fluentd 1.8.0.rc1 → 1.8.0.rc2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/fluentd.gemspec +1 -1
- data/lib/fluent/command/fluentd.rb +5 -5
- data/lib/fluent/config/configure_proxy.rb +4 -4
- data/lib/fluent/engine.rb +12 -86
- data/lib/fluent/fluent_log_event_router.rb +137 -0
- data/lib/fluent/plugin/buffer/chunk.rb +1 -1
- data/lib/fluent/plugin/formatter_csv.rb +2 -2
- data/lib/fluent/plugin_helper/child_process.rb +3 -3
- data/lib/fluent/plugin_helper/extract.rb +1 -1
- data/lib/fluent/plugin_helper/inject.rb +1 -1
- data/lib/fluent/plugin_helper/server.rb +2 -2
- data/lib/fluent/supervisor.rb +10 -3
- data/lib/fluent/test/driver/base_owned.rb +15 -2
- data/lib/fluent/time.rb +3 -3
- data/lib/fluent/version.rb +1 -1
- data/test/counter/test_client.rb +8 -4
- data/test/test_fluent_log_event_router.rb +99 -0
- metadata +15 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0c70380cd4a9f0f78150f7332a5e484a9f9523786fabae5c1bda5761cf21617e
|
4
|
+
data.tar.gz: 65e167aa06b31cc5aebce1e8d1bbb61d80e8e1ae9e1690dd58dc799f51683503
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f19fccb4e6d84dec936215ef8388d650a5063f30e0456c9aae5f47d2d0f1b73b86f01c95fc7e3d30e939b3d72e4224fcc5c8c1d69f59a0db7fc64b239cd9f8a2
|
7
|
+
data.tar.gz: dd4d743c7d1d73199c35458ba3cb09e1dd6dda5a49b2f30575d6696f573909b642b1e860687a2e6bec8575db14869cfa29039cb472eb75b00abfb944bf368e0a
|
data/CHANGELOG.md
CHANGED
@@ -11,17 +11,26 @@
|
|
11
11
|
https://github.com/fluent/fluentd/pull/2643
|
12
12
|
* record_accessor helper: Make code simple and bit faster
|
13
13
|
https://github.com/fluent/fluentd/pull/2660
|
14
|
+
* Relax tzinfo dependency to accept v1
|
15
|
+
https://github.com/fluent/fluentd/pull/2673
|
14
16
|
* clean up internal classes / code
|
15
17
|
https://github.com/fluent/fluentd/pull/2647
|
16
18
|
https://github.com/fluent/fluentd/pull/2648
|
17
19
|
https://github.com/fluent/fluentd/pull/2653
|
18
20
|
https://github.com/fluent/fluentd/pull/2654
|
19
21
|
https://github.com/fluent/fluentd/pull/2657
|
22
|
+
https://github.com/fluent/fluentd/pull/2667
|
20
23
|
|
21
24
|
### Bug fixes
|
22
25
|
|
23
26
|
* suppress warning
|
24
27
|
https://github.com/fluent/fluentd/pull/2652
|
28
|
+
* suppress keyword argument warning for ruby2.7
|
29
|
+
https://github.com/fluent/fluentd/pull/2664
|
30
|
+
* RPC: Fix debug log text
|
31
|
+
https://github.com/fluent/fluentd/pull/2666
|
32
|
+
* time: Properly show class names in error message
|
33
|
+
https://github.com/fluent/fluentd/pull/2671
|
25
34
|
|
26
35
|
|
27
36
|
# v1.7
|
data/fluentd.gemspec
CHANGED
@@ -24,7 +24,7 @@ Gem::Specification.new do |gem|
|
|
24
24
|
gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
|
25
25
|
gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
|
26
26
|
gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
|
27
|
-
gem.add_runtime_dependency("tzinfo", ["
|
27
|
+
gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
|
28
28
|
gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
|
29
29
|
gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
|
30
30
|
gem.add_runtime_dependency("dig_rb", ["~> 1.0.0"])
|
@@ -162,7 +162,7 @@ op.on('--conf-encoding ENCODING', "specify configuration file encoding") { |s|
|
|
162
162
|
if Fluent.windows?
|
163
163
|
require 'windows/library'
|
164
164
|
include Windows::Library
|
165
|
-
|
165
|
+
|
166
166
|
opts.merge!(
|
167
167
|
:winsvc_name => 'fluentdwinsvc',
|
168
168
|
:winsvc_display_name => 'Fluentd Windows Service',
|
@@ -184,15 +184,15 @@ if Fluent.windows?
|
|
184
184
|
op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option parameters for Windows Service. (Windows only)") {|s|
|
185
185
|
opts[:fluentdopt] = s
|
186
186
|
}
|
187
|
-
|
187
|
+
|
188
188
|
op.on('--winsvc-name NAME', "The Windows Service name to run as (Windows only)") {|s|
|
189
189
|
opts[:winsvc_name] = s
|
190
190
|
}
|
191
|
-
|
191
|
+
|
192
192
|
op.on('--winsvc-display-name DISPLAY_NAME', "The Windows Service display name (Windows only)") {|s|
|
193
193
|
opts[:winsvc_display_name] = s
|
194
194
|
}
|
195
|
-
|
195
|
+
|
196
196
|
op.on('--winsvc-desc DESC', "The Windows Service description (Windows only)") {|s|
|
197
197
|
opts[:winsvc_desc] = s
|
198
198
|
}
|
@@ -268,7 +268,7 @@ if winsvcinstmode = opts[:regwinsvc]
|
|
268
268
|
start_service = true
|
269
269
|
end
|
270
270
|
|
271
|
-
|
271
|
+
|
272
272
|
Service.create(
|
273
273
|
service_name: opts[:winsvc_name],
|
274
274
|
host: nil,
|
@@ -117,9 +117,9 @@ module Fluent
|
|
117
117
|
|
118
118
|
merged = if self.root?
|
119
119
|
options[:root] = true
|
120
|
-
self.class.new(other.name, options)
|
120
|
+
self.class.new(other.name, **options)
|
121
121
|
else
|
122
|
-
self.class.new(@name, options)
|
122
|
+
self.class.new(@name, **options)
|
123
123
|
end
|
124
124
|
|
125
125
|
# configured_in MUST be kept
|
@@ -172,9 +172,9 @@ module Fluent
|
|
172
172
|
|
173
173
|
merged = if self.root?
|
174
174
|
options[:root] = true
|
175
|
-
self.class.new(other.name, options)
|
175
|
+
self.class.new(other.name, **options)
|
176
176
|
else
|
177
|
-
self.class.new(@name, options)
|
177
|
+
self.class.new(@name, **options)
|
178
178
|
end
|
179
179
|
|
180
180
|
merged.configured_in_section = self.configured_in_section || other.configured_in_section
|
data/lib/fluent/engine.rb
CHANGED
@@ -22,6 +22,7 @@ require 'fluent/root_agent'
|
|
22
22
|
require 'fluent/time'
|
23
23
|
require 'fluent/system_config'
|
24
24
|
require 'fluent/plugin'
|
25
|
+
require 'fluent/fluent_log_event_router'
|
25
26
|
|
26
27
|
module Fluent
|
27
28
|
class EngineClass
|
@@ -30,19 +31,14 @@ module Fluent
|
|
30
31
|
|
31
32
|
def initialize
|
32
33
|
@root_agent = nil
|
33
|
-
@default_loop = nil
|
34
34
|
@engine_stopped = false
|
35
35
|
@_worker_id = nil
|
36
36
|
|
37
|
-
@log_event_router = nil
|
38
|
-
@log_emit_thread = nil
|
39
|
-
@log_event_loop_stop = false
|
40
|
-
@log_event_loop_graceful_stop = false
|
41
|
-
@log_event_queue = []
|
42
37
|
@log_event_verbose = false
|
43
|
-
|
44
38
|
@suppress_config_dump = false
|
39
|
+
@without_source = false
|
45
40
|
|
41
|
+
@fluent_log_event_router = nil
|
46
42
|
@system_config = SystemConfig.new
|
47
43
|
|
48
44
|
@dry_run_mode = false
|
@@ -50,13 +46,7 @@ module Fluent
|
|
50
46
|
|
51
47
|
MAINLOOP_SLEEP_INTERVAL = 0.3
|
52
48
|
|
53
|
-
|
54
|
-
LOG_EMIT_INTERVAL = 0.1
|
55
|
-
|
56
|
-
attr_reader :root_agent
|
57
|
-
attr_reader :matches, :sources
|
58
|
-
attr_reader :system_config
|
59
|
-
|
49
|
+
attr_reader :root_agent, :system_config
|
60
50
|
attr_accessor :dry_run_mode
|
61
51
|
|
62
52
|
def init(system_config)
|
@@ -116,43 +106,14 @@ module Fluent
|
|
116
106
|
end
|
117
107
|
|
118
108
|
def configure(conf)
|
119
|
-
# plugins / configuration dumps
|
120
|
-
Gem::Specification.find_all.select{|x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/}.each do |spec|
|
121
|
-
$log.info :worker0, "gem '#{spec.name}' version '#{spec.version}'"
|
122
|
-
end
|
123
|
-
|
124
109
|
@root_agent.configure(conf)
|
125
110
|
|
126
|
-
|
127
|
-
log_event_agent = @root_agent.find_label(Fluent::Log::LOG_EVENT_LABEL)
|
128
|
-
log_event_router = log_event_agent.event_router
|
129
|
-
|
130
|
-
# suppress mismatched tags only for <label @FLUENT_LOG> label.
|
131
|
-
# it's not suppressed in default event router for non-log-event events
|
132
|
-
log_event_router.suppress_missing_match!
|
111
|
+
@fleunt_log_event_router = FluentLogEventRouter.build(@root_agent)
|
133
112
|
|
134
|
-
|
135
|
-
|
136
|
-
unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
|
137
|
-
unless unmatched_tags.empty?
|
138
|
-
$log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
|
139
|
-
end
|
140
|
-
rescue ArgumentError # ArgumentError "#{label_name} label not found"
|
141
|
-
# use default event router if <label @FLUENT_LOG> is missing in configuration
|
142
|
-
log_event_router = @root_agent.event_router
|
143
|
-
|
144
|
-
if Fluent::Log.event_tags.any?{|t| log_event_router.match?(t) }
|
145
|
-
@log_event_router = log_event_router
|
146
|
-
|
147
|
-
unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
|
148
|
-
unless unmatched_tags.empty?
|
149
|
-
$log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
|
150
|
-
end
|
151
|
-
end
|
113
|
+
if @fleunt_log_event_router.emittable?
|
114
|
+
$log.enable_event(true)
|
152
115
|
end
|
153
116
|
|
154
|
-
$log.enable_event(true) if @log_event_router
|
155
|
-
|
156
117
|
unless @suppress_config_dump
|
157
118
|
$log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
|
158
119
|
end
|
@@ -183,39 +144,12 @@ module Fluent
|
|
183
144
|
Fluent::EventTime.now
|
184
145
|
end
|
185
146
|
|
186
|
-
def log_event_loop
|
187
|
-
$log.disable_events(Thread.current)
|
188
|
-
|
189
|
-
while sleep(LOG_EMIT_INTERVAL)
|
190
|
-
break if @log_event_loop_stop
|
191
|
-
break if @log_event_loop_graceful_stop && @log_event_queue.empty?
|
192
|
-
next if @log_event_queue.empty?
|
193
|
-
|
194
|
-
# NOTE: thead-safe of slice! depends on GVL
|
195
|
-
events = @log_event_queue.slice!(0..-1)
|
196
|
-
next if events.empty?
|
197
|
-
|
198
|
-
events.each {|tag,time,record|
|
199
|
-
begin
|
200
|
-
@log_event_router.emit(tag, time, record)
|
201
|
-
rescue => e
|
202
|
-
# This $log.error doesn't emit log events, because of `$log.disable_events(Thread.current)` above
|
203
|
-
$log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
|
204
|
-
end
|
205
|
-
}
|
206
|
-
end
|
207
|
-
end
|
208
|
-
|
209
147
|
def run
|
210
148
|
begin
|
211
149
|
$log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
|
212
150
|
start
|
213
151
|
|
214
|
-
|
215
|
-
$log.enable_event(true)
|
216
|
-
@log_emit_thread = Thread.new(&method(:log_event_loop))
|
217
|
-
@log_emit_thread.abort_on_exception = true
|
218
|
-
end
|
152
|
+
@fleunt_log_event_router.start
|
219
153
|
|
220
154
|
$log.info "fluentd worker is now running", worker: worker_id
|
221
155
|
sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
|
@@ -229,19 +163,12 @@ module Fluent
|
|
229
163
|
|
230
164
|
unless @log_event_verbose
|
231
165
|
$log.enable_event(false)
|
232
|
-
|
233
|
-
# to make sure to emit all log events into router, before shutting down
|
234
|
-
@log_event_loop_graceful_stop = true
|
235
|
-
@log_emit_thread.join
|
236
|
-
@log_emit_thread = nil
|
237
|
-
end
|
166
|
+
@fleunt_log_event_router.graceful_stop
|
238
167
|
end
|
239
168
|
$log.info "shutting down fluentd worker", worker: worker_id
|
240
169
|
shutdown
|
241
|
-
|
242
|
-
|
243
|
-
@log_emit_thread.join
|
244
|
-
end
|
170
|
+
|
171
|
+
@fleunt_log_event_router.stop
|
245
172
|
end
|
246
173
|
|
247
174
|
def stop
|
@@ -250,8 +177,7 @@ module Fluent
|
|
250
177
|
end
|
251
178
|
|
252
179
|
def push_log_event(tag, time, record)
|
253
|
-
|
254
|
-
@log_event_queue.push([tag, time, record])
|
180
|
+
@fleunt_log_event_router.emit_event([tag, time, record])
|
255
181
|
end
|
256
182
|
|
257
183
|
def worker_id
|
@@ -0,0 +1,137 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/log'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
# DO NOT write any logic here
|
21
|
+
class NullFluentLogEventRouter
|
22
|
+
def start; end
|
23
|
+
|
24
|
+
def stop; end
|
25
|
+
|
26
|
+
def graceful_stop; end
|
27
|
+
|
28
|
+
def emit_event(_event); end
|
29
|
+
|
30
|
+
def emittable?
|
31
|
+
self.class != NullFluentLogEventRouter
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
# This class is for handling fluentd's inner log
|
36
|
+
# e.g. <label @FLUNT_LOG> section and <match fluent.**> section
|
37
|
+
class FluentLogEventRouter < NullFluentLogEventRouter
|
38
|
+
# @param root_agent [Fluent::RootAgent]
|
39
|
+
def self.build(root_agent)
|
40
|
+
log_event_router = nil
|
41
|
+
|
42
|
+
begin
|
43
|
+
log_event_agent = root_agent.find_label(Fluent::Log::LOG_EVENT_LABEL)
|
44
|
+
log_event_router = log_event_agent.event_router
|
45
|
+
|
46
|
+
# suppress mismatched tags only for <label @FLUENT_LOG> label.
|
47
|
+
# it's not suppressed in default event router for non-log-event events
|
48
|
+
log_event_router.suppress_missing_match!
|
49
|
+
|
50
|
+
log_event_router = log_event_router
|
51
|
+
|
52
|
+
unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
|
53
|
+
unless unmatched_tags.empty?
|
54
|
+
$log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
|
55
|
+
end
|
56
|
+
|
57
|
+
rescue ArgumentError # ArgumentError "#{label_name} label not found"
|
58
|
+
# use default event router if <label @FLUENT_LOG> is missing in configuration
|
59
|
+
root_log_event_router = root_agent.event_router
|
60
|
+
|
61
|
+
if Fluent::Log.event_tags.any? { |t| root_log_event_router.match?(t) }
|
62
|
+
log_event_router = root_log_event_router
|
63
|
+
|
64
|
+
unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
|
65
|
+
unless unmatched_tags.empty?
|
66
|
+
$log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
if log_event_router
|
72
|
+
FluentLogEventRouter.new(log_event_router)
|
73
|
+
else
|
74
|
+
$log.debug('No fluent logger for internal event')
|
75
|
+
NullFluentLogEventRouter.new
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
STOP = :stop
|
80
|
+
GRACEFUL_STOP = :graceful_stop
|
81
|
+
|
82
|
+
# @param event_router [Fluent::EventRouter]
|
83
|
+
def initialize(event_router)
|
84
|
+
@event_router = event_router
|
85
|
+
@thread = nil
|
86
|
+
@graceful_stop = false
|
87
|
+
@event_queue = Queue.new
|
88
|
+
end
|
89
|
+
|
90
|
+
def start
|
91
|
+
@thread = Thread.new do
|
92
|
+
$log.disable_events(Thread.current)
|
93
|
+
|
94
|
+
loop do
|
95
|
+
event = @event_queue.pop
|
96
|
+
|
97
|
+
case event
|
98
|
+
when GRACEFUL_STOP
|
99
|
+
@graceful_stop = true
|
100
|
+
when STOP
|
101
|
+
break
|
102
|
+
else
|
103
|
+
begin
|
104
|
+
tag, time, record = event
|
105
|
+
@event_router.emit(tag, time, record)
|
106
|
+
rescue => e
|
107
|
+
# This $log.error doesn't emit log events, because of `$log.disable_events(Thread.current)` above
|
108
|
+
$log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
if @graceful_stop && @event_queue.empty?
|
113
|
+
break
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
@thread.abort_on_exception = true
|
119
|
+
end
|
120
|
+
|
121
|
+
def stop
|
122
|
+
@event_queue.push(STOP)
|
123
|
+
# there is no problem calling Thread#join multiple times.
|
124
|
+
@thread && @thread.join
|
125
|
+
end
|
126
|
+
|
127
|
+
def graceful_stop
|
128
|
+
# to make sure to emit all log events into router, before shutting down
|
129
|
+
@event_queue.push(GRACEFUL_STOP)
|
130
|
+
@thread && @thread.join
|
131
|
+
end
|
132
|
+
|
133
|
+
def emit_event(event)
|
134
|
+
@event_queue.push(event)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
@@ -56,7 +56,7 @@ module Fluent
|
|
56
56
|
end
|
57
57
|
|
58
58
|
def format(tag, time, record)
|
59
|
-
csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT),
|
59
|
+
csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
|
60
60
|
line = (csv << record).string.dup
|
61
61
|
# Need manual cleanup because CSV writer doesn't provide such method.
|
62
62
|
csv.rewind
|
@@ -65,7 +65,7 @@ module Fluent
|
|
65
65
|
end
|
66
66
|
|
67
67
|
def format_with_nested_fields(tag, time, record)
|
68
|
-
csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT),
|
68
|
+
csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
|
69
69
|
values = @accessors.map { |a| a.call(record) }
|
70
70
|
line = (csv << values).string.dup
|
71
71
|
# Need manual cleanup because CSV writer doesn't provide such method.
|
@@ -267,19 +267,19 @@ module Fluent
|
|
267
267
|
end
|
268
268
|
|
269
269
|
if mode.include?(:write)
|
270
|
-
writeio.set_encoding(external_encoding, internal_encoding, encoding_options)
|
270
|
+
writeio.set_encoding(external_encoding, internal_encoding, **encoding_options)
|
271
271
|
writeio_in_use = true
|
272
272
|
else
|
273
273
|
writeio.reopen(IO::NULL) if writeio
|
274
274
|
end
|
275
275
|
if mode.include?(:read) || mode.include?(:read_with_stderr)
|
276
|
-
readio.set_encoding(external_encoding, internal_encoding, encoding_options)
|
276
|
+
readio.set_encoding(external_encoding, internal_encoding, **encoding_options)
|
277
277
|
readio_in_use = true
|
278
278
|
else
|
279
279
|
readio.reopen(IO::NULL) if readio
|
280
280
|
end
|
281
281
|
if mode.include?(:stderr)
|
282
|
-
stderrio.set_encoding(external_encoding, internal_encoding, encoding_options)
|
282
|
+
stderrio.set_encoding(external_encoding, internal_encoding, **encoding_options)
|
283
283
|
stderrio_in_use = true
|
284
284
|
else
|
285
285
|
stderrio.reopen(IO::NULL) if stderrio && stderr == :discard
|
@@ -690,9 +690,9 @@ module Fluent
|
|
690
690
|
end
|
691
691
|
|
692
692
|
if RUBY_VERSION.to_f >= 2.3
|
693
|
-
NONBLOCK_ARG = {exception: false}
|
693
|
+
NONBLOCK_ARG = { exception: false }
|
694
694
|
def try_handshake
|
695
|
-
@_handler_socket.accept_nonblock(NONBLOCK_ARG)
|
695
|
+
@_handler_socket.accept_nonblock(**NONBLOCK_ARG)
|
696
696
|
end
|
697
697
|
else
|
698
698
|
def try_handshake
|
data/lib/fluent/supervisor.rb
CHANGED
@@ -121,7 +121,7 @@ module Fluent
|
|
121
121
|
}
|
122
122
|
|
123
123
|
@rpc_server.mount_proc('/api/config.getDump') { |req, res|
|
124
|
-
$log.debug "fluentd RPC got /api/config.
|
124
|
+
$log.debug "fluentd RPC got /api/config.getDump request"
|
125
125
|
$log.info "get dump in-memory config via HTTP"
|
126
126
|
res.body = supervisor_get_dump_config_handler
|
127
127
|
[nil, nil, res]
|
@@ -586,7 +586,7 @@ module Fluent
|
|
586
586
|
Fluent::Engine.dry_run_mode = true
|
587
587
|
change_privilege
|
588
588
|
MessagePackFactory.init
|
589
|
-
init_engine
|
589
|
+
init_engine(supervisor: true)
|
590
590
|
run_configure
|
591
591
|
rescue Fluent::ConfigError => e
|
592
592
|
$log.error "config error", file: @config_path, error: e
|
@@ -794,7 +794,7 @@ module Fluent
|
|
794
794
|
ServerEngine::Privilege.change(@chuser, @chgroup)
|
795
795
|
end
|
796
796
|
|
797
|
-
def init_engine
|
797
|
+
def init_engine(supervisor: false)
|
798
798
|
Fluent::Engine.init(@system_config)
|
799
799
|
|
800
800
|
@libs.each {|lib|
|
@@ -807,6 +807,13 @@ module Fluent
|
|
807
807
|
Fluent::Engine.add_plugin_dir(dir)
|
808
808
|
end
|
809
809
|
}
|
810
|
+
|
811
|
+
if supervisor
|
812
|
+
# plugins / configuration dumps
|
813
|
+
Gem::Specification.find_all.select { |x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/ }.each do |spec|
|
814
|
+
$log.info("gem '#{spec.name}' version '#{spec.version}'")
|
815
|
+
end
|
816
|
+
end
|
810
817
|
end
|
811
818
|
|
812
819
|
def run_configure
|
@@ -53,13 +53,26 @@ module Fluent
|
|
53
53
|
@section_name = ''
|
54
54
|
end
|
55
55
|
|
56
|
-
def configure(conf
|
56
|
+
def configure(conf)
|
57
57
|
if conf.is_a?(Fluent::Config::Element)
|
58
58
|
@config = conf
|
59
59
|
elsif conf.is_a?(Hash)
|
60
60
|
@config = Fluent::Config::Element.new(@section_name, "", Hash[conf.map{|k,v| [k.to_s, v]}], [])
|
61
61
|
else
|
62
|
-
@config = Fluent::Config.parse(conf, @section_name, "", syntax:
|
62
|
+
@config = Fluent::Config.parse(conf, @section_name, "", syntax: :v1)
|
63
|
+
end
|
64
|
+
@instance.configure(@config)
|
65
|
+
self
|
66
|
+
end
|
67
|
+
|
68
|
+
# this is special method for v0 and should be deleted
|
69
|
+
def configure_v0(conf)
|
70
|
+
if conf.is_a?(Fluent::Config::Element)
|
71
|
+
@config = conf
|
72
|
+
elsif conf.is_a?(Hash)
|
73
|
+
@config = Fluent::Config::Element.new(@section_name, "", Hash[conf.map{|k,v| [k.to_s, v]}], [])
|
74
|
+
else
|
75
|
+
@config = Fluent::Config.parse(conf, @section_name, "", syntax: :v0)
|
63
76
|
end
|
64
77
|
@instance.configure(@config)
|
65
78
|
self
|
data/lib/fluent/time.rb
CHANGED
@@ -147,7 +147,7 @@ module Fluent
|
|
147
147
|
module TimeParameters
|
148
148
|
include Fluent::Configurable
|
149
149
|
TIME_FULL_PARAMETERS.each do |name, type, opts|
|
150
|
-
config_param
|
150
|
+
config_param(name, type, **opts)
|
151
151
|
end
|
152
152
|
|
153
153
|
def configure(conf)
|
@@ -292,7 +292,7 @@ module Fluent
|
|
292
292
|
|
293
293
|
def parse_unixtime(value)
|
294
294
|
unless value.is_a?(String) || value.is_a?(Numeric)
|
295
|
-
raise TimeParseError, "value must be a string or a number: #{value}(value.class)"
|
295
|
+
raise TimeParseError, "value must be a string or a number: #{value}(#{value.class})"
|
296
296
|
end
|
297
297
|
|
298
298
|
if @cache1_key == value
|
@@ -323,7 +323,7 @@ module Fluent
|
|
323
323
|
## parse_by_to_r (msec): 28.232856 sec
|
324
324
|
def parse_float(value)
|
325
325
|
unless value.is_a?(String) || value.is_a?(Numeric)
|
326
|
-
raise TimeParseError, "value must be a string or a number: #{value}(value.class)"
|
326
|
+
raise TimeParseError, "value must be a string or a number: #{value}(#{value.class})"
|
327
327
|
end
|
328
328
|
|
329
329
|
if @cache1_key == value
|
data/lib/fluent/version.rb
CHANGED
data/test/counter/test_client.rb
CHANGED
@@ -128,7 +128,8 @@ class CounterClientTest < ::Test::Unit::TestCase
|
|
128
128
|
test 'raise an error when @scope is nil' do
|
129
129
|
@client.instance_variable_set(:@scope, nil)
|
130
130
|
assert_raise 'Call `establish` method to get a `scope` before calling this method' do
|
131
|
-
|
131
|
+
params = { name: 'key1', reset_interval: 10 }
|
132
|
+
@client.init(params).get
|
132
133
|
end
|
133
134
|
end
|
134
135
|
|
@@ -151,7 +152,8 @@ class CounterClientTest < ::Test::Unit::TestCase
|
|
151
152
|
]
|
152
153
|
)
|
153
154
|
test 'return an error object' do |(param, expected_error)|
|
154
|
-
|
155
|
+
params = { name: 'key1', reset_interval: 10 }
|
156
|
+
@client.init(params).get
|
155
157
|
response = @client.init(param).get
|
156
158
|
errors = response.errors.first
|
157
159
|
|
@@ -164,7 +166,8 @@ class CounterClientTest < ::Test::Unit::TestCase
|
|
164
166
|
end
|
165
167
|
|
166
168
|
test 'return an existing value when passed key already exists and ignore option is true' do
|
167
|
-
|
169
|
+
params = { name: 'key1', reset_interval: 10 }
|
170
|
+
res1 = @client.init(params).get
|
168
171
|
res2 = nil
|
169
172
|
assert_nothing_raised do
|
170
173
|
res2 = @client.init({ name: 'key1', reset_interval: 10 }, options: { ignore: true }).get
|
@@ -312,7 +315,8 @@ class CounterClientTest < ::Test::Unit::TestCase
|
|
312
315
|
test 'raise an error when @scope is nil' do
|
313
316
|
@client.instance_variable_set(:@scope, nil)
|
314
317
|
assert_raise 'Call `establish` method to get a `scope` before calling this method' do
|
315
|
-
|
318
|
+
params = { name: 'name', value: 1 }
|
319
|
+
@client.inc(params).get
|
316
320
|
end
|
317
321
|
end
|
318
322
|
|
@@ -0,0 +1,99 @@
|
|
1
|
+
require_relative 'helper'
|
2
|
+
require 'fluent/fluent_log_event_router'
|
3
|
+
require 'fluent/root_agent'
|
4
|
+
require 'fluent/system_config'
|
5
|
+
|
6
|
+
class FluentLogEventRouterTest < ::Test::Unit::TestCase
|
7
|
+
# @param config [String]
|
8
|
+
def build_config(config)
|
9
|
+
Fluent::Config.parse(config, 'fluent_log_event', '', syntax: :v1)
|
10
|
+
end
|
11
|
+
|
12
|
+
sub_test_case 'NullFluentLogEventRouter does nothing' do
|
13
|
+
test 'emittable? returns false but others does nothing' do
|
14
|
+
null_event_router = Fluent::NullFluentLogEventRouter.new
|
15
|
+
null_event_router.start
|
16
|
+
null_event_router.stop
|
17
|
+
null_event_router.graceful_stop
|
18
|
+
null_event_router.emit_event(nil)
|
19
|
+
assert_false null_event_router.emittable?
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
sub_test_case '#build' do
|
24
|
+
test 'NullFluentLogEventRouter if root_agent have not internal logger' do
|
25
|
+
root_agent = Fluent::RootAgent.new(log: $log, system_config: Fluent::SystemConfig.new)
|
26
|
+
root_agent.configure(build_config(''))
|
27
|
+
|
28
|
+
d = Fluent::FluentLogEventRouter.build(root_agent)
|
29
|
+
assert_equal Fluent::NullFluentLogEventRouter, d.class
|
30
|
+
end
|
31
|
+
|
32
|
+
test 'FluentLogEventRouter if <match fluent.*> exists in config' do
|
33
|
+
root_agent = Fluent::RootAgent.new(log: $log, system_config: Fluent::SystemConfig.new)
|
34
|
+
root_agent.configure(build_config(<<-CONFIG))
|
35
|
+
<match fluent.*>
|
36
|
+
@type null
|
37
|
+
</match>
|
38
|
+
CONFIG
|
39
|
+
|
40
|
+
d = Fluent::FluentLogEventRouter.build(root_agent)
|
41
|
+
assert_equal Fluent::FluentLogEventRouter, d.class
|
42
|
+
end
|
43
|
+
|
44
|
+
test 'FluentLogEventRouter if <label @FLUENT_LOG> exists in config' do
|
45
|
+
root_agent = Fluent::RootAgent.new(log: $log, system_config: Fluent::SystemConfig.new)
|
46
|
+
root_agent.configure(build_config(<<-CONFIG))
|
47
|
+
<label @FLUENT_LOG>
|
48
|
+
<match *>
|
49
|
+
@type null
|
50
|
+
</match>
|
51
|
+
</label>
|
52
|
+
CONFIG
|
53
|
+
|
54
|
+
d = Fluent::FluentLogEventRouter.build(root_agent)
|
55
|
+
assert_equal Fluent::FluentLogEventRouter, d.class
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
test 'when calling graceful_stop, it flushes all events' do
|
60
|
+
event_router = []
|
61
|
+
stub(event_router).emit do |tag, time, record|
|
62
|
+
event_router.push([tag, time, record])
|
63
|
+
end
|
64
|
+
|
65
|
+
d = Fluent::FluentLogEventRouter.new(event_router)
|
66
|
+
|
67
|
+
t = Time.now
|
68
|
+
msg = ['tag', t, { 'key' => 'value' }]
|
69
|
+
d.emit_event(msg)
|
70
|
+
d.graceful_stop
|
71
|
+
d.emit_event(msg)
|
72
|
+
d.start
|
73
|
+
|
74
|
+
d.graceful_stop # to call join
|
75
|
+
assert_equal 2, event_router.size
|
76
|
+
assert_equal msg, event_router[0]
|
77
|
+
assert_equal msg, event_router[1]
|
78
|
+
end
|
79
|
+
|
80
|
+
test 'when calling stop, it ignores existing events' do
|
81
|
+
event_router = []
|
82
|
+
stub(event_router).emit do |tag, time, record|
|
83
|
+
event_router.push([tag, time, record])
|
84
|
+
end
|
85
|
+
|
86
|
+
d = Fluent::FluentLogEventRouter.new(event_router)
|
87
|
+
|
88
|
+
t = Time.now
|
89
|
+
msg = ['tag', t, { 'key' => 'value' }]
|
90
|
+
d.emit_event(msg)
|
91
|
+
d.stop
|
92
|
+
d.emit_event(msg)
|
93
|
+
d.start
|
94
|
+
|
95
|
+
d.stop # to call join
|
96
|
+
assert_equal 1, event_router.size
|
97
|
+
assert_equal msg, event_router[0]
|
98
|
+
end
|
99
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluentd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.8.0.
|
4
|
+
version: 1.8.0.rc2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-10-
|
11
|
+
date: 2019-10-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|
@@ -122,16 +122,22 @@ dependencies:
|
|
122
122
|
name: tzinfo
|
123
123
|
requirement: !ruby/object:Gem::Requirement
|
124
124
|
requirements:
|
125
|
-
- - "
|
125
|
+
- - ">="
|
126
126
|
- !ruby/object:Gem::Version
|
127
|
-
version: '
|
127
|
+
version: '1.0'
|
128
|
+
- - "<"
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '3.0'
|
128
131
|
type: :runtime
|
129
132
|
prerelease: false
|
130
133
|
version_requirements: !ruby/object:Gem::Requirement
|
131
134
|
requirements:
|
132
|
-
- - "
|
135
|
+
- - ">="
|
133
136
|
- !ruby/object:Gem::Version
|
134
|
-
version: '
|
137
|
+
version: '1.0'
|
138
|
+
- - "<"
|
139
|
+
- !ruby/object:Gem::Version
|
140
|
+
version: '3.0'
|
135
141
|
- !ruby/object:Gem::Dependency
|
136
142
|
name: tzinfo-data
|
137
143
|
requirement: !ruby/object:Gem::Requirement
|
@@ -457,6 +463,7 @@ files:
|
|
457
463
|
- lib/fluent/event.rb
|
458
464
|
- lib/fluent/event_router.rb
|
459
465
|
- lib/fluent/filter.rb
|
466
|
+
- lib/fluent/fluent_log_event_router.rb
|
460
467
|
- lib/fluent/formatter.rb
|
461
468
|
- lib/fluent/input.rb
|
462
469
|
- lib/fluent/label.rb
|
@@ -797,6 +804,7 @@ files:
|
|
797
804
|
- test/test_event_router.rb
|
798
805
|
- test/test_event_time.rb
|
799
806
|
- test/test_filter.rb
|
807
|
+
- test/test_fluent_log_event_router.rb
|
800
808
|
- test/test_formatter.rb
|
801
809
|
- test/test_input.rb
|
802
810
|
- test/test_log.rb
|
@@ -999,6 +1007,7 @@ test_files:
|
|
999
1007
|
- test/test_event_router.rb
|
1000
1008
|
- test/test_event_time.rb
|
1001
1009
|
- test/test_filter.rb
|
1010
|
+
- test/test_fluent_log_event_router.rb
|
1002
1011
|
- test/test_formatter.rb
|
1003
1012
|
- test/test_input.rb
|
1004
1013
|
- test/test_log.rb
|