fluentd 1.13.3-x86-mingw32 → 1.14.3-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.drone.yml +6 -6
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
- data/.github/workflows/windows-test.yaml +3 -3
- data/CHANGELOG.md +126 -0
- data/README.md +2 -0
- data/SECURITY.md +18 -0
- data/fluentd.gemspec +3 -3
- data/lib/fluent/command/cat.rb +13 -3
- data/lib/fluent/command/fluentd.rb +8 -0
- data/lib/fluent/compat/output.rb +9 -6
- data/lib/fluent/config/parser.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +1 -1
- data/lib/fluent/event_router.rb +28 -1
- data/lib/fluent/plugin/bare_output.rb +49 -8
- data/lib/fluent/plugin/buf_file.rb +2 -2
- data/lib/fluent/plugin/buffer.rb +123 -27
- data/lib/fluent/plugin/filter.rb +35 -1
- data/lib/fluent/plugin/in_http.rb +21 -2
- data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
- data/lib/fluent/plugin/in_syslog.rb +13 -1
- data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
- data/lib/fluent/plugin/in_tail.rb +47 -8
- data/lib/fluent/plugin/input.rb +39 -1
- data/lib/fluent/plugin/metrics.rb +119 -0
- data/lib/fluent/plugin/metrics_local.rb +96 -0
- data/lib/fluent/plugin/multi_output.rb +43 -6
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_forward.rb +15 -7
- data/lib/fluent/plugin/output.rb +80 -38
- data/lib/fluent/plugin/parser_apache2.rb +1 -1
- data/lib/fluent/plugin/storage_local.rb +3 -5
- data/lib/fluent/plugin.rb +10 -1
- data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
- data/lib/fluent/plugin_helper/metrics.rb +129 -0
- data/lib/fluent/plugin_helper/server.rb +4 -2
- data/lib/fluent/plugin_helper.rb +1 -0
- data/lib/fluent/plugin_id.rb +2 -1
- data/lib/fluent/root_agent.rb +6 -0
- data/lib/fluent/supervisor.rb +4 -2
- data/lib/fluent/system_config.rb +9 -1
- data/lib/fluent/time.rb +21 -20
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_cat.rb +31 -2
- data/test/config/test_system_config.rb +6 -0
- data/test/plugin/in_tail/test_io_handler.rb +12 -4
- data/test/plugin/in_tail/test_position_file.rb +26 -4
- data/test/plugin/test_bare_output.rb +13 -0
- data/test/plugin/test_buffer.rb +80 -3
- data/test/plugin/test_filter.rb +11 -0
- data/test/plugin/test_in_http.rb +40 -0
- data/test/plugin/test_in_monitor_agent.rb +214 -8
- data/test/plugin/test_in_syslog.rb +35 -0
- data/test/plugin/test_in_tail.rb +83 -35
- data/test/plugin/test_input.rb +11 -0
- data/test/plugin/test_metrics.rb +294 -0
- data/test/plugin/test_metrics_local.rb +96 -0
- data/test/plugin/test_multi_output.rb +25 -1
- data/test/plugin/test_out_exec_filter.rb +4 -0
- data/test/plugin/test_output.rb +16 -0
- data/test/plugin_helper/test_child_process.rb +9 -9
- data/test/plugin_helper/test_event_emitter.rb +29 -0
- data/test/plugin_helper/test_metrics.rb +137 -0
- data/test/test_plugin_classes.rb +102 -0
- data/test/test_root_agent.rb +30 -1
- data/test/test_time_parser.rb +22 -0
- metadata +18 -8
data/lib/fluent/plugin/output.rb
CHANGED
@@ -14,6 +14,7 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
+
require 'fluent/env'
|
17
18
|
require 'fluent/error'
|
18
19
|
require 'fluent/plugin/base'
|
19
20
|
require 'fluent/plugin/buffer'
|
@@ -37,7 +38,7 @@ module Fluent
|
|
37
38
|
include PluginHelper::Mixin
|
38
39
|
include UniqueId::Mixin
|
39
40
|
|
40
|
-
helpers_internal :thread, :retry_state
|
41
|
+
helpers_internal :thread, :retry_state, :metrics
|
41
42
|
|
42
43
|
CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
|
43
44
|
CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
|
@@ -164,7 +165,6 @@ module Fluent
|
|
164
165
|
end
|
165
166
|
|
166
167
|
attr_reader :as_secondary, :delayed_commit, :delayed_commit_timeout, :timekey_zone
|
167
|
-
attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
|
168
168
|
|
169
169
|
# for tests
|
170
170
|
attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
|
@@ -172,6 +172,30 @@ module Fluent
|
|
172
172
|
# output_enqueue_thread_waiting: for test of output.rb itself
|
173
173
|
attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
|
174
174
|
|
175
|
+
def num_errors
|
176
|
+
@num_errors_metrics.get
|
177
|
+
end
|
178
|
+
|
179
|
+
def emit_count
|
180
|
+
@emit_count_metrics.get
|
181
|
+
end
|
182
|
+
|
183
|
+
def emit_size
|
184
|
+
@emit_size_metrics.get
|
185
|
+
end
|
186
|
+
|
187
|
+
def emit_records
|
188
|
+
@emit_records_metrics.get
|
189
|
+
end
|
190
|
+
|
191
|
+
def write_count
|
192
|
+
@write_count_metrics.get
|
193
|
+
end
|
194
|
+
|
195
|
+
def rollback_count
|
196
|
+
@rollback_count_metrics.get
|
197
|
+
end
|
198
|
+
|
175
199
|
def initialize
|
176
200
|
super
|
177
201
|
@counter_mutex = Mutex.new
|
@@ -181,13 +205,15 @@ module Fluent
|
|
181
205
|
@primary_instance = nil
|
182
206
|
|
183
207
|
# TODO: well organized counters
|
184
|
-
@
|
185
|
-
@
|
186
|
-
@
|
187
|
-
@
|
188
|
-
@
|
189
|
-
@
|
190
|
-
@
|
208
|
+
@num_errors_metrics = nil
|
209
|
+
@emit_count_metrics = nil
|
210
|
+
@emit_records_metrics = nil
|
211
|
+
@emit_size_metrics = nil
|
212
|
+
@write_count_metrics = nil
|
213
|
+
@rollback_count_metrics = nil
|
214
|
+
@flush_time_count_metrics = nil
|
215
|
+
@slow_flush_count_metrics = nil
|
216
|
+
@enable_size_metrics = false
|
191
217
|
|
192
218
|
# How to process events is decided here at once, but it will be decided in delayed way on #configure & #start
|
193
219
|
if implement?(:synchronous)
|
@@ -246,6 +272,15 @@ module Fluent
|
|
246
272
|
|
247
273
|
super
|
248
274
|
|
275
|
+
@num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
|
276
|
+
@emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of count emits")
|
277
|
+
@emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
|
278
|
+
@emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
|
279
|
+
@write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
|
280
|
+
@rollback_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "rollback_count", help_text: "Number of rollbacking operations")
|
281
|
+
@flush_time_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "flush_time_count", help_text: "Count of flush time")
|
282
|
+
@slow_flush_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "slow_flush_count", help_text: "Count of slow flush occurred time(s)")
|
283
|
+
|
249
284
|
if has_buffer_section
|
250
285
|
unless implement?(:buffered) || implement?(:delayed_commit)
|
251
286
|
raise Fluent::ConfigError, "<buffer> section is configured, but plugin '#{self.class}' doesn't support buffering"
|
@@ -271,6 +306,8 @@ module Fluent
|
|
271
306
|
@buffering = true
|
272
307
|
end
|
273
308
|
end
|
309
|
+
# Enable to update record size metrics or not
|
310
|
+
@enable_size_metrics = !!system_config.enable_size_metrics
|
274
311
|
|
275
312
|
if @as_secondary
|
276
313
|
if !@buffering && !@buffering.nil?
|
@@ -797,18 +834,19 @@ module Fluent
|
|
797
834
|
end
|
798
835
|
|
799
836
|
def emit_sync(tag, es)
|
800
|
-
@
|
837
|
+
@emit_count_metrics.inc
|
801
838
|
begin
|
802
839
|
process(tag, es)
|
803
|
-
@
|
840
|
+
@emit_records_metrics.add(es.size)
|
841
|
+
@emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
|
804
842
|
rescue
|
805
|
-
@
|
843
|
+
@num_errors_metrics.inc
|
806
844
|
raise
|
807
845
|
end
|
808
846
|
end
|
809
847
|
|
810
848
|
def emit_buffered(tag, es)
|
811
|
-
@
|
849
|
+
@emit_count_metrics.inc
|
812
850
|
begin
|
813
851
|
execute_chunking(tag, es, enqueue: (@flush_mode == :immediate))
|
814
852
|
if !@retry && @buffer.queued?(nil, optimistic: true)
|
@@ -816,7 +854,7 @@ module Fluent
|
|
816
854
|
end
|
817
855
|
rescue
|
818
856
|
# TODO: separate number of errors into emit errors and write/flush errors
|
819
|
-
@
|
857
|
+
@num_errors_metrics.inc
|
820
858
|
raise
|
821
859
|
end
|
822
860
|
end
|
@@ -966,7 +1004,8 @@ module Fluent
|
|
966
1004
|
write_guard do
|
967
1005
|
@buffer.write(meta_and_data, enqueue: enqueue)
|
968
1006
|
end
|
969
|
-
@
|
1007
|
+
@emit_records_metrics.add(es.size)
|
1008
|
+
@emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
|
970
1009
|
true
|
971
1010
|
end
|
972
1011
|
|
@@ -983,7 +1022,8 @@ module Fluent
|
|
983
1022
|
write_guard do
|
984
1023
|
@buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
|
985
1024
|
end
|
986
|
-
@
|
1025
|
+
@emit_records_metrics.add(es.size)
|
1026
|
+
@emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
|
987
1027
|
true
|
988
1028
|
end
|
989
1029
|
|
@@ -1008,7 +1048,8 @@ module Fluent
|
|
1008
1048
|
write_guard do
|
1009
1049
|
@buffer.write({meta => data}, format: format_proc, enqueue: enqueue)
|
1010
1050
|
end
|
1011
|
-
@
|
1051
|
+
@emit_records_metrics.add(es.size)
|
1052
|
+
@emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
|
1012
1053
|
true
|
1013
1054
|
end
|
1014
1055
|
|
@@ -1046,7 +1087,7 @@ module Fluent
|
|
1046
1087
|
# false if chunk was already flushed and couldn't be rollbacked unexpectedly
|
1047
1088
|
# in many cases, false can be just ignored
|
1048
1089
|
if @buffer.takeback_chunk(chunk_id)
|
1049
|
-
@
|
1090
|
+
@rollback_count_metrics.inc
|
1050
1091
|
if update_retry
|
1051
1092
|
primary = @as_secondary ? @primary_instance : self
|
1052
1093
|
primary.update_retry_state(chunk_id, @as_secondary)
|
@@ -1062,7 +1103,7 @@ module Fluent
|
|
1062
1103
|
while @dequeued_chunks.first && @dequeued_chunks.first.expired?
|
1063
1104
|
info = @dequeued_chunks.shift
|
1064
1105
|
if @buffer.takeback_chunk(info.chunk_id)
|
1065
|
-
@
|
1106
|
+
@rollback_count_metrics.inc
|
1066
1107
|
log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
|
1067
1108
|
primary = @as_secondary ? @primary_instance : self
|
1068
1109
|
primary.update_retry_state(info.chunk_id, @as_secondary)
|
@@ -1077,7 +1118,7 @@ module Fluent
|
|
1077
1118
|
until @dequeued_chunks.empty?
|
1078
1119
|
info = @dequeued_chunks.shift
|
1079
1120
|
if @buffer.takeback_chunk(info.chunk_id)
|
1080
|
-
@
|
1121
|
+
@rollback_count_metrics.inc
|
1081
1122
|
log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
|
1082
1123
|
primary = @as_secondary ? @primary_instance : self
|
1083
1124
|
primary.update_retry_state(info.chunk_id, @as_secondary)
|
@@ -1120,7 +1161,7 @@ module Fluent
|
|
1120
1161
|
|
1121
1162
|
if output.delayed_commit
|
1122
1163
|
log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
|
1123
|
-
@
|
1164
|
+
@write_count_metrics.inc
|
1124
1165
|
@dequeued_chunks_mutex.synchronize do
|
1125
1166
|
# delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
|
1126
1167
|
@dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
|
@@ -1132,7 +1173,7 @@ module Fluent
|
|
1132
1173
|
chunk_id = chunk.unique_id
|
1133
1174
|
dump_chunk_id = dump_unique_id_hex(chunk_id)
|
1134
1175
|
log.trace "adding write count", instance: self.object_id
|
1135
|
-
@
|
1176
|
+
@write_count_metrics.inc
|
1136
1177
|
log.trace "executing sync write", chunk: dump_chunk_id
|
1137
1178
|
|
1138
1179
|
output.write(chunk)
|
@@ -1188,7 +1229,7 @@ module Fluent
|
|
1188
1229
|
end
|
1189
1230
|
|
1190
1231
|
if @buffer.takeback_chunk(chunk.unique_id)
|
1191
|
-
@
|
1232
|
+
@rollback_count_metrics.inc
|
1192
1233
|
end
|
1193
1234
|
|
1194
1235
|
update_retry_state(chunk.unique_id, using_secondary, e)
|
@@ -1208,8 +1249,8 @@ module Fluent
|
|
1208
1249
|
backup_dir = File.dirname(backup_file)
|
1209
1250
|
|
1210
1251
|
log.warn "bad chunk is moved to #{backup_file}"
|
1211
|
-
FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission ||
|
1212
|
-
File.open(backup_file, 'ab', system_config.file_permission ||
|
1252
|
+
FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
|
1253
|
+
File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f|
|
1213
1254
|
chunk.write_to(f)
|
1214
1255
|
}
|
1215
1256
|
end
|
@@ -1219,9 +1260,9 @@ module Fluent
|
|
1219
1260
|
def check_slow_flush(start)
|
1220
1261
|
elapsed_time = Fluent::Clock.now - start
|
1221
1262
|
elapsed_millsec = (elapsed_time * 1000).to_i
|
1222
|
-
@
|
1263
|
+
@flush_time_count_metrics.add(elapsed_millsec)
|
1223
1264
|
if elapsed_time > @slow_flush_log_threshold
|
1224
|
-
@
|
1265
|
+
@slow_flush_count_metrics.inc
|
1225
1266
|
log.warn "buffer flush took longer time than slow_flush_log_threshold:",
|
1226
1267
|
elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
|
1227
1268
|
end
|
@@ -1229,13 +1270,13 @@ module Fluent
|
|
1229
1270
|
|
1230
1271
|
def update_retry_state(chunk_id, using_secondary, error = nil)
|
1231
1272
|
@retry_mutex.synchronize do
|
1232
|
-
@
|
1273
|
+
@num_errors_metrics.inc
|
1233
1274
|
chunk_id_hex = dump_unique_id_hex(chunk_id)
|
1234
1275
|
|
1235
1276
|
unless @retry
|
1236
1277
|
@retry = retry_state(@buffer_config.retry_randomize)
|
1237
1278
|
if error
|
1238
|
-
log.warn "failed to flush the buffer.",
|
1279
|
+
log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
|
1239
1280
|
log.warn_backtrace error.backtrace
|
1240
1281
|
end
|
1241
1282
|
return
|
@@ -1264,11 +1305,11 @@ module Fluent
|
|
1264
1305
|
if error
|
1265
1306
|
if using_secondary
|
1266
1307
|
msg = "failed to flush the buffer with secondary output."
|
1267
|
-
log.warn msg,
|
1308
|
+
log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
|
1268
1309
|
log.warn_backtrace error.backtrace
|
1269
1310
|
else
|
1270
1311
|
msg = "failed to flush the buffer."
|
1271
|
-
log.warn msg,
|
1312
|
+
log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
|
1272
1313
|
log.warn_backtrace error.backtrace
|
1273
1314
|
end
|
1274
1315
|
end
|
@@ -1490,15 +1531,16 @@ module Fluent
|
|
1490
1531
|
|
1491
1532
|
def statistics
|
1492
1533
|
stats = {
|
1493
|
-
'emit_records' => @
|
1534
|
+
'emit_records' => @emit_records_metrics.get,
|
1535
|
+
'emit_size' => @emit_size_metrics.get,
|
1494
1536
|
# Respect original name
|
1495
1537
|
# https://github.com/fluent/fluentd/blob/45c7b75ba77763eaf87136864d4942c4e0c5bfcd/lib/fluent/plugin/in_monitor_agent.rb#L284
|
1496
|
-
'retry_count' => @
|
1497
|
-
'emit_count' => @
|
1498
|
-
'write_count' => @
|
1499
|
-
'rollback_count' => @
|
1500
|
-
'slow_flush_count' => @
|
1501
|
-
'flush_time_count' => @
|
1538
|
+
'retry_count' => @num_errors_metrics.get,
|
1539
|
+
'emit_count' => @emit_count_metrics.get,
|
1540
|
+
'write_count' => @write_count_metrics.get,
|
1541
|
+
'rollback_count' => @rollback_count_metrics.get,
|
1542
|
+
'slow_flush_count' => @slow_flush_count_metrics.get,
|
1543
|
+
'flush_time_count' => @flush_time_count_metrics.get,
|
1502
1544
|
}
|
1503
1545
|
|
1504
1546
|
if @buffer && @buffer.respond_to?(:statistics)
|
@@ -21,7 +21,7 @@ module Fluent
|
|
21
21
|
class Apache2Parser < Parser
|
22
22
|
Plugin.register_parser('apache2', self)
|
23
23
|
|
24
|
-
REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]
|
24
|
+
REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\")*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\")*)" "(?<agent>(?:[^\"]|\\")*)")?$/
|
25
25
|
TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
|
26
26
|
|
27
27
|
def initialize
|
@@ -14,6 +14,7 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
+
require 'fluent/env'
|
17
18
|
require 'fluent/plugin'
|
18
19
|
require 'fluent/plugin/storage'
|
19
20
|
|
@@ -25,14 +26,11 @@ module Fluent
|
|
25
26
|
class LocalStorage < Storage
|
26
27
|
Fluent::Plugin.register_storage('local', self)
|
27
28
|
|
28
|
-
DEFAULT_DIR_MODE = 0755
|
29
|
-
DEFAULT_FILE_MODE = 0644
|
30
|
-
|
31
29
|
config_param :path, :string, default: nil
|
32
|
-
config_param :mode, default:
|
30
|
+
config_param :mode, default: Fluent::DEFAULT_FILE_PERMISSION do |v|
|
33
31
|
v.to_i(8)
|
34
32
|
end
|
35
|
-
config_param :dir_mode, default:
|
33
|
+
config_param :dir_mode, default: Fluent::DEFAULT_DIR_PERMISSION do |v|
|
36
34
|
v.to_i(8)
|
37
35
|
end
|
38
36
|
config_param :pretty_print, :bool, default: false
|
data/lib/fluent/plugin.rb
CHANGED
@@ -36,8 +36,9 @@ module Fluent
|
|
36
36
|
FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_', dir_search_prefix: 'formatter_')
|
37
37
|
STORAGE_REGISTRY = Registry.new(:storage, 'fluent/plugin/storage_', dir_search_prefix: 'storage_')
|
38
38
|
SD_REGISTRY = Registry.new(:sd, 'fluent/plugin/sd_', dir_search_prefix: 'sd_')
|
39
|
+
METRICS_REGISTRY = Registry.new(:metrics, 'fluent/plugin/metrics_', dir_search_prefix: 'metrics_')
|
39
40
|
|
40
|
-
REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY]
|
41
|
+
REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY, METRICS_REGISTRY]
|
41
42
|
|
42
43
|
def self.register_input(type, klass)
|
43
44
|
register_impl('input', INPUT_REGISTRY, type, klass)
|
@@ -59,6 +60,10 @@ module Fluent
|
|
59
60
|
register_impl('sd', SD_REGISTRY, type, klass)
|
60
61
|
end
|
61
62
|
|
63
|
+
def self.register_metrics(type, klass)
|
64
|
+
register_impl('metrics', METRICS_REGISTRY, type, klass)
|
65
|
+
end
|
66
|
+
|
62
67
|
def self.register_parser(type, klass_or_proc)
|
63
68
|
if klass_or_proc.is_a?(Regexp)
|
64
69
|
# This usage is not recommended for new API
|
@@ -121,6 +126,10 @@ module Fluent
|
|
121
126
|
new_impl('sd', SD_REGISTRY, type, parent)
|
122
127
|
end
|
123
128
|
|
129
|
+
def self.new_metrics(type, parent: nil)
|
130
|
+
new_impl('metrics', METRICS_REGISTRY, type, parent)
|
131
|
+
end
|
132
|
+
|
124
133
|
class << self
|
125
134
|
# This should be defined for fluent-plugin-config-formatter type arguments.
|
126
135
|
alias_method :new_service_discovery, :new_sd
|
@@ -29,6 +29,9 @@ module Fluent
|
|
29
29
|
if @_event_emitter_lazy_init
|
30
30
|
@router = @primary_instance.router
|
31
31
|
end
|
32
|
+
if @router.respond_to?(:caller_plugin_id=)
|
33
|
+
@router.caller_plugin_id = self.plugin_id
|
34
|
+
end
|
32
35
|
@router
|
33
36
|
end
|
34
37
|
|
@@ -47,7 +50,11 @@ module Fluent
|
|
47
50
|
|
48
51
|
def event_emitter_router(label_name)
|
49
52
|
if label_name
|
50
|
-
|
53
|
+
if label_name == "@ROOT"
|
54
|
+
Engine.root_agent.event_router
|
55
|
+
else
|
56
|
+
Engine.root_agent.find_label(label_name).event_router
|
57
|
+
end
|
51
58
|
elsif self.respond_to?(:as_secondary) && self.as_secondary
|
52
59
|
if @primary_instance.has_router?
|
53
60
|
@_event_emitter_lazy_init = true
|
@@ -0,0 +1,129 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'forwardable'
|
18
|
+
|
19
|
+
require 'fluent/plugin'
|
20
|
+
require 'fluent/plugin/metrics'
|
21
|
+
require 'fluent/plugin_helper/timer'
|
22
|
+
require 'fluent/config/element'
|
23
|
+
require 'fluent/configurable'
|
24
|
+
require 'fluent/system_config'
|
25
|
+
|
26
|
+
module Fluent
|
27
|
+
module PluginHelper
|
28
|
+
module Metrics
|
29
|
+
include Fluent::SystemConfig::Mixin
|
30
|
+
|
31
|
+
attr_reader :_metrics # For tests.
|
32
|
+
|
33
|
+
def initialize
|
34
|
+
super
|
35
|
+
@_metrics_started = false
|
36
|
+
@_metrics = {} # usage => metrics_state
|
37
|
+
end
|
38
|
+
|
39
|
+
def configure(conf)
|
40
|
+
super
|
41
|
+
|
42
|
+
@plugin_type_or_id = if self.plugin_id_configured?
|
43
|
+
self.plugin_id
|
44
|
+
else
|
45
|
+
if type = (conf["@type"] || conf["type"])
|
46
|
+
"#{type}.#{self.plugin_id}"
|
47
|
+
else
|
48
|
+
"#{self.class.to_s.split("::").last.downcase}.#{self.plugin_id}"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def metrics_create(namespace: "fluentd", subsystem: "metrics", name:, help_text:, labels: {}, prefer_gauge: false)
|
54
|
+
metrics = if system_config.metrics
|
55
|
+
Fluent::Plugin.new_metrics(system_config.metrics[:@type], parent: self)
|
56
|
+
else
|
57
|
+
Fluent::Plugin.new_metrics(Fluent::Plugin::Metrics::DEFAULT_TYPE, parent: self)
|
58
|
+
end
|
59
|
+
config = if system_config.metrics
|
60
|
+
system_config.metrics.corresponding_config_element
|
61
|
+
else
|
62
|
+
Fluent::Config::Element.new('metrics', '', {'@type' => Fluent::Plugin::Metrics::DEFAULT_TYPE}, [])
|
63
|
+
end
|
64
|
+
metrics.use_gauge_metric = prefer_gauge
|
65
|
+
metrics.configure(config)
|
66
|
+
# For multi workers environment, cmetrics should be distinguish with static labels.
|
67
|
+
if Fluent::Engine.system_config.workers > 1
|
68
|
+
labels.merge!(worker_id: fluentd_worker_id.to_s)
|
69
|
+
end
|
70
|
+
labels.merge!(plugin: @plugin_type_or_id)
|
71
|
+
metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)
|
72
|
+
|
73
|
+
@_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics
|
74
|
+
|
75
|
+
metrics
|
76
|
+
end
|
77
|
+
|
78
|
+
def metrics_operate(method_name, &block)
|
79
|
+
@_metrics.each_pair do |key, m|
|
80
|
+
begin
|
81
|
+
block.call(s) if block_given?
|
82
|
+
m.__send__(method_name)
|
83
|
+
rescue => e
|
84
|
+
log.error "unexpected error while #{method_name}", key: key, metrics: m, error: e
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def start
|
90
|
+
super
|
91
|
+
|
92
|
+
metrics_operate(:start)
|
93
|
+
@_metrics_started = true
|
94
|
+
end
|
95
|
+
|
96
|
+
def stop
|
97
|
+
super
|
98
|
+
# timer stops automatically in super
|
99
|
+
metrics_operate(:stop)
|
100
|
+
end
|
101
|
+
|
102
|
+
def before_shutdown
|
103
|
+
metrics_operate(:before_shutdown)
|
104
|
+
super
|
105
|
+
end
|
106
|
+
|
107
|
+
def shutdown
|
108
|
+
metrics_operate(:shutdown)
|
109
|
+
super
|
110
|
+
end
|
111
|
+
|
112
|
+
def after_shutdown
|
113
|
+
metrics_operate(:after_shutdown)
|
114
|
+
super
|
115
|
+
end
|
116
|
+
|
117
|
+
def close
|
118
|
+
metrics_operate(:close)
|
119
|
+
super
|
120
|
+
end
|
121
|
+
|
122
|
+
def terminate
|
123
|
+
metrics_operate(:terminate)
|
124
|
+
@_metrics = {}
|
125
|
+
super
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
@@ -709,13 +709,15 @@ module Fluent
|
|
709
709
|
return true
|
710
710
|
end
|
711
711
|
rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
|
712
|
+
peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
|
712
713
|
@log.trace "unexpected error before accepting TLS connection",
|
713
|
-
|
714
|
+
addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
|
714
715
|
close rescue nil
|
715
716
|
rescue OpenSSL::SSL::SSLError => e
|
717
|
+
peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
|
716
718
|
# Use same log level as on_readable
|
717
719
|
@log.warn "unexpected error before accepting TLS connection by OpenSSL",
|
718
|
-
|
720
|
+
addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
|
719
721
|
close rescue nil
|
720
722
|
end
|
721
723
|
|
data/lib/fluent/plugin_helper.rb
CHANGED
@@ -32,6 +32,7 @@ require 'fluent/plugin_helper/retry_state'
|
|
32
32
|
require 'fluent/plugin_helper/record_accessor'
|
33
33
|
require 'fluent/plugin_helper/compat_parameters'
|
34
34
|
require 'fluent/plugin_helper/service_discovery'
|
35
|
+
require 'fluent/plugin_helper/metrics'
|
35
36
|
|
36
37
|
module Fluent
|
37
38
|
module PluginHelper
|
data/lib/fluent/plugin_id.rb
CHANGED
@@ -15,6 +15,7 @@
|
|
15
15
|
#
|
16
16
|
|
17
17
|
require 'set'
|
18
|
+
require 'fluent/env'
|
18
19
|
require 'fluent/variable_store'
|
19
20
|
|
20
21
|
module Fluent
|
@@ -76,7 +77,7 @@ module Fluent
|
|
76
77
|
|
77
78
|
# Fluent::Plugin::Base#fluentd_worker_id
|
78
79
|
dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
|
79
|
-
FileUtils.mkdir_p(dir, mode: system_config.dir_permission ||
|
80
|
+
FileUtils.mkdir_p(dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(dir)
|
80
81
|
@_plugin_root_dir = dir.freeze
|
81
82
|
dir
|
82
83
|
end
|
data/lib/fluent/root_agent.rb
CHANGED
@@ -55,9 +55,11 @@ module Fluent
|
|
55
55
|
@suppress_emit_error_log_interval = 0
|
56
56
|
@next_emit_error_log_time = nil
|
57
57
|
@without_source = false
|
58
|
+
@enable_input_metrics = false
|
58
59
|
|
59
60
|
suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
|
60
61
|
@without_source = system_config.without_source unless system_config.without_source.nil?
|
62
|
+
@enable_input_metrics = !!system_config.enable_input_metrics
|
61
63
|
end
|
62
64
|
|
63
65
|
attr_reader :inputs
|
@@ -131,6 +133,7 @@ module Fluent
|
|
131
133
|
end
|
132
134
|
name = e.arg
|
133
135
|
raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
|
136
|
+
raise ConfigError, "@ROOT for <label> is not permitted, reserved for getting root router" if name == '@ROOT'
|
134
137
|
|
135
138
|
if name == ERROR_LABEL
|
136
139
|
error_label_config = e
|
@@ -315,6 +318,9 @@ module Fluent
|
|
315
318
|
# See also 'fluentd/plugin/input.rb'
|
316
319
|
input.context_router = @event_router
|
317
320
|
input.configure(conf)
|
321
|
+
if @enable_input_metrics
|
322
|
+
@event_router.add_metric_callbacks(input.plugin_id, Proc.new {|es| input.metric_callback(es) })
|
323
|
+
end
|
318
324
|
@inputs << input
|
319
325
|
|
320
326
|
input
|
data/lib/fluent/supervisor.rb
CHANGED
@@ -544,7 +544,7 @@ module Fluent
|
|
544
544
|
$log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
|
545
545
|
|
546
546
|
if @path && log_dir_perm
|
547
|
-
File.chmod(log_dir_perm ||
|
547
|
+
File.chmod(log_dir_perm || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(@path))
|
548
548
|
end
|
549
549
|
end
|
550
550
|
|
@@ -570,6 +570,8 @@ module Fluent
|
|
570
570
|
suppress_repeated_stacktrace: true,
|
571
571
|
ignore_repeated_log_interval: nil,
|
572
572
|
without_source: nil,
|
573
|
+
enable_input_metrics: nil,
|
574
|
+
enable_size_metrics: nil,
|
573
575
|
use_v1_config: true,
|
574
576
|
strict_config_value: nil,
|
575
577
|
supervise: true,
|
@@ -649,7 +651,7 @@ module Fluent
|
|
649
651
|
end
|
650
652
|
else
|
651
653
|
begin
|
652
|
-
FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission ||
|
654
|
+
FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION)
|
653
655
|
rescue => e
|
654
656
|
raise Fluent::InvalidRootDirectory, "failed to create root directory:#{root_dir}, #{e.inspect}"
|
655
657
|
end
|
data/lib/fluent/system_config.rb
CHANGED
@@ -27,7 +27,8 @@ module Fluent
|
|
27
27
|
:log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
|
28
28
|
:without_source, :rpc_endpoint, :enable_get_dump, :process_name,
|
29
29
|
:file_permission, :dir_permission, :counter_server, :counter_client,
|
30
|
-
:strict_config_value, :enable_msgpack_time_support, :disable_shared_socket
|
30
|
+
:strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
|
31
|
+
:metrics, :enable_input_metrics, :enable_size_metrics
|
31
32
|
]
|
32
33
|
|
33
34
|
config_param :workers, :integer, default: 1
|
@@ -46,6 +47,8 @@ module Fluent
|
|
46
47
|
config_param :strict_config_value, :bool, default: nil
|
47
48
|
config_param :enable_msgpack_time_support, :bool, default: nil
|
48
49
|
config_param :disable_shared_socket, :bool, default: nil
|
50
|
+
config_param :enable_input_metrics, :bool, default: nil
|
51
|
+
config_param :enable_size_metrics, :bool, default: nil
|
49
52
|
config_param :file_permission, default: nil do |v|
|
50
53
|
v.to_i(8)
|
51
54
|
end
|
@@ -93,6 +96,11 @@ module Fluent
|
|
93
96
|
config_param :timeout, :time, default: nil
|
94
97
|
end
|
95
98
|
|
99
|
+
config_section :metrics, multi: false do
|
100
|
+
config_param :@type, :string, default: "local"
|
101
|
+
config_param :labels, :hash, default: {}
|
102
|
+
end
|
103
|
+
|
96
104
|
def self.create(conf, strict_config_value=false)
|
97
105
|
systems = conf.elements(name: 'system')
|
98
106
|
return SystemConfig.new if systems.empty?
|