fluentd 1.17.0-x86-mingw32 → 1.18.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (278) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +100 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/command/fluentd.rb +7 -1
  8. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  9. data/lib/fluent/compat/propagate_default.rb +4 -4
  10. data/lib/fluent/config/literal_parser.rb +9 -2
  11. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  12. data/lib/fluent/engine.rb +49 -33
  13. data/lib/fluent/env.rb +3 -0
  14. data/lib/fluent/event_router.rb +2 -2
  15. data/lib/fluent/log/console_adapter.rb +4 -2
  16. data/lib/fluent/plugin/filter_parser.rb +27 -51
  17. data/lib/fluent/plugin/in_exec.rb +14 -2
  18. data/lib/fluent/plugin/in_http.rb +6 -1
  19. data/lib/fluent/plugin/in_sample.rb +13 -7
  20. data/lib/fluent/plugin/in_syslog.rb +4 -0
  21. data/lib/fluent/plugin/in_tail.rb +65 -23
  22. data/lib/fluent/plugin/in_tcp.rb +4 -0
  23. data/lib/fluent/plugin/in_udp.rb +10 -1
  24. data/lib/fluent/plugin/input.rb +4 -0
  25. data/lib/fluent/plugin/out_buffer.rb +40 -0
  26. data/lib/fluent/plugin/out_copy.rb +1 -1
  27. data/lib/fluent/plugin/out_file.rb +8 -0
  28. data/lib/fluent/plugin/out_http.rb +12 -0
  29. data/lib/fluent/plugin/output.rb +2 -0
  30. data/lib/fluent/plugin/parser_json.rb +4 -12
  31. data/lib/fluent/plugin_helper/cert_option.rb +8 -0
  32. data/lib/fluent/plugin_helper/event_emitter.rb +12 -0
  33. data/lib/fluent/plugin_helper/http_server/server.rb +24 -8
  34. data/lib/fluent/plugin_helper/server.rb +9 -0
  35. data/lib/fluent/root_agent.rb +114 -19
  36. data/lib/fluent/source_only_buffer_agent.rb +102 -0
  37. data/lib/fluent/supervisor.rb +207 -34
  38. data/lib/fluent/system_config.rb +15 -3
  39. data/lib/fluent/version.rb +1 -1
  40. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  41. metadata +24 -483
  42. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  43. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  44. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  45. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  46. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  47. data/.github/ISSUE_TEMPLATE.md +0 -17
  48. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  49. data/.github/workflows/stale-actions.yml +0 -24
  50. data/.github/workflows/test-ruby-head.yml +0 -31
  51. data/.github/workflows/test.yml +0 -32
  52. data/.gitignore +0 -30
  53. data/Gemfile +0 -9
  54. data/fluentd.gemspec +0 -62
  55. data/test/command/test_binlog_reader.rb +0 -362
  56. data/test/command/test_ca_generate.rb +0 -70
  57. data/test/command/test_cap_ctl.rb +0 -100
  58. data/test/command/test_cat.rb +0 -128
  59. data/test/command/test_ctl.rb +0 -56
  60. data/test/command/test_fluentd.rb +0 -1291
  61. data/test/command/test_plugin_config_formatter.rb +0 -397
  62. data/test/command/test_plugin_generator.rb +0 -109
  63. data/test/compat/test_calls_super.rb +0 -166
  64. data/test/compat/test_parser.rb +0 -92
  65. data/test/config/assertions.rb +0 -42
  66. data/test/config/test_config_parser.rb +0 -551
  67. data/test/config/test_configurable.rb +0 -1784
  68. data/test/config/test_configure_proxy.rb +0 -604
  69. data/test/config/test_dsl.rb +0 -415
  70. data/test/config/test_element.rb +0 -518
  71. data/test/config/test_literal_parser.rb +0 -309
  72. data/test/config/test_plugin_configuration.rb +0 -56
  73. data/test/config/test_section.rb +0 -191
  74. data/test/config/test_system_config.rb +0 -195
  75. data/test/config/test_types.rb +0 -408
  76. data/test/counter/test_client.rb +0 -563
  77. data/test/counter/test_error.rb +0 -44
  78. data/test/counter/test_mutex_hash.rb +0 -179
  79. data/test/counter/test_server.rb +0 -589
  80. data/test/counter/test_store.rb +0 -258
  81. data/test/counter/test_validator.rb +0 -137
  82. data/test/helper.rb +0 -155
  83. data/test/helpers/fuzzy_assert.rb +0 -89
  84. data/test/helpers/process_extenstion.rb +0 -33
  85. data/test/log/test_console_adapter.rb +0 -117
  86. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  87. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  88. data/test/plugin/data/2010/01/20100102.log +0 -0
  89. data/test/plugin/data/log/bar +0 -0
  90. data/test/plugin/data/log/foo/bar.log +0 -0
  91. data/test/plugin/data/log/foo/bar2 +0 -0
  92. data/test/plugin/data/log/test.log +0 -0
  93. data/test/plugin/data/log_numeric/01.log +0 -0
  94. data/test/plugin/data/log_numeric/02.log +0 -0
  95. data/test/plugin/data/log_numeric/12.log +0 -0
  96. data/test/plugin/data/log_numeric/14.log +0 -0
  97. data/test/plugin/data/sd_file/config +0 -11
  98. data/test/plugin/data/sd_file/config.json +0 -17
  99. data/test/plugin/data/sd_file/config.yaml +0 -11
  100. data/test/plugin/data/sd_file/config.yml +0 -11
  101. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  102. data/test/plugin/in_tail/test_fifo.rb +0 -121
  103. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  104. data/test/plugin/in_tail/test_position_file.rb +0 -346
  105. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  106. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  107. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  108. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  109. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  110. data/test/plugin/test_bare_output.rb +0 -131
  111. data/test/plugin/test_base.rb +0 -247
  112. data/test/plugin/test_buf_file.rb +0 -1314
  113. data/test/plugin/test_buf_file_single.rb +0 -898
  114. data/test/plugin/test_buf_memory.rb +0 -42
  115. data/test/plugin/test_buffer.rb +0 -1493
  116. data/test/plugin/test_buffer_chunk.rb +0 -209
  117. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  118. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  119. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  120. data/test/plugin/test_compressable.rb +0 -87
  121. data/test/plugin/test_file_util.rb +0 -96
  122. data/test/plugin/test_filter.rb +0 -368
  123. data/test/plugin/test_filter_grep.rb +0 -697
  124. data/test/plugin/test_filter_parser.rb +0 -731
  125. data/test/plugin/test_filter_record_transformer.rb +0 -577
  126. data/test/plugin/test_filter_stdout.rb +0 -207
  127. data/test/plugin/test_formatter_csv.rb +0 -136
  128. data/test/plugin/test_formatter_hash.rb +0 -38
  129. data/test/plugin/test_formatter_json.rb +0 -61
  130. data/test/plugin/test_formatter_ltsv.rb +0 -70
  131. data/test/plugin/test_formatter_msgpack.rb +0 -28
  132. data/test/plugin/test_formatter_out_file.rb +0 -116
  133. data/test/plugin/test_formatter_single_value.rb +0 -44
  134. data/test/plugin/test_formatter_tsv.rb +0 -76
  135. data/test/plugin/test_in_debug_agent.rb +0 -49
  136. data/test/plugin/test_in_exec.rb +0 -261
  137. data/test/plugin/test_in_forward.rb +0 -1178
  138. data/test/plugin/test_in_gc_stat.rb +0 -62
  139. data/test/plugin/test_in_http.rb +0 -1124
  140. data/test/plugin/test_in_monitor_agent.rb +0 -922
  141. data/test/plugin/test_in_object_space.rb +0 -66
  142. data/test/plugin/test_in_sample.rb +0 -190
  143. data/test/plugin/test_in_syslog.rb +0 -505
  144. data/test/plugin/test_in_tail.rb +0 -3429
  145. data/test/plugin/test_in_tcp.rb +0 -328
  146. data/test/plugin/test_in_udp.rb +0 -296
  147. data/test/plugin/test_in_unix.rb +0 -181
  148. data/test/plugin/test_input.rb +0 -137
  149. data/test/plugin/test_metadata.rb +0 -89
  150. data/test/plugin/test_metrics.rb +0 -294
  151. data/test/plugin/test_metrics_local.rb +0 -96
  152. data/test/plugin/test_multi_output.rb +0 -204
  153. data/test/plugin/test_out_copy.rb +0 -308
  154. data/test/plugin/test_out_exec.rb +0 -312
  155. data/test/plugin/test_out_exec_filter.rb +0 -606
  156. data/test/plugin/test_out_file.rb +0 -1038
  157. data/test/plugin/test_out_forward.rb +0 -1349
  158. data/test/plugin/test_out_http.rb +0 -557
  159. data/test/plugin/test_out_null.rb +0 -105
  160. data/test/plugin/test_out_relabel.rb +0 -28
  161. data/test/plugin/test_out_roundrobin.rb +0 -146
  162. data/test/plugin/test_out_secondary_file.rb +0 -458
  163. data/test/plugin/test_out_stdout.rb +0 -205
  164. data/test/plugin/test_out_stream.rb +0 -103
  165. data/test/plugin/test_output.rb +0 -1334
  166. data/test/plugin/test_output_as_buffered.rb +0 -2024
  167. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  168. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  169. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  170. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  171. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  172. data/test/plugin/test_output_as_standard.rb +0 -374
  173. data/test/plugin/test_owned_by.rb +0 -34
  174. data/test/plugin/test_parser.rb +0 -399
  175. data/test/plugin/test_parser_apache.rb +0 -42
  176. data/test/plugin/test_parser_apache2.rb +0 -47
  177. data/test/plugin/test_parser_apache_error.rb +0 -45
  178. data/test/plugin/test_parser_csv.rb +0 -200
  179. data/test/plugin/test_parser_json.rb +0 -244
  180. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  181. data/test/plugin/test_parser_msgpack.rb +0 -127
  182. data/test/plugin/test_parser_multiline.rb +0 -111
  183. data/test/plugin/test_parser_nginx.rb +0 -88
  184. data/test/plugin/test_parser_none.rb +0 -52
  185. data/test/plugin/test_parser_regexp.rb +0 -284
  186. data/test/plugin/test_parser_syslog.rb +0 -650
  187. data/test/plugin/test_parser_tsv.rb +0 -122
  188. data/test/plugin/test_sd_file.rb +0 -228
  189. data/test/plugin/test_sd_srv.rb +0 -230
  190. data/test/plugin/test_storage.rb +0 -166
  191. data/test/plugin/test_storage_local.rb +0 -335
  192. data/test/plugin/test_string_util.rb +0 -26
  193. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  195. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  196. data/test/plugin_helper/data/cert/cert.pem +0 -19
  197. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  198. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  199. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  200. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  201. data/test/plugin_helper/data/cert/empty.pem +0 -0
  202. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  203. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  204. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  205. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  206. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  207. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  208. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  209. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  210. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  211. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  212. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  213. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  214. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  215. data/test/plugin_helper/http_server/test_app.rb +0 -65
  216. data/test/plugin_helper/http_server/test_route.rb +0 -32
  217. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  218. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  219. data/test/plugin_helper/test_cert_option.rb +0 -25
  220. data/test/plugin_helper/test_child_process.rb +0 -862
  221. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  222. data/test/plugin_helper/test_event_emitter.rb +0 -80
  223. data/test/plugin_helper/test_event_loop.rb +0 -52
  224. data/test/plugin_helper/test_extract.rb +0 -194
  225. data/test/plugin_helper/test_formatter.rb +0 -255
  226. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  227. data/test/plugin_helper/test_inject.rb +0 -561
  228. data/test/plugin_helper/test_metrics.rb +0 -137
  229. data/test/plugin_helper/test_parser.rb +0 -264
  230. data/test/plugin_helper/test_record_accessor.rb +0 -238
  231. data/test/plugin_helper/test_retry_state.rb +0 -1006
  232. data/test/plugin_helper/test_server.rb +0 -1895
  233. data/test/plugin_helper/test_service_discovery.rb +0 -165
  234. data/test/plugin_helper/test_socket.rb +0 -146
  235. data/test/plugin_helper/test_storage.rb +0 -542
  236. data/test/plugin_helper/test_thread.rb +0 -164
  237. data/test/plugin_helper/test_timer.rb +0 -130
  238. data/test/scripts/exec_script.rb +0 -32
  239. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  240. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  241. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  242. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  243. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  244. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  245. data/test/test_capability.rb +0 -74
  246. data/test/test_clock.rb +0 -164
  247. data/test/test_config.rb +0 -369
  248. data/test/test_configdsl.rb +0 -148
  249. data/test/test_daemonizer.rb +0 -91
  250. data/test/test_engine.rb +0 -203
  251. data/test/test_event.rb +0 -531
  252. data/test/test_event_router.rb +0 -348
  253. data/test/test_event_time.rb +0 -199
  254. data/test/test_file_wrapper.rb +0 -53
  255. data/test/test_filter.rb +0 -121
  256. data/test/test_fluent_log_event_router.rb +0 -99
  257. data/test/test_formatter.rb +0 -369
  258. data/test/test_input.rb +0 -31
  259. data/test/test_log.rb +0 -1076
  260. data/test/test_match.rb +0 -148
  261. data/test/test_mixin.rb +0 -351
  262. data/test/test_msgpack_factory.rb +0 -50
  263. data/test/test_oj_options.rb +0 -55
  264. data/test/test_output.rb +0 -278
  265. data/test/test_plugin.rb +0 -251
  266. data/test/test_plugin_classes.rb +0 -370
  267. data/test/test_plugin_helper.rb +0 -81
  268. data/test/test_plugin_id.rb +0 -119
  269. data/test/test_process.rb +0 -14
  270. data/test/test_root_agent.rb +0 -951
  271. data/test/test_static_config_analysis.rb +0 -177
  272. data/test/test_supervisor.rb +0 -821
  273. data/test/test_test_drivers.rb +0 -136
  274. data/test/test_time_formatter.rb +0 -301
  275. data/test/test_time_parser.rb +0 -362
  276. data/test/test_tls.rb +0 -65
  277. data/test/test_unique_id.rb +0 -47
  278. data/test/test_variable_store.rb +0 -65
@@ -84,6 +84,8 @@ module Fluent::Plugin
84
84
  config_param :dump_error_log, :bool, default: true
85
85
  desc 'Add QUERY_ prefix query params to record'
86
86
  config_param :add_query_params, :bool, default: false
87
+ desc "Add prefix to incoming tag"
88
+ config_param :add_tag_prefix, :string, default: nil
87
89
 
88
90
  config_section :parse do
89
91
  config_set_default :@type, 'in_http'
@@ -120,6 +122,8 @@ module Fluent::Plugin
120
122
  end
121
123
  end
122
124
 
125
+ raise Fluent::ConfigError, "'add_tag_prefix' parameter must not be empty" if @add_tag_prefix && @add_tag_prefix.empty?
126
+
123
127
  m = if @parser_configs.first['@type'] == 'in_http'
124
128
  @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
125
129
  @parser_msgpack.time_key = nil
@@ -203,6 +207,7 @@ module Fluent::Plugin
203
207
  begin
204
208
  path = path_info[1..-1] # remove /
205
209
  tag = path.split('/').join('.')
210
+ tag = "#{@add_tag_prefix}.#{tag}" if @add_tag_prefix
206
211
 
207
212
  mes = Fluent::MultiEventStream.new
208
213
  parse_params(params) do |record_time, record|
@@ -448,7 +453,7 @@ module Fluent::Plugin
448
453
  # Azure App Service sends GET requests for health checking purpose.
449
454
  # Respond with `200 OK` to accommodate it.
450
455
  def handle_get_request
451
- return send_response_and_close(RES_200_STATUS, {}, "")
456
+ return send_response_and_close(RES_200_STATUS, {}, "")
452
457
  end
453
458
 
454
459
  # Web browsers can send an OPTIONS request before performing POST
@@ -39,6 +39,8 @@ module Fluent::Plugin
39
39
  config_param :auto_increment_key, :string, default: nil
40
40
  desc "The boolean to suspend-and-resume incremental value after restart"
41
41
  config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
42
+ desc "Reuse the sample data to reduce the load when sending large amounts of data. You can enable it if filter does not do destructive change"
43
+ config_param :reuse_record, :bool, default: false
42
44
  desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
43
45
  config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
44
46
  begin
@@ -117,15 +119,19 @@ module Fluent::Plugin
117
119
  end
118
120
  end
119
121
 
120
- def generate
121
- d = @sample[@sample_index]
122
- unless d
123
- @sample_index = 0
124
- d = @sample[@sample_index]
125
- end
122
+ def next_sample
123
+ d = @reuse_record ? @sample[@sample_index] : @sample[@sample_index].dup
126
124
  @sample_index += 1
125
+ return d if d
126
+
127
+ @sample_index = 0
128
+ next_sample
129
+ end
130
+
131
+ def generate
132
+ d = next_sample
127
133
  if @auto_increment_key
128
- d = d.dup
134
+ d = d.dup if @reuse_record
129
135
  d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
130
136
  end
131
137
  d
@@ -156,6 +156,10 @@ module Fluent::Plugin
156
156
  true
157
157
  end
158
158
 
159
+ def zero_downtime_restart_ready?
160
+ true
161
+ end
162
+
159
163
  def start
160
164
  super
161
165
 
@@ -36,7 +36,7 @@ module Fluent::Plugin
36
36
  helpers :timer, :event_loop, :parser, :compat_parameters
37
37
 
38
38
  RESERVED_CHARS = ['/', '*', '%'].freeze
39
- MetricsInfo = Struct.new(:opened, :closed, :rotated)
39
+ MetricsInfo = Struct.new(:opened, :closed, :rotated, :throttled)
40
40
 
41
41
  class WatcherSetupError < StandardError
42
42
  def initialize(msg)
@@ -208,7 +208,8 @@ module Fluent::Plugin
208
208
  opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files")
209
209
  closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files")
210
210
  rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files")
211
- @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
211
+ throttling_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_throttled_total", help_text: "Total number of times throttling occurs per file when throttling enabled")
212
+ @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics, throttling_metrics)
212
213
  end
213
214
 
214
215
  def configure_tag
@@ -566,7 +567,7 @@ module Fluent::Plugin
566
567
  if @follow_inodes && new_inode.nil?
567
568
  # nil inode means the file disappeared, so we only need to stop it.
568
569
  @tails.delete(tail_watcher.path)
569
- # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
570
+ # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
570
571
  # Because of this problem, log duplication can occur during `rotate_wait`.
571
572
  # Need to set `rotate_wait 0` for a workaround.
572
573
  # Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.
@@ -696,14 +697,6 @@ module Fluent::Plugin
696
697
 
697
698
  # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
698
699
  def receive_lines(lines, tail_watcher)
699
- lines = lines.reject do |line|
700
- skip_line = @max_line_size ? line.bytesize > @max_line_size : false
701
- if skip_line
702
- log.warn "received line length is longer than #{@max_line_size}"
703
- log.debug "skipped line: #{line.chomp}"
704
- end
705
- skip_line
706
- end
707
700
  es = @receive_handler.call(lines, tail_watcher)
708
701
  unless es.empty?
709
702
  tag = if @tag_prefix || @tag_suffix
@@ -801,6 +794,7 @@ module Fluent::Plugin
801
794
  'opened_file_count' => @metrics.opened.get,
802
795
  'closed_file_count' => @metrics.closed.get,
803
796
  'rotated_file_count' => @metrics.rotated.get,
797
+ 'throttled_log_count' => @metrics.throttled.get,
804
798
  })
805
799
  }
806
800
  stats
@@ -819,6 +813,7 @@ module Fluent::Plugin
819
813
  from_encoding: @from_encoding,
820
814
  encoding: @encoding,
821
815
  metrics: @metrics,
816
+ max_line_size: @max_line_size,
822
817
  &method(:receive_lines)
823
818
  )
824
819
  end
@@ -1011,15 +1006,19 @@ module Fluent::Plugin
1011
1006
  end
1012
1007
 
1013
1008
  class FIFO
1014
- def initialize(from_encoding, encoding)
1009
+ def initialize(from_encoding, encoding, log, max_line_size=nil)
1015
1010
  @from_encoding = from_encoding
1016
1011
  @encoding = encoding
1017
1012
  @need_enc = from_encoding != encoding
1018
1013
  @buffer = ''.force_encoding(from_encoding)
1019
1014
  @eol = "\n".encode(from_encoding).freeze
1015
+ @max_line_size = max_line_size
1016
+ @skip_current_line = false
1017
+ @skipping_current_line_bytesize = 0
1018
+ @log = log
1020
1019
  end
1021
1020
 
1022
- attr_reader :from_encoding, :encoding, :buffer
1021
+ attr_reader :from_encoding, :encoding, :buffer, :max_line_size
1023
1022
 
1024
1023
  def <<(chunk)
1025
1024
  # Although "chunk" is most likely transient besides String#force_encoding itself
@@ -1051,6 +1050,7 @@ module Fluent::Plugin
1051
1050
 
1052
1051
  def read_lines(lines)
1053
1052
  idx = @buffer.index(@eol)
1053
+ has_skipped_line = false
1054
1054
 
1055
1055
  until idx.nil?
1056
1056
  # Using freeze and slice is faster than slice!
@@ -1059,11 +1059,47 @@ module Fluent::Plugin
1059
1059
  rbuf = @buffer.slice(0, idx + 1)
1060
1060
  @buffer = @buffer.slice(idx + 1, @buffer.size)
1061
1061
  idx = @buffer.index(@eol)
1062
+
1063
+ is_long_line = @max_line_size && (
1064
+ @skip_current_line || rbuf.bytesize > @max_line_size
1065
+ )
1066
+
1067
+ if is_long_line
1068
+ @log.warn "received line length is longer than #{@max_line_size}"
1069
+ if @skip_current_line
1070
+ @log.debug("The continuing line is finished. Finally discarded data: ") { convert(rbuf).chomp }
1071
+ else
1072
+ @log.debug("skipped line: ") { convert(rbuf).chomp }
1073
+ end
1074
+ has_skipped_line = true
1075
+ @skip_current_line = false
1076
+ @skipping_current_line_bytesize = 0
1077
+ next
1078
+ end
1079
+
1062
1080
  lines << convert(rbuf)
1063
1081
  end
1082
+
1083
+ is_long_current_line = @max_line_size && (
1084
+ @skip_current_line || @buffer.bytesize > @max_line_size
1085
+ )
1086
+
1087
+ if is_long_current_line
1088
+ @log.debug(
1089
+ "The continuing current line length is longer than #{@max_line_size}." +
1090
+ " The received data will be discarded until this line is finished." +
1091
+ " Discarded data: "
1092
+ ) { convert(@buffer).chomp }
1093
+ @skip_current_line = true
1094
+ @skipping_current_line_bytesize += @buffer.bytesize
1095
+ @buffer.clear
1096
+ end
1097
+
1098
+ return has_skipped_line
1064
1099
  end
1065
1100
 
1066
- def bytesize
1101
+ def reading_bytesize
1102
+ return @skipping_current_line_bytesize if @skip_current_line
1067
1103
  @buffer.bytesize
1068
1104
  end
1069
1105
  end
@@ -1074,14 +1110,14 @@ module Fluent::Plugin
1074
1110
 
1075
1111
  attr_accessor :shutdown_timeout
1076
1112
 
1077
- def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
1113
+ def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, max_line_size: nil, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
1078
1114
  @watcher = watcher
1079
1115
  @path = path
1080
1116
  @read_lines_limit = read_lines_limit
1081
1117
  @read_bytes_limit_per_second = read_bytes_limit_per_second
1082
1118
  @receive_lines = receive_lines
1083
1119
  @open_on_every_update = open_on_every_update
1084
- @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT)
1120
+ @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT, log, max_line_size)
1085
1121
  @iobuf = ''.force_encoding('ASCII-8BIT')
1086
1122
  @lines = []
1087
1123
  @io = nil
@@ -1158,12 +1194,15 @@ module Fluent::Plugin
1158
1194
  end
1159
1195
 
1160
1196
  def handle_notify
1161
- return if limit_bytes_per_second_reached?
1162
- return if group_watcher&.limit_lines_reached?(@path)
1197
+ if limit_bytes_per_second_reached? || group_watcher&.limit_lines_reached?(@path)
1198
+ @metrics.throttled.inc
1199
+ return
1200
+ end
1163
1201
 
1164
1202
  with_io do |io|
1165
1203
  begin
1166
1204
  read_more = false
1205
+ has_skipped_line = false
1167
1206
 
1168
1207
  if !io.nil? && @lines.empty?
1169
1208
  begin
@@ -1177,7 +1216,7 @@ module Fluent::Plugin
1177
1216
  @fifo << data
1178
1217
 
1179
1218
  n_lines_before_read = @lines.size
1180
- @fifo.read_lines(@lines)
1219
+ has_skipped_line = @fifo.read_lines(@lines) || has_skipped_line
1181
1220
  group_watcher&.update_lines_read(@path, @lines.size - n_lines_before_read)
1182
1221
 
1183
1222
  group_watcher_limit = group_watcher&.limit_lines_reached?(@path)
@@ -1185,6 +1224,7 @@ module Fluent::Plugin
1185
1224
 
1186
1225
  if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now?
1187
1226
  # Just get out from tailing loop.
1227
+ @metrics.throttled.inc if group_watcher_limit || limit_bytes_per_second_reached?
1188
1228
  read_more = false
1189
1229
  break
1190
1230
  end
@@ -1200,9 +1240,11 @@ module Fluent::Plugin
1200
1240
  end
1201
1241
  end
1202
1242
 
1203
- unless @lines.empty?
1243
+ if @lines.empty?
1244
+ @watcher.pe.update_pos(io.pos - @fifo.reading_bytesize) if has_skipped_line
1245
+ else
1204
1246
  if @receive_lines.call(@lines, @watcher)
1205
- @watcher.pe.update_pos(io.pos - @fifo.bytesize)
1247
+ @watcher.pe.update_pos(io.pos - @fifo.reading_bytesize)
1206
1248
  @lines.clear
1207
1249
  else
1208
1250
  read_more = false
@@ -1214,12 +1256,12 @@ module Fluent::Plugin
1214
1256
 
1215
1257
  def open
1216
1258
  io = Fluent::FileWrapper.open(@path)
1217
- io.seek(@watcher.pe.read_pos + @fifo.bytesize)
1259
+ io.seek(@watcher.pe.read_pos + @fifo.reading_bytesize)
1218
1260
  @metrics.opened.inc
1219
1261
  io
1220
1262
  rescue RangeError
1221
1263
  io.close if io
1222
- raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
1264
+ raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.reading_bytesize.to_s(16)}"
1223
1265
  rescue Errno::EACCES => e
1224
1266
  @log.warn "#{e}"
1225
1267
  nil
@@ -101,6 +101,10 @@ module Fluent::Plugin
101
101
  true
102
102
  end
103
103
 
104
+ def zero_downtime_restart_ready?
105
+ true
106
+ end
107
+
104
108
  def start
105
109
  super
106
110
 
@@ -43,10 +43,15 @@ module Fluent::Plugin
43
43
  desc "Remove newline from the end of incoming payload"
44
44
  config_param :remove_newline, :bool, default: true
45
45
  desc "The max size of socket receive buffer. SO_RCVBUF"
46
- config_param :receive_buffer_size, :size, default: nil
46
+ config_param :receive_buffer_size, :size, default: nil, deprecated: "use receive_buffer_size in transport section instead."
47
47
 
48
48
  config_param :blocking_timeout, :time, default: 0.5
49
49
 
50
+ # overwrite server plugin to change default to :udp and remove tcp/tls protocol from list
51
+ config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
52
+ config_argument :protocol, :enum, list: [:udp], default: :udp
53
+ end
54
+
50
55
  def configure(conf)
51
56
  compat_parameters_convert(conf, :parser)
52
57
  parser_config = conf.elements('parse').first
@@ -65,6 +70,10 @@ module Fluent::Plugin
65
70
  true
66
71
  end
67
72
 
73
+ def zero_downtime_restart_ready?
74
+ true
75
+ end
76
+
68
77
  def start
69
78
  super
70
79
 
@@ -70,6 +70,10 @@ module Fluent
70
70
  def multi_workers_ready?
71
71
  false
72
72
  end
73
+
74
+ def zero_downtime_restart_ready?
75
+ false
76
+ end
73
77
  end
74
78
  end
75
79
  end
@@ -0,0 +1,40 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/output'
18
+
19
+ module Fluent::Plugin
20
+ class BufferOutput < Output
21
+ Fluent::Plugin.register_output("buffer", self)
22
+ helpers :event_emitter
23
+
24
+ config_section :buffer do
25
+ config_set_default :@type, "file"
26
+ config_set_default :chunk_keys, ["tag"]
27
+ config_set_default :flush_mode, :interval
28
+ config_set_default :flush_interval, 10
29
+ end
30
+
31
+ def multi_workers_ready?
32
+ true
33
+ end
34
+
35
+ def write(chunk)
36
+ return if chunk.empty?
37
+ router.emit_stream(chunk.metadata.tag, Fluent::MessagePackEventStream.new(chunk.read))
38
+ end
39
+ end
40
+ end
@@ -89,7 +89,7 @@ module Fluent::Plugin
89
89
 
90
90
  case @copy_mode
91
91
  when :no_copy
92
- nil
92
+ nil
93
93
  when :shallow
94
94
  Proc.new { |es| es.dup }
95
95
  when :deep
@@ -172,6 +172,14 @@ module Fluent::Plugin
172
172
  log.warn "symlink_path is unavailable on Windows platform. disabled."
173
173
  @symlink_path = nil
174
174
  else
175
+ placeholder_validators(:symlink_path, @symlink_path).reject{ |v| v.type == :time }.each do |v|
176
+ begin
177
+ v.validate!
178
+ rescue Fluent::ConfigError => e
179
+ log.warn "#{e}. This means multiple chunks are competing for a single symlink_path, so some logs may not be taken from the symlink."
180
+ end
181
+ end
182
+
175
183
  @buffer.extend SymlinkBufferMixin
176
184
  @buffer.symlink_path = @symlink_path
177
185
  @buffer.output_plugin_for_symlink = self
@@ -55,6 +55,8 @@ module Fluent::Plugin
55
55
  config_param :headers, :hash, default: nil
56
56
  desc 'Additional placeholder based headers for HTTP request'
57
57
  config_param :headers_from_placeholders, :hash, default: nil
58
+ desc 'Compress HTTP request body'
59
+ config_param :compress, :enum, list: [:text, :gzip], default: :text
58
60
 
59
61
  desc 'The connection open timeout in seconds'
60
62
  config_param :open_timeout, :integer, default: nil
@@ -290,6 +292,9 @@ module Fluent::Plugin
290
292
  req[k] = extract_placeholders(v, chunk)
291
293
  end
292
294
  end
295
+ if @compress == :gzip
296
+ req['Content-Encoding'] = "gzip"
297
+ end
293
298
  req['Content-Type'] = @content_type
294
299
  end
295
300
 
@@ -323,8 +328,15 @@ module Fluent::Plugin
323
328
  Net::HTTP::Put.new(uri.request_uri)
324
329
  end
325
330
  set_headers(req, uri, chunk)
331
+
326
332
  req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
327
333
 
334
+ if @compress == :gzip
335
+ gz = Zlib::GzipWriter.new(StringIO.new)
336
+ gz << req.body
337
+ req.body = gz.close.string
338
+ end
339
+
328
340
  # At least one authentication method requires the body and other headers, so the order of this call matters
329
341
  set_auth(req, uri)
330
342
  req
@@ -1384,6 +1384,7 @@ module Fluent
1384
1384
  end
1385
1385
 
1386
1386
  def submit_flush_once
1387
+ return unless @buffer_config.flush_thread_count > 0
1387
1388
  # Without locks: it is rough but enough to select "next" writer selection
1388
1389
  @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count
1389
1390
  state = @output_flush_threads[@output_flush_thread_current_position]
@@ -1406,6 +1407,7 @@ module Fluent
1406
1407
  end
1407
1408
 
1408
1409
  def submit_flush_all
1410
+ return unless @buffer_config.flush_thread_count > 0
1409
1411
  while !@retry && @buffer.queued?
1410
1412
  submit_flush_once
1411
1413
  sleep @buffer_config.flush_thread_burst_interval
@@ -50,23 +50,15 @@ module Fluent
50
50
  def configure_json_parser(name)
51
51
  case name
52
52
  when :oj
53
- raise LoadError unless Fluent::OjOptions.available?
54
- [Oj.method(:load), Oj::ParseError]
53
+ return [Oj.method(:load), Oj::ParseError] if Fluent::OjOptions.available?
54
+
55
+ log&.info "Oj is not installed, and failing back to Yajl for json parser"
56
+ configure_json_parser(:yajl)
55
57
  when :json then [JSON.method(:load), JSON::ParserError]
56
58
  when :yajl then [Yajl.method(:load), Yajl::ParseError]
57
59
  else
58
60
  raise "BUG: unknown json parser specified: #{name}"
59
61
  end
60
- rescue LoadError => ex
61
- name = :yajl
62
- if log
63
- if /\boj\z/.match?(ex.message)
64
- log.info "Oj is not installed, and failing back to Yajl for json parser"
65
- else
66
- log.warn ex.message
67
- end
68
- end
69
- retry
70
62
  end
71
63
 
72
64
  def parse(text)
@@ -33,6 +33,14 @@ module Fluent
33
33
 
34
34
  if conf.client_cert_auth
35
35
  ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
36
+ else
37
+ ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
38
+ end
39
+
40
+ if conf.ensure_fips
41
+ unless OpenSSL.fips_mode
42
+ raise Fluent::ConfigError, "Cannot enable FIPS compliant mode. OpenSSL FIPS configuration is disabled"
43
+ end
36
44
  end
37
45
 
38
46
  ctx.ca_file = conf.ca_path
@@ -26,6 +26,9 @@ module Fluent
26
26
 
27
27
  def router
28
28
  @_event_emitter_used_actually = true
29
+
30
+ return Engine.root_agent.source_only_router if @_event_emitter_force_source_only_router
31
+
29
32
  if @_event_emitter_lazy_init
30
33
  @router = @primary_instance.router
31
34
  end
@@ -48,6 +51,14 @@ module Fluent
48
51
  @_event_emitter_used_actually
49
52
  end
50
53
 
54
+ def event_emitter_apply_source_only
55
+ @_event_emitter_force_source_only_router = true
56
+ end
57
+
58
+ def event_emitter_cancel_source_only
59
+ @_event_emitter_force_source_only_router = false
60
+ end
61
+
51
62
  def event_emitter_router(label_name)
52
63
  if label_name
53
64
  if label_name == "@ROOT"
@@ -72,6 +83,7 @@ module Fluent
72
83
  super
73
84
  @_event_emitter_used_actually = false
74
85
  @_event_emitter_lazy_init = false
86
+ @_event_emitter_force_source_only_router = false
75
87
  @router = nil
76
88
  end
77
89
 
@@ -39,7 +39,8 @@ module Fluent
39
39
  scheme = tls_context ? 'https' : 'http'
40
40
  @uri = URI("#{scheme}://#{@addr}:#{@port}").to_s
41
41
  @router = Router.new(default_app)
42
- @reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(@logger))
42
+ @server_task = nil
43
+ Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)
43
44
 
44
45
  opts = if tls_context
45
46
  { ssl_context: tls_context }
@@ -54,31 +55,42 @@ module Fluent
54
55
  end
55
56
 
56
57
  def start(notify = nil)
58
+ Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)
57
59
  @logger.debug("Start async HTTP server listening #{@uri}")
58
- task = @reactor.run do
59
- @server.run
60
60
 
61
+ Async do |task|
62
+ Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)
63
+ @server_task = task.async do
64
+ Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)
65
+ @server.run
66
+ end
61
67
  if notify
62
68
  notify.push(:ready)
63
69
  end
70
+
71
+ if async_v2?
72
+ @server_task_queue = ::Thread::Queue.new
73
+ @server_task_queue.pop
74
+ @server_task&.stop
75
+ end
64
76
  end
65
77
 
66
- task.stop
67
78
  @logger.debug('Finished HTTP server')
68
79
  end
69
80
 
70
81
  def stop
71
82
  @logger.debug('closing HTTP server')
72
-
73
- if @reactor
74
- @reactor.stop
83
+ if async_v2?
84
+ @server_task_queue&.push(:stop)
85
+ else
86
+ @server_task&.stop
75
87
  end
76
88
  end
77
89
 
78
90
  HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
79
91
  define_method(name) do |path, app = nil, &block|
80
92
  unless path.end_with?('/')
81
- path << '/'
93
+ path += '/'
82
94
  end
83
95
 
84
96
  if (block && app) || (!block && !app)
@@ -88,6 +100,10 @@ module Fluent
88
100
  @router.mount(name, path, app || block)
89
101
  end
90
102
  end
103
+
104
+ private def async_v2?
105
+ Gem::Version.new(Async::VERSION) >= Gem::Version.new('2.0')
106
+ end
91
107
  end
92
108
  end
93
109
  end
@@ -84,6 +84,8 @@ module Fluent
84
84
  socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
85
85
  end
86
86
 
87
+ socket_options[:receive_buffer_size] ||= @transport_config&.receive_buffer_size
88
+
87
89
  socket_option_validate!(proto, **socket_options)
88
90
  socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
89
91
 
@@ -136,6 +138,8 @@ module Fluent
136
138
  socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
137
139
  end
138
140
 
141
+ socket_options[:receive_buffer_size] ||= @transport_config&.receive_buffer_size
142
+
139
143
  unless socket
140
144
  socket_option_validate!(proto, **socket_options)
141
145
  socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
@@ -247,6 +251,7 @@ module Fluent
247
251
  :generate_cert_country, :generate_cert_state, :generate_cert_state,
248
252
  :generate_cert_locality, :generate_cert_common_name,
249
253
  :generate_cert_expiration, :generate_cert_digest,
254
+ :ensure_fips,
250
255
  ]
251
256
 
252
257
  def server_create_transport_section_object(opts)
@@ -266,6 +271,9 @@ module Fluent
266
271
 
267
272
  ### Socket Params ###
268
273
 
274
+ desc "The max size of socket receive buffer. SO_RCVBUF"
275
+ config_param :receive_buffer_size, :size, default: nil
276
+
269
277
  # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src.
270
278
  # Set positive value if needing to send FIN on closing on non-Windows.
271
279
  # (On Windows, Fluentd can send FIN with zero `linger_timeout` since Fluentd doesn't set 0 to SO_LINGER on Windows.
@@ -287,6 +295,7 @@ module Fluent
287
295
  config_param :max_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
288
296
  config_param :ciphers, :string, default: Fluent::TLS::CIPHERS_DEFAULT
289
297
  config_param :insecure, :bool, default: false
298
+ config_param :ensure_fips, :bool, default: false
290
299
 
291
300
  # Cert signed by public CA
292
301
  config_param :ca_path, :string, default: nil