fluentd 1.17.0-x86-mingw32 → 1.18.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (278) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +100 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/command/fluentd.rb +7 -1
  8. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  9. data/lib/fluent/compat/propagate_default.rb +4 -4
  10. data/lib/fluent/config/literal_parser.rb +9 -2
  11. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  12. data/lib/fluent/engine.rb +49 -33
  13. data/lib/fluent/env.rb +3 -0
  14. data/lib/fluent/event_router.rb +2 -2
  15. data/lib/fluent/log/console_adapter.rb +4 -2
  16. data/lib/fluent/plugin/filter_parser.rb +27 -51
  17. data/lib/fluent/plugin/in_exec.rb +14 -2
  18. data/lib/fluent/plugin/in_http.rb +6 -1
  19. data/lib/fluent/plugin/in_sample.rb +13 -7
  20. data/lib/fluent/plugin/in_syslog.rb +4 -0
  21. data/lib/fluent/plugin/in_tail.rb +65 -23
  22. data/lib/fluent/plugin/in_tcp.rb +4 -0
  23. data/lib/fluent/plugin/in_udp.rb +10 -1
  24. data/lib/fluent/plugin/input.rb +4 -0
  25. data/lib/fluent/plugin/out_buffer.rb +40 -0
  26. data/lib/fluent/plugin/out_copy.rb +1 -1
  27. data/lib/fluent/plugin/out_file.rb +8 -0
  28. data/lib/fluent/plugin/out_http.rb +12 -0
  29. data/lib/fluent/plugin/output.rb +2 -0
  30. data/lib/fluent/plugin/parser_json.rb +4 -12
  31. data/lib/fluent/plugin_helper/cert_option.rb +8 -0
  32. data/lib/fluent/plugin_helper/event_emitter.rb +12 -0
  33. data/lib/fluent/plugin_helper/http_server/server.rb +24 -8
  34. data/lib/fluent/plugin_helper/server.rb +9 -0
  35. data/lib/fluent/root_agent.rb +114 -19
  36. data/lib/fluent/source_only_buffer_agent.rb +102 -0
  37. data/lib/fluent/supervisor.rb +207 -34
  38. data/lib/fluent/system_config.rb +15 -3
  39. data/lib/fluent/version.rb +1 -1
  40. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  41. metadata +24 -483
  42. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  43. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  44. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  45. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  46. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  47. data/.github/ISSUE_TEMPLATE.md +0 -17
  48. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  49. data/.github/workflows/stale-actions.yml +0 -24
  50. data/.github/workflows/test-ruby-head.yml +0 -31
  51. data/.github/workflows/test.yml +0 -32
  52. data/.gitignore +0 -30
  53. data/Gemfile +0 -9
  54. data/fluentd.gemspec +0 -62
  55. data/test/command/test_binlog_reader.rb +0 -362
  56. data/test/command/test_ca_generate.rb +0 -70
  57. data/test/command/test_cap_ctl.rb +0 -100
  58. data/test/command/test_cat.rb +0 -128
  59. data/test/command/test_ctl.rb +0 -56
  60. data/test/command/test_fluentd.rb +0 -1291
  61. data/test/command/test_plugin_config_formatter.rb +0 -397
  62. data/test/command/test_plugin_generator.rb +0 -109
  63. data/test/compat/test_calls_super.rb +0 -166
  64. data/test/compat/test_parser.rb +0 -92
  65. data/test/config/assertions.rb +0 -42
  66. data/test/config/test_config_parser.rb +0 -551
  67. data/test/config/test_configurable.rb +0 -1784
  68. data/test/config/test_configure_proxy.rb +0 -604
  69. data/test/config/test_dsl.rb +0 -415
  70. data/test/config/test_element.rb +0 -518
  71. data/test/config/test_literal_parser.rb +0 -309
  72. data/test/config/test_plugin_configuration.rb +0 -56
  73. data/test/config/test_section.rb +0 -191
  74. data/test/config/test_system_config.rb +0 -195
  75. data/test/config/test_types.rb +0 -408
  76. data/test/counter/test_client.rb +0 -563
  77. data/test/counter/test_error.rb +0 -44
  78. data/test/counter/test_mutex_hash.rb +0 -179
  79. data/test/counter/test_server.rb +0 -589
  80. data/test/counter/test_store.rb +0 -258
  81. data/test/counter/test_validator.rb +0 -137
  82. data/test/helper.rb +0 -155
  83. data/test/helpers/fuzzy_assert.rb +0 -89
  84. data/test/helpers/process_extenstion.rb +0 -33
  85. data/test/log/test_console_adapter.rb +0 -117
  86. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  87. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  88. data/test/plugin/data/2010/01/20100102.log +0 -0
  89. data/test/plugin/data/log/bar +0 -0
  90. data/test/plugin/data/log/foo/bar.log +0 -0
  91. data/test/plugin/data/log/foo/bar2 +0 -0
  92. data/test/plugin/data/log/test.log +0 -0
  93. data/test/plugin/data/log_numeric/01.log +0 -0
  94. data/test/plugin/data/log_numeric/02.log +0 -0
  95. data/test/plugin/data/log_numeric/12.log +0 -0
  96. data/test/plugin/data/log_numeric/14.log +0 -0
  97. data/test/plugin/data/sd_file/config +0 -11
  98. data/test/plugin/data/sd_file/config.json +0 -17
  99. data/test/plugin/data/sd_file/config.yaml +0 -11
  100. data/test/plugin/data/sd_file/config.yml +0 -11
  101. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  102. data/test/plugin/in_tail/test_fifo.rb +0 -121
  103. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  104. data/test/plugin/in_tail/test_position_file.rb +0 -346
  105. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  106. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  107. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  108. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  109. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  110. data/test/plugin/test_bare_output.rb +0 -131
  111. data/test/plugin/test_base.rb +0 -247
  112. data/test/plugin/test_buf_file.rb +0 -1314
  113. data/test/plugin/test_buf_file_single.rb +0 -898
  114. data/test/plugin/test_buf_memory.rb +0 -42
  115. data/test/plugin/test_buffer.rb +0 -1493
  116. data/test/plugin/test_buffer_chunk.rb +0 -209
  117. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  118. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  119. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  120. data/test/plugin/test_compressable.rb +0 -87
  121. data/test/plugin/test_file_util.rb +0 -96
  122. data/test/plugin/test_filter.rb +0 -368
  123. data/test/plugin/test_filter_grep.rb +0 -697
  124. data/test/plugin/test_filter_parser.rb +0 -731
  125. data/test/plugin/test_filter_record_transformer.rb +0 -577
  126. data/test/plugin/test_filter_stdout.rb +0 -207
  127. data/test/plugin/test_formatter_csv.rb +0 -136
  128. data/test/plugin/test_formatter_hash.rb +0 -38
  129. data/test/plugin/test_formatter_json.rb +0 -61
  130. data/test/plugin/test_formatter_ltsv.rb +0 -70
  131. data/test/plugin/test_formatter_msgpack.rb +0 -28
  132. data/test/plugin/test_formatter_out_file.rb +0 -116
  133. data/test/plugin/test_formatter_single_value.rb +0 -44
  134. data/test/plugin/test_formatter_tsv.rb +0 -76
  135. data/test/plugin/test_in_debug_agent.rb +0 -49
  136. data/test/plugin/test_in_exec.rb +0 -261
  137. data/test/plugin/test_in_forward.rb +0 -1178
  138. data/test/plugin/test_in_gc_stat.rb +0 -62
  139. data/test/plugin/test_in_http.rb +0 -1124
  140. data/test/plugin/test_in_monitor_agent.rb +0 -922
  141. data/test/plugin/test_in_object_space.rb +0 -66
  142. data/test/plugin/test_in_sample.rb +0 -190
  143. data/test/plugin/test_in_syslog.rb +0 -505
  144. data/test/plugin/test_in_tail.rb +0 -3429
  145. data/test/plugin/test_in_tcp.rb +0 -328
  146. data/test/plugin/test_in_udp.rb +0 -296
  147. data/test/plugin/test_in_unix.rb +0 -181
  148. data/test/plugin/test_input.rb +0 -137
  149. data/test/plugin/test_metadata.rb +0 -89
  150. data/test/plugin/test_metrics.rb +0 -294
  151. data/test/plugin/test_metrics_local.rb +0 -96
  152. data/test/plugin/test_multi_output.rb +0 -204
  153. data/test/plugin/test_out_copy.rb +0 -308
  154. data/test/plugin/test_out_exec.rb +0 -312
  155. data/test/plugin/test_out_exec_filter.rb +0 -606
  156. data/test/plugin/test_out_file.rb +0 -1038
  157. data/test/plugin/test_out_forward.rb +0 -1349
  158. data/test/plugin/test_out_http.rb +0 -557
  159. data/test/plugin/test_out_null.rb +0 -105
  160. data/test/plugin/test_out_relabel.rb +0 -28
  161. data/test/plugin/test_out_roundrobin.rb +0 -146
  162. data/test/plugin/test_out_secondary_file.rb +0 -458
  163. data/test/plugin/test_out_stdout.rb +0 -205
  164. data/test/plugin/test_out_stream.rb +0 -103
  165. data/test/plugin/test_output.rb +0 -1334
  166. data/test/plugin/test_output_as_buffered.rb +0 -2024
  167. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  168. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  169. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  170. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  171. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  172. data/test/plugin/test_output_as_standard.rb +0 -374
  173. data/test/plugin/test_owned_by.rb +0 -34
  174. data/test/plugin/test_parser.rb +0 -399
  175. data/test/plugin/test_parser_apache.rb +0 -42
  176. data/test/plugin/test_parser_apache2.rb +0 -47
  177. data/test/plugin/test_parser_apache_error.rb +0 -45
  178. data/test/plugin/test_parser_csv.rb +0 -200
  179. data/test/plugin/test_parser_json.rb +0 -244
  180. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  181. data/test/plugin/test_parser_msgpack.rb +0 -127
  182. data/test/plugin/test_parser_multiline.rb +0 -111
  183. data/test/plugin/test_parser_nginx.rb +0 -88
  184. data/test/plugin/test_parser_none.rb +0 -52
  185. data/test/plugin/test_parser_regexp.rb +0 -284
  186. data/test/plugin/test_parser_syslog.rb +0 -650
  187. data/test/plugin/test_parser_tsv.rb +0 -122
  188. data/test/plugin/test_sd_file.rb +0 -228
  189. data/test/plugin/test_sd_srv.rb +0 -230
  190. data/test/plugin/test_storage.rb +0 -166
  191. data/test/plugin/test_storage_local.rb +0 -335
  192. data/test/plugin/test_string_util.rb +0 -26
  193. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  195. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  196. data/test/plugin_helper/data/cert/cert.pem +0 -19
  197. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  198. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  199. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  200. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  201. data/test/plugin_helper/data/cert/empty.pem +0 -0
  202. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  203. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  204. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  205. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  206. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  207. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  208. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  209. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  210. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  211. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  212. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  213. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  214. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  215. data/test/plugin_helper/http_server/test_app.rb +0 -65
  216. data/test/plugin_helper/http_server/test_route.rb +0 -32
  217. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  218. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  219. data/test/plugin_helper/test_cert_option.rb +0 -25
  220. data/test/plugin_helper/test_child_process.rb +0 -862
  221. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  222. data/test/plugin_helper/test_event_emitter.rb +0 -80
  223. data/test/plugin_helper/test_event_loop.rb +0 -52
  224. data/test/plugin_helper/test_extract.rb +0 -194
  225. data/test/plugin_helper/test_formatter.rb +0 -255
  226. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  227. data/test/plugin_helper/test_inject.rb +0 -561
  228. data/test/plugin_helper/test_metrics.rb +0 -137
  229. data/test/plugin_helper/test_parser.rb +0 -264
  230. data/test/plugin_helper/test_record_accessor.rb +0 -238
  231. data/test/plugin_helper/test_retry_state.rb +0 -1006
  232. data/test/plugin_helper/test_server.rb +0 -1895
  233. data/test/plugin_helper/test_service_discovery.rb +0 -165
  234. data/test/plugin_helper/test_socket.rb +0 -146
  235. data/test/plugin_helper/test_storage.rb +0 -542
  236. data/test/plugin_helper/test_thread.rb +0 -164
  237. data/test/plugin_helper/test_timer.rb +0 -130
  238. data/test/scripts/exec_script.rb +0 -32
  239. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  240. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  241. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  242. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  243. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  244. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  245. data/test/test_capability.rb +0 -74
  246. data/test/test_clock.rb +0 -164
  247. data/test/test_config.rb +0 -369
  248. data/test/test_configdsl.rb +0 -148
  249. data/test/test_daemonizer.rb +0 -91
  250. data/test/test_engine.rb +0 -203
  251. data/test/test_event.rb +0 -531
  252. data/test/test_event_router.rb +0 -348
  253. data/test/test_event_time.rb +0 -199
  254. data/test/test_file_wrapper.rb +0 -53
  255. data/test/test_filter.rb +0 -121
  256. data/test/test_fluent_log_event_router.rb +0 -99
  257. data/test/test_formatter.rb +0 -369
  258. data/test/test_input.rb +0 -31
  259. data/test/test_log.rb +0 -1076
  260. data/test/test_match.rb +0 -148
  261. data/test/test_mixin.rb +0 -351
  262. data/test/test_msgpack_factory.rb +0 -50
  263. data/test/test_oj_options.rb +0 -55
  264. data/test/test_output.rb +0 -278
  265. data/test/test_plugin.rb +0 -251
  266. data/test/test_plugin_classes.rb +0 -370
  267. data/test/test_plugin_helper.rb +0 -81
  268. data/test/test_plugin_id.rb +0 -119
  269. data/test/test_process.rb +0 -14
  270. data/test/test_root_agent.rb +0 -951
  271. data/test/test_static_config_analysis.rb +0 -177
  272. data/test/test_supervisor.rb +0 -821
  273. data/test/test_test_drivers.rb +0 -136
  274. data/test/test_time_formatter.rb +0 -301
  275. data/test/test_time_parser.rb +0 -362
  276. data/test/test_tls.rb +0 -65
  277. data/test/test_unique_id.rb +0 -47
  278. data/test/test_variable_store.rb +0 -65
@@ -43,11 +43,16 @@ module Fluent
43
43
  @rpc_endpoint = nil
44
44
  @rpc_server = nil
45
45
  @counter = nil
46
+ @socket_manager_server = nil
47
+ @starting_new_supervisor_with_zero_downtime = false
48
+ @new_supervisor_pid = nil
49
+ start_in_parallel = ENV.key?("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD")
50
+ @zero_downtime_restart_mutex = Mutex.new
46
51
 
47
52
  @fluentd_lock_dir = Dir.mktmpdir("fluentd-lock-")
48
53
  ENV['FLUENTD_LOCK_DIR'] = @fluentd_lock_dir
49
54
 
50
- if config[:rpc_endpoint]
55
+ if config[:rpc_endpoint] and not start_in_parallel
51
56
  @rpc_endpoint = config[:rpc_endpoint]
52
57
  @enable_get_dump = config[:enable_get_dump]
53
58
  run_rpc_server
@@ -59,16 +64,27 @@ module Fluent
59
64
  install_supervisor_signal_handlers
60
65
  end
61
66
 
62
- if counter = config[:counter_server]
67
+ if counter = config[:counter_server] and not start_in_parallel
63
68
  run_counter_server(counter)
64
69
  end
65
70
 
66
71
  if config[:disable_shared_socket]
67
72
  $log.info "shared socket for multiple workers is disabled"
73
+ elsif start_in_parallel
74
+ begin
75
+ raise "[BUG] SERVERENGINE_SOCKETMANAGER_PATH env var must exist when starting in parallel" unless ENV.key?('SERVERENGINE_SOCKETMANAGER_PATH')
76
+ @socket_manager_server = ServerEngine::SocketManager::Server.share_sockets_with_another_server(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
77
+ $log.info "zero-downtime-restart: took over the shared sockets", path: ENV['SERVERENGINE_SOCKETMANAGER_PATH']
78
+ rescue => e
79
+ $log.error "zero-downtime-restart: cancel sequence because failed to take over the shared sockets", error: e
80
+ raise
81
+ end
68
82
  else
69
- server = ServerEngine::SocketManager::Server.open
70
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = server.path.to_s
83
+ @socket_manager_server = ServerEngine::SocketManager::Server.open
84
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_server.path.to_s
71
85
  end
86
+
87
+ stop_parallel_old_supervisor_after_delay if start_in_parallel
72
88
  end
73
89
 
74
90
  def after_run
@@ -76,7 +92,9 @@ module Fluent
76
92
  stop_rpc_server if @rpc_endpoint
77
93
  stop_counter_server if @counter
78
94
  cleanup_lock_dir
79
- Fluent::Supervisor.cleanup_resources
95
+ Fluent::Supervisor.cleanup_socketmanager_path unless @starting_new_supervisor_with_zero_downtime
96
+ ensure
97
+ notify_new_supervisor_that_old_one_has_stopped if @starting_new_supervisor_with_zero_downtime
80
98
  end
81
99
 
82
100
  def cleanup_lock_dir
@@ -109,6 +127,13 @@ module Fluent
109
127
  end
110
128
  nil
111
129
  }
130
+ unless Fluent.windows?
131
+ @rpc_server.mount_proc('/api/processes.zeroDowntimeRestart') { |req, res|
132
+ $log.debug "fluentd RPC got /api/processes.zeroDowntimeRestart request"
133
+ Process.kill :USR2, Process.pid
134
+ nil
135
+ }
136
+ end
112
137
  @rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|
113
138
  $log.debug "fluentd RPC got /api/plugins.flushBuffers request"
114
139
  if Fluent.windows?
@@ -137,27 +162,24 @@ module Fluent
137
162
 
138
163
  @rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|
139
164
  $log.debug "fluentd RPC got /api/config.gracefulReload request"
140
- if Fluent.windows?
141
- supervisor_sigusr2_handler
142
- else
143
- Process.kill :USR2, Process.pid
144
- end
145
-
165
+ graceful_reload
146
166
  nil
147
167
  }
148
168
 
149
- @rpc_server.mount_proc('/api/config.getDump') { |req, res|
150
- $log.debug "fluentd RPC got /api/config.getDump request"
151
- $log.info "get dump in-memory config via HTTP"
152
- res.body = supervisor_get_dump_config_handler
153
- [nil, nil, res]
154
- } if @enable_get_dump
169
+ if @enable_get_dump
170
+ @rpc_server.mount_proc('/api/config.getDump') { |req, res|
171
+ $log.debug "fluentd RPC got /api/config.getDump request"
172
+ $log.info "get dump in-memory config via HTTP"
173
+ res.body = supervisor_get_dump_config_handler
174
+ [nil, nil, res]
175
+ }
176
+ end
155
177
 
156
178
  @rpc_server.start
157
179
  end
158
180
 
159
181
  def stop_rpc_server
160
- @rpc_server.shutdown
182
+ @rpc_server&.shutdown
161
183
  end
162
184
 
163
185
  def run_counter_server(counter_conf)
@@ -172,6 +194,44 @@ module Fluent
172
194
  @counter.stop
173
195
  end
174
196
 
197
+ def stop_parallel_old_supervisor_after_delay
198
+ Thread.new do
199
+ # Delay to wait the new workers to start up.
200
+ # Even if it takes a long time to start the new workers and stop the old Fluentd first,
201
+ # it is no problem because the socket buffer works, as long as the capacity is not exceeded.
202
+ sleep 10
203
+ old_pid = ENV["FLUENT_RUNNING_IN_PARALLEL_WITH_OLD"]&.to_i
204
+ if old_pid
205
+ $log.info "zero-downtime-restart: stop the old supervisor"
206
+ Process.kill :TERM, old_pid
207
+ end
208
+ rescue => e
209
+ $log.warn "zero-downtime-restart: failed to stop the old supervisor." +
210
+ " If the old one does not exist, please send SIGWINCH to this new process to start to work fully." +
211
+ " If it exists, something went wrong. Please kill the old one manually.",
212
+ error: e
213
+ end
214
+ end
215
+
216
+ def notify_new_supervisor_that_old_one_has_stopped
217
+ if config[:pid_path]
218
+ new_pid = File.read(config[:pid_path]).to_i
219
+ else
220
+ raise "[BUG] new_supervisor_pid is not saved" unless @new_supervisor_pid
221
+ new_pid = @new_supervisor_pid
222
+ end
223
+
224
+ $log.info "zero-downtime-restart: notify the new supervisor (pid: #{new_pid}) that old one has stopped"
225
+ Process.kill :WINCH, new_pid
226
+ rescue => e
227
+ $log.error(
228
+ "zero-downtime-restart: failed to notify the new supervisor." +
229
+ " Please send SIGWINCH to the new supervisor process manually" +
230
+ " if it does not start to work fully.",
231
+ error: e
232
+ )
233
+ end
234
+
175
235
  def install_supervisor_signal_handlers
176
236
  return if Fluent.windows?
177
237
 
@@ -187,7 +247,16 @@ module Fluent
187
247
 
188
248
  trap :USR2 do
189
249
  $log.debug 'fluentd supervisor process got SIGUSR2'
190
- supervisor_sigusr2_handler
250
+ if Fluent.windows?
251
+ graceful_reload
252
+ else
253
+ zero_downtime_restart
254
+ end
255
+ end
256
+
257
+ trap :WINCH do
258
+ $log.debug 'fluentd supervisor process got SIGWINCH'
259
+ cancel_source_only
191
260
  end
192
261
  end
193
262
 
@@ -254,7 +323,7 @@ module Fluent
254
323
  when :usr1
255
324
  supervisor_sigusr1_handler
256
325
  when :usr2
257
- supervisor_sigusr2_handler
326
+ graceful_reload
258
327
  when :cont
259
328
  supervisor_dump_handler_for_windows
260
329
  when :stop_event_thread
@@ -284,7 +353,7 @@ module Fluent
284
353
  send_signal_to_workers(:USR1)
285
354
  end
286
355
 
287
- def supervisor_sigusr2_handler
356
+ def graceful_reload
288
357
  conf = nil
289
358
  t = Thread.new do
290
359
  $log.info 'Reloading new config'
@@ -312,6 +381,79 @@ module Fluent
312
381
  $log.error "Failed to reload config file: #{e}"
313
382
  end
314
383
 
384
+ def zero_downtime_restart
385
+ Thread.new do
386
+ @zero_downtime_restart_mutex.synchronize do
387
+ $log.info "start zero-downtime-restart sequence"
388
+
389
+ if @starting_new_supervisor_with_zero_downtime
390
+ $log.warn "zero-downtime-restart: canceled because it is already starting"
391
+ Thread.exit
392
+ end
393
+ if ENV.key?("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD")
394
+ $log.warn "zero-downtime-restart: canceled because the previous sequence is still running"
395
+ Thread.exit
396
+ end
397
+
398
+ @starting_new_supervisor_with_zero_downtime = true
399
+ commands = [ServerEngine.ruby_bin_path, $0] + ARGV
400
+ env_to_add = {
401
+ "SERVERENGINE_SOCKETMANAGER_INTERNAL_TOKEN" => ServerEngine::SocketManager::INTERNAL_TOKEN,
402
+ "FLUENT_RUNNING_IN_PARALLEL_WITH_OLD" => "#{Process.pid}",
403
+ }
404
+ pid = Process.spawn(env_to_add, commands.join(" "))
405
+ @new_supervisor_pid = pid unless config[:daemonize]
406
+
407
+ if config[:daemonize]
408
+ Thread.new(pid) do |pid|
409
+ _, status = Process.wait2(pid)
410
+ # check if `ServerEngine::Daemon#daemonize_with_double_fork` succeeded or not
411
+ unless status.success?
412
+ @starting_new_supervisor_with_zero_downtime = false
413
+ $log.error "zero-downtime-restart: failed because new supervisor exits unexpectedly"
414
+ end
415
+ end
416
+ else
417
+ Thread.new(pid) do |pid|
418
+ _, status = Process.wait2(pid)
419
+ @starting_new_supervisor_with_zero_downtime = false
420
+ $log.error "zero-downtime-restart: failed because new supervisor exits unexpectedly", status: status
421
+ end
422
+ end
423
+ end
424
+ rescue => e
425
+ $log.error "zero-downtime-restart: failed", error: e
426
+ @starting_new_supervisor_with_zero_downtime = false
427
+ end
428
+ end
429
+
430
+ def cancel_source_only
431
+ if ENV.key?("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD")
432
+ if config[:rpc_endpoint]
433
+ begin
434
+ @rpc_endpoint = config[:rpc_endpoint]
435
+ @enable_get_dump = config[:enable_get_dump]
436
+ run_rpc_server
437
+ rescue => e
438
+ $log.error "failed to start RPC server", error: e
439
+ end
440
+ end
441
+
442
+ if counter = config[:counter_server]
443
+ begin
444
+ run_counter_server(counter)
445
+ rescue => e
446
+ $log.error "failed to start counter server", error: e
447
+ end
448
+ end
449
+
450
+ $log.info "zero-downtime-restart: done all sequences, now new processes start to work fully"
451
+ ENV.delete("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD")
452
+ end
453
+
454
+ send_signal_to_workers(:WINCH)
455
+ end
456
+
315
457
  def supervisor_dump_handler_for_windows
316
458
  # As for UNIX-like, SIGCONT signal to each process makes the process output its dump-file,
317
459
  # and it is implemented before the implementation of the function for Windows.
@@ -409,6 +551,7 @@ module Fluent
409
551
  main_cmd = config[:main_cmd]
410
552
  env = {
411
553
  'SERVERENGINE_WORKER_ID' => @worker_id.to_i.to_s,
554
+ 'FLUENT_INSTANCE_ID' => Fluent::INSTANCE_ID,
412
555
  }
413
556
  @pm = process_manager.spawn(env, *main_cmd)
414
557
  end
@@ -440,7 +583,7 @@ module Fluent
440
583
  stop_immediately_at_unrecoverable_exit: true,
441
584
  root_dir: params['root_dir'],
442
585
  logger: $log,
443
- log: $log.out,
586
+ log: $log&.out,
444
587
  log_level: params['log_level'],
445
588
  chuser: params['chuser'],
446
589
  chgroup: params['chgroup'],
@@ -486,6 +629,7 @@ module Fluent
486
629
  suppress_repeated_stacktrace: true,
487
630
  ignore_repeated_log_interval: nil,
488
631
  without_source: nil,
632
+ with_source_only: nil,
489
633
  enable_input_metrics: nil,
490
634
  enable_size_metrics: nil,
491
635
  use_v1_config: true,
@@ -499,12 +643,11 @@ module Fluent
499
643
  }
500
644
  end
501
645
 
502
- def self.cleanup_resources
503
- unless Fluent.windows?
504
- if ENV.has_key?('SERVERENGINE_SOCKETMANAGER_PATH')
505
- FileUtils.rm_f(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
506
- end
507
- end
646
+ def self.cleanup_socketmanager_path
647
+ return if Fluent.windows?
648
+ return unless ENV.key?('SERVERENGINE_SOCKETMANAGER_PATH')
649
+
650
+ FileUtils.rm_f(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
508
651
  end
509
652
 
510
653
  def initialize(cl_opt)
@@ -518,7 +661,6 @@ module Fluent
518
661
  @inline_config = opt[:inline_config]
519
662
  @use_v1_config = opt[:use_v1_config]
520
663
  @conf_encoding = opt[:conf_encoding]
521
- @log_path = opt[:log_path]
522
664
  @show_plugin_config = opt[:show_plugin_config]
523
665
  @libs = opt[:libs]
524
666
  @plugin_dirs = opt[:plugin_dirs]
@@ -527,13 +669,15 @@ module Fluent
527
669
  @chumask = opt[:chumask]
528
670
  @signame = opt[:signame]
529
671
 
530
- # TODO: `@log_rotate_age` and `@log_rotate_size` should be removed
672
+ # TODO: `@log_path`, `@log_rotate_age` and `@log_rotate_size` should be removed
531
673
  # since it should be merged with SystemConfig in `build_system_config()`.
532
- # We should always use `system_config.log.rotate_age` and `system_config.log.rotate_size`.
674
+ # We should always use `system_config.log.path`, `system_config.log.rotate_age`
675
+ # and `system_config.log.rotate_size`.
533
676
  # However, currently, there is a bug that `system_config.log` parameters
534
677
  # are not in `Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS`, and these
535
678
  # parameters are not merged in `build_system_config()`.
536
679
  # Until we fix the bug of `Fluent::SystemConfig`, we need to use these instance variables.
680
+ @log_path = opt[:log_path]
537
681
  @log_rotate_age = opt[:log_rotate_age]
538
682
  @log_rotate_size = opt[:log_rotate_size]
539
683
 
@@ -549,6 +693,10 @@ module Fluent
549
693
  raise Fluent::ConfigError, "invalid number of workers (must be > 0):#{@system_config.workers}"
550
694
  end
551
695
 
696
+ if Fluent.windows? && @system_config.with_source_only
697
+ raise Fluent::ConfigError, "with-source-only is not supported on Windows"
698
+ end
699
+
552
700
  root_dir = @system_config.root_dir
553
701
  if root_dir
554
702
  if File.exist?(root_dir)
@@ -567,7 +715,7 @@ module Fluent
567
715
  begin
568
716
  ServerEngine::Privilege.change(@chuser, @chgroup)
569
717
  MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
570
- Fluent::Engine.init(@system_config, supervisor_mode: true)
718
+ Fluent::Engine.init(@system_config, supervisor_mode: true, start_in_parallel: ENV.key?("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD"))
571
719
  Fluent::Engine.run_configure(@conf, dry_run: dry_run)
572
720
  rescue Fluent::ConfigError => e
573
721
  $log.error 'config error', file: @config_path, error: e
@@ -600,6 +748,10 @@ module Fluent
600
748
  raise Fluent::ConfigError, "invalid number of workers (must be 1 or unspecified) with --no-supervisor: #{@system_config.workers}"
601
749
  end
602
750
 
751
+ if Fluent.windows? && @system_config.with_source_only
752
+ raise Fluent::ConfigError, "with-source-only is not supported on Windows"
753
+ end
754
+
603
755
  install_main_process_signal_handlers
604
756
 
605
757
  # This is the only log messsage for @standalone_worker
@@ -612,10 +764,10 @@ module Fluent
612
764
  File.umask(@chumask.to_i(8))
613
765
  end
614
766
  MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
615
- Fluent::Engine.init(@system_config)
767
+ Fluent::Engine.init(@system_config, start_in_parallel: ENV.key?("FLUENT_RUNNING_IN_PARALLEL_WITH_OLD"))
616
768
  Fluent::Engine.run_configure(@conf)
617
769
  Fluent::Engine.run
618
- self.class.cleanup_resources if @standalone_worker
770
+ self.class.cleanup_socketmanager_path if @standalone_worker
619
771
  exit 0
620
772
  end
621
773
  end
@@ -690,6 +842,7 @@ module Fluent
690
842
 
691
843
  # TODO: we should remove this logic. This merging process should be done
692
844
  # in `build_system_config()`.
845
+ @log_path ||= system_config.log.path
693
846
  @log_rotate_age ||= system_config.log.rotate_age
694
847
  @log_rotate_size ||= system_config.log.rotate_size
695
848
 
@@ -832,12 +985,20 @@ module Fluent
832
985
  end
833
986
 
834
987
  trap :USR2 do
988
+ # Leave the old GracefulReload feature, just in case.
989
+ # We can send SIGUSR2 to the worker process to use this old GracefulReload feature.
990
+ # (Note: Normally, we can send SIGUSR2 to the supervisor process to use
991
+ # zero-downtime-restart feature as GracefulReload on non-Windows.)
835
992
  reload_config
836
993
  end
837
994
 
838
995
  trap :CONT do
839
996
  dump_non_windows
840
997
  end
998
+
999
+ trap :WINCH do
1000
+ cancel_source_only
1001
+ end
841
1002
  end
842
1003
  end
843
1004
 
@@ -891,6 +1052,18 @@ module Fluent
891
1052
  end
892
1053
  end
893
1054
 
1055
+ def cancel_source_only
1056
+ Thread.new do
1057
+ begin
1058
+ $log.debug "fluentd main process get SIGWINCH"
1059
+ $log.info "try to cancel with-source-only mode"
1060
+ Fluent::Engine.cancel_source_only!
1061
+ rescue Exception => e
1062
+ $log.warn "failed to cancel source only", error: e
1063
+ end
1064
+ end
1065
+ end
1066
+
894
1067
  def reload_config
895
1068
  Thread.new do
896
1069
  $log.debug('worker got SIGUSR2')
@@ -25,10 +25,10 @@ module Fluent
25
25
  :workers, :restart_worker_interval, :root_dir, :log_level,
26
26
  :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
27
27
  :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
28
- :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
28
+ :without_source, :with_source_only, :rpc_endpoint, :enable_get_dump, :process_name,
29
29
  :file_permission, :dir_permission, :counter_server, :counter_client,
30
30
  :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
31
- :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit
31
+ :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer
32
32
  ]
33
33
 
34
34
  config_param :workers, :integer, default: 1
@@ -41,7 +41,8 @@ module Fluent
41
41
  config_param :emit_error_log_interval, :time, default: nil
42
42
  config_param :suppress_config_dump, :bool, default: nil
43
43
  config_param :log_event_verbose, :bool, default: nil
44
- config_param :without_source, :bool, default: nil
44
+ config_param :without_source, :bool, default: nil
45
+ config_param :with_source_only, :bool, default: nil
45
46
  config_param :rpc_endpoint, :string, default: nil
46
47
  config_param :enable_get_dump, :bool, default: nil
47
48
  config_param :process_name, :string, default: nil
@@ -58,6 +59,7 @@ module Fluent
58
59
  v.to_i(8)
59
60
  end
60
61
  config_section :log, required: false, init: true, multi: false do
62
+ config_param :path, :string, default: nil
61
63
  config_param :format, :enum, list: [:text, :json], default: :text
62
64
  config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
63
65
  config_param :rotate_age, default: nil do |v|
@@ -103,6 +105,16 @@ module Fluent
103
105
  config_param :labels, :hash, default: {}
104
106
  end
105
107
 
108
+ config_section :source_only_buffer, init: true, multi: false do
109
+ config_param :flush_thread_count, :integer, default: 1
110
+ config_param :overflow_action, :enum, list: [:throw_exception, :block, :drop_oldest_chunk], default: :drop_oldest_chunk
111
+ config_param :path, :string, default: nil
112
+ config_param :flush_interval, :time, default: nil
113
+ config_param :chunk_limit_size, :size, default: nil
114
+ config_param :total_limit_size, :size, default: nil
115
+ config_param :compress, :enum, list: [:text, :gzip], default: nil
116
+ end
117
+
106
118
  def self.create(conf, strict_config_value=false)
107
119
  systems = conf.elements(name: 'system')
108
120
  return SystemConfig.new if systems.empty?
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.17.0'
19
+ VERSION = '1.18.0'
20
20
 
21
21
  end
@@ -12,12 +12,13 @@ Gem::Specification.new do |spec|
12
12
  spec.homepage = "TODO: Put your gem's website or public repo URL here."
13
13
  spec.license = "<%= @license.name %>"
14
14
 
15
- test_files, files = `git ls-files -z`.split("\x0").partition do |f|
16
- f.match(%r{^(test|spec|features)/})
15
+ spec.files = Dir.chdir(__dir__) do
16
+ `git ls-files -z`.split("\x0").reject do |f|
17
+ (File.expand_path(f) == __FILE__) ||
18
+ f.start_with?(*%w[test/ spec/ features/ .git .circleci appveyor Gemfile])
19
+ end
17
20
  end
18
- spec.files = files
19
- spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
20
- spec.test_files = test_files
21
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
21
22
  spec.require_paths = ["lib"]
22
23
 
23
24
  spec.add_development_dependency "bundler", "~> <%= bundler_version %>"