fluentd 1.16.5-x64-mingw32 → 1.17.1-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (268) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +88 -0
  3. data/README.md +2 -1
  4. data/Rakefile +1 -1
  5. data/SECURITY.md +2 -2
  6. data/fluent.conf +14 -14
  7. data/lib/fluent/command/binlog_reader.rb +1 -1
  8. data/lib/fluent/command/cap_ctl.rb +4 -4
  9. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  10. data/lib/fluent/compat/propagate_default.rb +4 -4
  11. data/lib/fluent/config/configure_proxy.rb +2 -2
  12. data/lib/fluent/config/types.rb +1 -1
  13. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  14. data/lib/fluent/configurable.rb +2 -2
  15. data/lib/fluent/counter/mutex_hash.rb +1 -1
  16. data/lib/fluent/fluent_log_event_router.rb +0 -2
  17. data/lib/fluent/log/console_adapter.rb +4 -2
  18. data/lib/fluent/plugin/buf_file.rb +1 -1
  19. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  20. data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
  21. data/lib/fluent/plugin/filter_parser.rb +26 -8
  22. data/lib/fluent/plugin/in_exec.rb +14 -2
  23. data/lib/fluent/plugin/in_http.rb +19 -54
  24. data/lib/fluent/plugin/in_sample.rb +13 -7
  25. data/lib/fluent/plugin/in_tail.rb +99 -25
  26. data/lib/fluent/plugin/out_copy.rb +1 -1
  27. data/lib/fluent/plugin/out_file.rb +8 -0
  28. data/lib/fluent/plugin/out_http.rb +137 -13
  29. data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
  30. data/lib/fluent/plugin/parser_json.rb +26 -17
  31. data/lib/fluent/plugin/parser_msgpack.rb +24 -3
  32. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  33. data/lib/fluent/plugin_helper/metrics.rb +2 -2
  34. data/lib/fluent/registry.rb +6 -6
  35. data/lib/fluent/test/output_test.rb +1 -1
  36. data/lib/fluent/unique_id.rb +1 -1
  37. data/lib/fluent/version.rb +1 -1
  38. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  39. metadata +109 -459
  40. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  41. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  42. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  43. data/.github/ISSUE_TEMPLATE.md +0 -17
  44. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  45. data/.github/workflows/stale-actions.yml +0 -24
  46. data/.github/workflows/test.yml +0 -32
  47. data/.gitignore +0 -30
  48. data/Gemfile +0 -9
  49. data/fluentd.gemspec +0 -54
  50. data/test/command/test_binlog_reader.rb +0 -362
  51. data/test/command/test_ca_generate.rb +0 -70
  52. data/test/command/test_cap_ctl.rb +0 -100
  53. data/test/command/test_cat.rb +0 -128
  54. data/test/command/test_ctl.rb +0 -56
  55. data/test/command/test_fluentd.rb +0 -1291
  56. data/test/command/test_plugin_config_formatter.rb +0 -397
  57. data/test/command/test_plugin_generator.rb +0 -109
  58. data/test/compat/test_calls_super.rb +0 -166
  59. data/test/compat/test_parser.rb +0 -92
  60. data/test/config/assertions.rb +0 -42
  61. data/test/config/test_config_parser.rb +0 -551
  62. data/test/config/test_configurable.rb +0 -1784
  63. data/test/config/test_configure_proxy.rb +0 -604
  64. data/test/config/test_dsl.rb +0 -415
  65. data/test/config/test_element.rb +0 -518
  66. data/test/config/test_literal_parser.rb +0 -309
  67. data/test/config/test_plugin_configuration.rb +0 -56
  68. data/test/config/test_section.rb +0 -191
  69. data/test/config/test_system_config.rb +0 -195
  70. data/test/config/test_types.rb +0 -408
  71. data/test/counter/test_client.rb +0 -563
  72. data/test/counter/test_error.rb +0 -44
  73. data/test/counter/test_mutex_hash.rb +0 -179
  74. data/test/counter/test_server.rb +0 -589
  75. data/test/counter/test_store.rb +0 -258
  76. data/test/counter/test_validator.rb +0 -137
  77. data/test/helper.rb +0 -155
  78. data/test/helpers/fuzzy_assert.rb +0 -89
  79. data/test/helpers/process_extenstion.rb +0 -33
  80. data/test/log/test_console_adapter.rb +0 -110
  81. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  82. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  83. data/test/plugin/data/2010/01/20100102.log +0 -0
  84. data/test/plugin/data/log/bar +0 -0
  85. data/test/plugin/data/log/foo/bar.log +0 -0
  86. data/test/plugin/data/log/foo/bar2 +0 -0
  87. data/test/plugin/data/log/test.log +0 -0
  88. data/test/plugin/data/sd_file/config +0 -11
  89. data/test/plugin/data/sd_file/config.json +0 -17
  90. data/test/plugin/data/sd_file/config.yaml +0 -11
  91. data/test/plugin/data/sd_file/config.yml +0 -11
  92. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  93. data/test/plugin/in_tail/test_fifo.rb +0 -121
  94. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  95. data/test/plugin/in_tail/test_position_file.rb +0 -346
  96. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  97. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  98. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  99. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  100. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  101. data/test/plugin/test_bare_output.rb +0 -131
  102. data/test/plugin/test_base.rb +0 -247
  103. data/test/plugin/test_buf_file.rb +0 -1314
  104. data/test/plugin/test_buf_file_single.rb +0 -898
  105. data/test/plugin/test_buf_memory.rb +0 -42
  106. data/test/plugin/test_buffer.rb +0 -1493
  107. data/test/plugin/test_buffer_chunk.rb +0 -209
  108. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  109. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  110. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  111. data/test/plugin/test_compressable.rb +0 -87
  112. data/test/plugin/test_file_util.rb +0 -96
  113. data/test/plugin/test_filter.rb +0 -368
  114. data/test/plugin/test_filter_grep.rb +0 -697
  115. data/test/plugin/test_filter_parser.rb +0 -731
  116. data/test/plugin/test_filter_record_transformer.rb +0 -577
  117. data/test/plugin/test_filter_stdout.rb +0 -207
  118. data/test/plugin/test_formatter_csv.rb +0 -136
  119. data/test/plugin/test_formatter_hash.rb +0 -38
  120. data/test/plugin/test_formatter_json.rb +0 -61
  121. data/test/plugin/test_formatter_ltsv.rb +0 -70
  122. data/test/plugin/test_formatter_msgpack.rb +0 -28
  123. data/test/plugin/test_formatter_out_file.rb +0 -116
  124. data/test/plugin/test_formatter_single_value.rb +0 -44
  125. data/test/plugin/test_formatter_tsv.rb +0 -76
  126. data/test/plugin/test_in_debug_agent.rb +0 -49
  127. data/test/plugin/test_in_exec.rb +0 -261
  128. data/test/plugin/test_in_forward.rb +0 -1178
  129. data/test/plugin/test_in_gc_stat.rb +0 -62
  130. data/test/plugin/test_in_http.rb +0 -1102
  131. data/test/plugin/test_in_monitor_agent.rb +0 -922
  132. data/test/plugin/test_in_object_space.rb +0 -66
  133. data/test/plugin/test_in_sample.rb +0 -190
  134. data/test/plugin/test_in_syslog.rb +0 -505
  135. data/test/plugin/test_in_tail.rb +0 -3288
  136. data/test/plugin/test_in_tcp.rb +0 -328
  137. data/test/plugin/test_in_udp.rb +0 -296
  138. data/test/plugin/test_in_unix.rb +0 -181
  139. data/test/plugin/test_input.rb +0 -137
  140. data/test/plugin/test_metadata.rb +0 -89
  141. data/test/plugin/test_metrics.rb +0 -294
  142. data/test/plugin/test_metrics_local.rb +0 -96
  143. data/test/plugin/test_multi_output.rb +0 -204
  144. data/test/plugin/test_out_copy.rb +0 -308
  145. data/test/plugin/test_out_exec.rb +0 -312
  146. data/test/plugin/test_out_exec_filter.rb +0 -606
  147. data/test/plugin/test_out_file.rb +0 -1038
  148. data/test/plugin/test_out_forward.rb +0 -1349
  149. data/test/plugin/test_out_http.rb +0 -429
  150. data/test/plugin/test_out_null.rb +0 -105
  151. data/test/plugin/test_out_relabel.rb +0 -28
  152. data/test/plugin/test_out_roundrobin.rb +0 -146
  153. data/test/plugin/test_out_secondary_file.rb +0 -458
  154. data/test/plugin/test_out_stdout.rb +0 -205
  155. data/test/plugin/test_out_stream.rb +0 -103
  156. data/test/plugin/test_output.rb +0 -1334
  157. data/test/plugin/test_output_as_buffered.rb +0 -2024
  158. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  159. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  160. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  161. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  162. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  163. data/test/plugin/test_output_as_standard.rb +0 -374
  164. data/test/plugin/test_owned_by.rb +0 -35
  165. data/test/plugin/test_parser.rb +0 -399
  166. data/test/plugin/test_parser_apache.rb +0 -42
  167. data/test/plugin/test_parser_apache2.rb +0 -47
  168. data/test/plugin/test_parser_apache_error.rb +0 -45
  169. data/test/plugin/test_parser_csv.rb +0 -200
  170. data/test/plugin/test_parser_json.rb +0 -138
  171. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  172. data/test/plugin/test_parser_multiline.rb +0 -111
  173. data/test/plugin/test_parser_nginx.rb +0 -88
  174. data/test/plugin/test_parser_none.rb +0 -52
  175. data/test/plugin/test_parser_regexp.rb +0 -284
  176. data/test/plugin/test_parser_syslog.rb +0 -650
  177. data/test/plugin/test_parser_tsv.rb +0 -122
  178. data/test/plugin/test_sd_file.rb +0 -228
  179. data/test/plugin/test_sd_srv.rb +0 -230
  180. data/test/plugin/test_storage.rb +0 -167
  181. data/test/plugin/test_storage_local.rb +0 -335
  182. data/test/plugin/test_string_util.rb +0 -26
  183. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  184. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  185. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  186. data/test/plugin_helper/data/cert/cert.pem +0 -19
  187. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  188. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  189. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  191. data/test/plugin_helper/data/cert/empty.pem +0 -0
  192. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  193. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  194. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  195. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  196. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  197. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  198. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  199. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  200. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  201. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  202. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  203. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  204. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  205. data/test/plugin_helper/http_server/test_app.rb +0 -65
  206. data/test/plugin_helper/http_server/test_route.rb +0 -32
  207. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  208. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  209. data/test/plugin_helper/test_cert_option.rb +0 -25
  210. data/test/plugin_helper/test_child_process.rb +0 -862
  211. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  212. data/test/plugin_helper/test_event_emitter.rb +0 -80
  213. data/test/plugin_helper/test_event_loop.rb +0 -52
  214. data/test/plugin_helper/test_extract.rb +0 -194
  215. data/test/plugin_helper/test_formatter.rb +0 -255
  216. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  217. data/test/plugin_helper/test_inject.rb +0 -561
  218. data/test/plugin_helper/test_metrics.rb +0 -137
  219. data/test/plugin_helper/test_parser.rb +0 -264
  220. data/test/plugin_helper/test_record_accessor.rb +0 -238
  221. data/test/plugin_helper/test_retry_state.rb +0 -1006
  222. data/test/plugin_helper/test_server.rb +0 -1895
  223. data/test/plugin_helper/test_service_discovery.rb +0 -165
  224. data/test/plugin_helper/test_socket.rb +0 -146
  225. data/test/plugin_helper/test_storage.rb +0 -542
  226. data/test/plugin_helper/test_thread.rb +0 -164
  227. data/test/plugin_helper/test_timer.rb +0 -130
  228. data/test/scripts/exec_script.rb +0 -32
  229. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  230. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  231. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  232. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  233. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  234. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  235. data/test/test_capability.rb +0 -74
  236. data/test/test_clock.rb +0 -164
  237. data/test/test_config.rb +0 -369
  238. data/test/test_configdsl.rb +0 -148
  239. data/test/test_daemonizer.rb +0 -91
  240. data/test/test_engine.rb +0 -203
  241. data/test/test_event.rb +0 -531
  242. data/test/test_event_router.rb +0 -348
  243. data/test/test_event_time.rb +0 -199
  244. data/test/test_file_wrapper.rb +0 -53
  245. data/test/test_filter.rb +0 -121
  246. data/test/test_fluent_log_event_router.rb +0 -99
  247. data/test/test_formatter.rb +0 -369
  248. data/test/test_input.rb +0 -31
  249. data/test/test_log.rb +0 -1076
  250. data/test/test_match.rb +0 -148
  251. data/test/test_mixin.rb +0 -351
  252. data/test/test_msgpack_factory.rb +0 -50
  253. data/test/test_oj_options.rb +0 -55
  254. data/test/test_output.rb +0 -278
  255. data/test/test_plugin.rb +0 -251
  256. data/test/test_plugin_classes.rb +0 -370
  257. data/test/test_plugin_helper.rb +0 -81
  258. data/test/test_plugin_id.rb +0 -119
  259. data/test/test_process.rb +0 -14
  260. data/test/test_root_agent.rb +0 -951
  261. data/test/test_static_config_analysis.rb +0 -177
  262. data/test/test_supervisor.rb +0 -821
  263. data/test/test_test_drivers.rb +0 -136
  264. data/test/test_time_formatter.rb +0 -301
  265. data/test/test_time_parser.rb +0 -362
  266. data/test/test_tls.rb +0 -65
  267. data/test/test_unique_id.rb +0 -47
  268. data/test/test_variable_store.rb +0 -65
@@ -1,1291 +0,0 @@
1
- require_relative '../helper'
2
-
3
- # require 'fluent/command/fluentd'
4
- # don't require it... it runs immediately
5
-
6
- require 'fileutils'
7
- require 'timeout'
8
- require 'securerandom'
9
- require 'fluent/file_wrapper'
10
-
11
- class TestFluentdCommand < ::Test::Unit::TestCase
12
- SUPERVISOR_PID_PATTERN = /starting fluentd-[.0-9]+ pid=(\d+)/
13
- WORKER_PID_PATTERN = /starting fluentd worker pid=(\d+) /
14
-
15
- def tmp_dir
16
- File.join(File.dirname(__FILE__), "..", "tmp", "command" "fluentd#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
17
- end
18
-
19
- setup do
20
- @tmp_dir = tmp_dir
21
- FileUtils.mkdir_p(@tmp_dir)
22
- @supervisor_pid = nil
23
- @worker_pids = []
24
- ENV["TEST_RUBY_PATH"] = nil
25
- end
26
-
27
- teardown do
28
- begin
29
- FileUtils.rm_rf(@tmp_dir)
30
- rescue Errno::EACCES
31
- # It may occur on Windows because of delete pending state due to delayed GC.
32
- # Ruby 3.2 or later doesn't ignore Errno::EACCES:
33
- # https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81
34
- end
35
- end
36
-
37
- def process_exist?(pid)
38
- begin
39
- r = Process.waitpid(pid, Process::WNOHANG)
40
- return true if r.nil?
41
- false
42
- rescue SystemCallError
43
- false
44
- end
45
- end
46
-
47
- def create_conf_file(name, content, ext_enc = 'utf-8')
48
- conf_path = File.join(@tmp_dir, name)
49
- Fluent::FileWrapper.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
50
- file.write content
51
- end
52
- conf_path
53
- end
54
-
55
- def create_plugin_file(name, content)
56
- file_path = File.join(@tmp_dir, 'plugin', name)
57
- FileUtils.mkdir_p(File.dirname(file_path))
58
- Fluent::FileWrapper.open(file_path, 'w') do |file|
59
- file.write content
60
- end
61
- file_path
62
- end
63
-
64
- def create_cmdline(conf_path, *fluentd_options)
65
- if Fluent.windows?
66
- cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd")
67
- ["bundle", "exec", ServerEngine.ruby_bin_path, cmd_path, "-c", conf_path, *fluentd_options]
68
- else
69
- cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd")
70
- ["bundle", "exec", cmd_path, "-c", conf_path, *fluentd_options]
71
- end
72
- end
73
-
74
- def process_kill(pid)
75
- if Fluent.windows?
76
- Process.kill(:KILL, pid) rescue nil
77
- return
78
- end
79
-
80
- begin
81
- Process.kill(:TERM, pid) rescue nil
82
- Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
83
- rescue Timeout::Error
84
- Process.kill(:KILL, pid) rescue nil
85
- end
86
- end
87
-
88
- def execute_command(cmdline, chdir=@tmp_dir, env = {})
89
- null_stream = Fluent::FileWrapper.open(File::NULL, 'w')
90
- gemfile_path = File.expand_path(File.dirname(__FILE__) + "../../../Gemfile")
91
-
92
- env = { "BUNDLE_GEMFILE" => gemfile_path }.merge(env)
93
- cmdname = cmdline.shift
94
- arg0 = "testing-fluentd"
95
- # p(here: "executing process", env: env, cmdname: cmdname, arg0: arg0, args: cmdline)
96
- IO.popen(env, [[cmdname, arg0], *cmdline], chdir: chdir, err: [:child, :out]) do |io|
97
- pid = io.pid
98
- begin
99
- yield pid, io
100
- # p(here: "execute command", pid: pid, worker_pids: @worker_pids)
101
- ensure
102
- process_kill(pid)
103
- if @supervisor_pid
104
- process_kill(@supervisor_pid)
105
- end
106
- @worker_pids.each do |cpid|
107
- process_kill(cpid)
108
- end
109
- # p(here: "execute command", pid: pid, exist: process_exist?(pid), worker_pids: @worker_pids, exists: @worker_pids.map{|i| process_exist?(i) })
110
- Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
111
- end
112
- end
113
- ensure
114
- null_stream.close rescue nil
115
- end
116
-
117
- def eager_read(io)
118
- buf = +''
119
-
120
- loop do
121
- b = io.read_nonblock(1024, nil, exception: false)
122
- if b == :wait_readable || b.nil?
123
- return buf
124
- end
125
- buf << b
126
- end
127
- end
128
-
129
- # ATTENTION: This stops taking logs when all `pattern_list` match or timeout,
130
- # so `patterns_not_match` can test only logs up to that point.
131
- def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 20, env: {})
132
- matched = false
133
- matched_wrongly = false
134
- assert_error_msg = ""
135
- stdio_buf = ""
136
- begin
137
- execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
138
- begin
139
- waiting(timeout) do
140
- while process_exist?(pid)
141
- readables, _, _ = IO.select([stdout], nil, nil, 1)
142
- next unless readables
143
- break if readables.first.eof?
144
-
145
- buf = eager_read(readables.first)
146
- # puts buf
147
- stdio_buf << buf
148
- lines = stdio_buf.split("\n")
149
- if pattern_list.all?{|ptn| lines.any?{|line| ptn.is_a?(Regexp) ? ptn.match(line) : line.include?(ptn) } }
150
- matched = true
151
- end
152
-
153
- if Fluent.windows?
154
- # https://github.com/fluent/fluentd/issues/4095
155
- # On Windows, the initial process is different from the supervisor process,
156
- # so we need to wait until `SUPERVISOR_PID_PATTERN` appears in the logs to get the pid.
157
- # (Worker processes will be killed by the supervisor process, so we don't need it-)
158
- break if matched && SUPERVISOR_PID_PATTERN =~ stdio_buf
159
- else
160
- # On Non-Windows, the initial process is the supervisor process,
161
- # so we don't need to wait `SUPERVISOR_PID_PATTERN`.
162
- break if matched
163
- end
164
- end
165
- end
166
- ensure
167
- if SUPERVISOR_PID_PATTERN =~ stdio_buf
168
- @supervisor_pid = $1.to_i
169
- end
170
- stdio_buf.scan(WORKER_PID_PATTERN) do |worker_pid|
171
- @worker_pids << worker_pid.first.to_i
172
- end
173
- end
174
- end
175
- rescue Timeout::Error
176
- assert_error_msg = "execution timeout"
177
- # https://github.com/fluent/fluentd/issues/4095
178
- # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
179
- # since the supervisor process will survive without being killed correctly.
180
- flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
181
- rescue => e
182
- assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
183
- else
184
- assert_error_msg = "log doesn't match" unless matched
185
- end
186
-
187
- if patterns_not_match.empty?
188
- assert_error_msg = build_message(assert_error_msg,
189
- "<?>\nwas expected to include:\n<?>",
190
- stdio_buf, pattern_list)
191
- else
192
- lines = stdio_buf.split("\n")
193
- patterns_not_match.each do |ptn|
194
- matched_wrongly = if ptn.is_a? Regexp
195
- lines.any?{|line| ptn.match(line) }
196
- else
197
- lines.any?{|line| line.include?(ptn) }
198
- end
199
- if matched_wrongly
200
- assert_error_msg << "\n" unless assert_error_msg.empty?
201
- assert_error_msg << "pattern exists in logs wrongly: #{ptn}"
202
- end
203
- end
204
- assert_error_msg = build_message(assert_error_msg,
205
- "<?>\nwas expected to include:\n<?>\nand not include:\n<?>",
206
- stdio_buf, pattern_list, patterns_not_match)
207
- end
208
-
209
- assert matched && !matched_wrongly, assert_error_msg
210
- end
211
-
212
- def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 20)
213
- # empty_list.all?{ ... } is always true
214
- matched = false
215
- running = false
216
- assert_error_msg = "failed to start correctly"
217
- stdio_buf = ""
218
- begin
219
- execute_command(cmdline) do |pid, stdout|
220
- begin
221
- waiting(timeout) do
222
- while process_exist?(pid) && !running
223
- readables, _, _ = IO.select([stdout], nil, nil, 1)
224
- next unless readables
225
- next if readables.first.eof?
226
-
227
- stdio_buf << eager_read(readables.first)
228
- lines = stdio_buf.split("\n")
229
- if lines.any?{|line| line.include?("fluentd worker is now running") }
230
- running = true
231
- end
232
- if pattern_list.all?{|ptn| lines.any?{|line| ptn.is_a?(Regexp) ? ptn.match(line) : line.include?(ptn) } }
233
- matched = true
234
- end
235
- end
236
- end
237
- ensure
238
- if SUPERVISOR_PID_PATTERN =~ stdio_buf
239
- @supervisor_pid = $1.to_i
240
- end
241
- stdio_buf.scan(WORKER_PID_PATTERN) do |worker_pid|
242
- @worker_pids << worker_pid.first.to_i
243
- end
244
- end
245
- end
246
- rescue Timeout::Error
247
- assert_error_msg = "execution timeout with command out:\n" + stdio_buf
248
- # https://github.com/fluent/fluentd/issues/4095
249
- # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
250
- # since the supervisor process will survive without being killed correctly.
251
- flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
252
- rescue => e
253
- assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}\n" + stdio_buf
254
- assert false, assert_error_msg
255
- end
256
- assert !running, "fluentd started to run incorrectly:\n" + stdio_buf
257
- unless matched
258
- assert_error_msg = "fluentd failed to start, without specified regular expressions:\n" + stdio_buf
259
- end
260
- assert matched, assert_error_msg
261
- end
262
-
263
- sub_test_case 'with valid configuration' do
264
- test 'runs successfully' do
265
- conf = <<CONF
266
- <source>
267
- @type dummy
268
- @id dummy
269
- @label @dummydata
270
- tag dummy
271
- dummy {"message": "yay!"}
272
- </source>
273
- <label @dummydata>
274
- <match dummy>
275
- @type null
276
- @id blackhole
277
- </match>
278
- </label>
279
- CONF
280
- conf_path = create_conf_file('valid.conf', conf)
281
- assert File.exist?(conf_path)
282
-
283
- assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", 'worker=0')
284
- end
285
- end
286
-
287
- sub_test_case 'with --conf-encoding' do
288
- test 'runs successfully' do
289
- conf = <<CONF
290
- # テスト
291
- <source>
292
- @type dummy
293
- tag dummy
294
- dummy {"message": "yay!"}
295
- </source>
296
- <match dummy>
297
- @type null
298
- </match>
299
- CONF
300
- conf_path = create_conf_file('shift_jis.conf', conf, 'shift_jis')
301
- assert_log_matches(create_cmdline(conf_path, '--conf-encoding', 'shift_jis'), "fluentd worker is now running", 'worker=0')
302
- end
303
-
304
- test 'failed to run by invalid encoding' do
305
- conf = <<CONF
306
- # テスト
307
- <source>
308
- @type dummy
309
- tag dummy
310
- dummy {"message": "yay!"}
311
- </source>
312
- <match dummy>
313
- @type null
314
- </match>
315
- CONF
316
- conf_path = create_conf_file('shift_jis.conf', conf, 'shift_jis')
317
- assert_fluentd_fails_to_start(create_cmdline(conf_path), "invalid byte sequence in UTF-8")
318
- end
319
- end
320
-
321
- sub_test_case 'with system configuration about root directory' do
322
- setup do
323
- @root_path = File.join(@tmp_dir, "rootpath")
324
- FileUtils.rm_rf(@root_path)
325
- @conf = <<CONF
326
- <system>
327
- root_dir #{@root_path}
328
- </system>
329
- <source>
330
- @type dummy
331
- @id dummy
332
- @label @dummydata
333
- tag dummy
334
- dummy {"message": "yay!"}
335
- </source>
336
- <label @dummydata>
337
- <match dummy>
338
- @type null
339
- @id blackhole
340
- </match>
341
- </label>
342
- CONF
343
- end
344
-
345
- test 'use the specified existing directory as root' do
346
- FileUtils.mkdir_p(@root_path)
347
- conf_path = create_conf_file('existing_root_dir.conf', @conf)
348
- assert Dir.exist?(@root_path)
349
-
350
- assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", 'worker=0')
351
- end
352
-
353
- test 'creates the specified root directory if missing' do
354
- conf_path = create_conf_file('missing_root_dir.conf', @conf)
355
- assert_false Dir.exist?(@root_path)
356
-
357
- assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", 'worker=0')
358
- assert Dir.exist?(@root_path)
359
- end
360
-
361
- test 'fails to launch fluentd if specified root path is invalid path for directory' do
362
- Fluent::FileWrapper.open(@root_path, 'w') do |_|
363
- # create file and close it
364
- end
365
- conf_path = create_conf_file('existing_root_dir.conf', @conf)
366
-
367
- assert_fluentd_fails_to_start(
368
- create_cmdline(conf_path),
369
- "non directory entry exists:#{@root_path}",
370
- )
371
- end
372
- end
373
-
374
- sub_test_case 'configured to route log events to plugins' do
375
- setup do
376
- @basic_conf = <<CONF
377
- <source>
378
- @type dummy
379
- @id dummy
380
- tag dummy
381
- dummy {"message": "yay!"}
382
- </source>
383
- <match dummy>
384
- @type null
385
- @id blackhole
386
- </match>
387
- CONF
388
- end
389
-
390
- test 'by top level <match fluent.*> section' do
391
- conf = @basic_conf + <<CONF
392
- <match fluent.**>
393
- @type stdout
394
- </match>
395
- CONF
396
- conf_path = create_conf_file('logevent_1.conf', conf)
397
- assert_log_matches(
398
- create_cmdline(conf_path),
399
- "fluentd worker is now running",
400
- 'fluent.info: {"worker":0,"message":"fluentd worker is now running worker=0"}',
401
- "define <match fluent.**> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead",
402
- patterns_not_match: ['[warn]: some tags for log events are not defined in top level (to be ignored) tags=["fluent.trace", "fluent.debug"]'],
403
- )
404
- end
405
-
406
- test 'by top level <match> section with warning for missing log levels (and warnings for each log event records)' do
407
- conf = @basic_conf + <<CONF
408
- <match fluent.warn fluent.error fluent.fatal>
409
- @type stdout
410
- </match>
411
- CONF
412
- conf_path = create_conf_file('logevent_2.conf', conf)
413
- assert_log_matches(
414
- create_cmdline(conf_path),
415
- "fluentd worker is now running",
416
- '[warn]: #0 match for some tags of log events are not defined in top level (to be ignored) tags=["fluent.trace", "fluent.debug", "fluent.info"]',
417
- "define <match fluent.warn>, <match fluent.error>, <match fluent.fatal> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead",
418
- '[warn]: #0 no patterns matched tag="fluent.info"',
419
- )
420
- end
421
-
422
- test 'by <label @FLUENT_LOG> section' do
423
- conf = @basic_conf + <<CONF
424
- <label @FLUENT_LOG>
425
- <match **>
426
- @type stdout
427
- </match>
428
- </label>
429
- CONF
430
- conf_path = create_conf_file('logevent_3.conf', conf)
431
- assert_log_matches(
432
- create_cmdline(conf_path),
433
- "fluentd worker is now running",
434
- 'fluent.info: {"worker":0,"message":"fluentd worker is now running worker=0"}',
435
- patterns_not_match: ['[warn]: some tags for log events are not defined in @FLUENT_LOG label (to be ignored)'],
436
- )
437
- end
438
-
439
- test 'by <label> section with warning for missing log levels' do
440
- conf = @basic_conf + <<CONF
441
- <label @FLUENT_LOG>
442
- <match fluent.{trace,debug}>
443
- @type null
444
- </match>
445
- <match fluent.warn fluent.error>
446
- @type stdout
447
- </match>
448
- </label>
449
- CONF
450
- conf_path = create_conf_file('logevent_4.conf', conf)
451
- assert_log_matches(
452
- create_cmdline(conf_path),
453
- "fluentd worker is now running",
454
- '[warn]: #0 match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored) tags=["fluent.info", "fluent.fatal"]',
455
- patterns_not_match: ['[warn]: no patterns matched tag="fluent.info"'],
456
- )
457
- end
458
- end
459
-
460
- sub_test_case 'configured to suppress configuration dump' do
461
- setup do
462
- @basic_conf = <<CONF
463
- <source>
464
- @type dummy
465
- @id dummy
466
- @label @dummydata
467
- tag dummy
468
- dummy {"message": "yay!"}
469
- </source>
470
- <label @dummydata>
471
- <match dummy>
472
- @type null
473
- @id blackhole
474
- </match>
475
- </label>
476
- CONF
477
- end
478
-
479
- test 'configured by system config' do
480
- conf = <<SYSTEM + @basic_conf
481
- <system>
482
- suppress_config_dump
483
- </system>
484
- SYSTEM
485
- conf_path = create_conf_file('suppress_conf_dump_1.conf', conf)
486
- assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", patterns_not_match: ["tag dummy"])
487
- end
488
-
489
- test 'configured by command line option' do
490
- conf_path = create_conf_file('suppress_conf_dump_2.conf', @basic_conf)
491
- assert_log_matches(create_cmdline(conf_path, '--suppress-config-dump'), "fluentd worker is now running", patterns_not_match: ["tag dummy"])
492
- end
493
-
494
- test 'configured as false by system config, but overridden as true by command line option' do
495
- conf = <<SYSTEM + @basic_conf
496
- <system>
497
- suppress_config_dump false
498
- </system>
499
- SYSTEM
500
- conf_path = create_conf_file('suppress_conf_dump_3.conf', conf)
501
- assert_log_matches(create_cmdline(conf_path, '--suppress-config-dump'), "fluentd worker is now running", patterns_not_match: ["tag dummy"])
502
- end
503
- end
504
-
505
- sub_test_case 'configuration with wrong plugin type' do
506
- test 'failed to start' do
507
- conf = <<CONF
508
- <source>
509
- @type
510
- @id dummy
511
- @label @dummydata
512
- tag dummy
513
- dummy {"message": "yay!"}
514
- </source>
515
- <label @dummydata>
516
- <match dummy>
517
- @type null
518
- @id blackhole
519
- </match>
520
- </label>
521
- CONF
522
- conf_path = create_conf_file('type_missing.conf', conf)
523
- assert File.exist?(conf_path)
524
-
525
- assert_fluentd_fails_to_start(
526
- create_cmdline(conf_path),
527
- "config error",
528
- "error=\"Unknown input plugin ''. Run 'gem search -rd fluent-plugin' to find plugins",
529
- )
530
- end
531
- end
532
-
533
- sub_test_case 'configuration to load plugin file with syntax error' do
534
- test 'failed to start' do
535
- script = "require 'fluent/plugin/input'\n"
536
- script << "module Fluent::Plugin\n"
537
- script << " class BuggyInput < Input\n"
538
- script << " Fluent::Plugin.register_input('buggy', self)\n"
539
- script << " end\n"
540
- plugin_path = create_plugin_file('in_buggy.rb', script)
541
-
542
- conf = <<CONF
543
- <source>
544
- @type buggy
545
- @id dummy
546
- @label @dummydata
547
- tag dummy
548
- dummy {"message": "yay!"}
549
- </source>
550
- <label @dummydata>
551
- <match dummy>
552
- @type null
553
- @id blackhole
554
- </match>
555
- </label>
556
- CONF
557
- conf_path = create_conf_file('buggy_plugin.conf', conf)
558
- assert File.exist?(conf_path)
559
-
560
- assert_fluentd_fails_to_start(
561
- create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
562
- "in_buggy.rb:5: syntax error, unexpected end-of-input"
563
- )
564
- end
565
- end
566
-
567
- sub_test_case 'configuration to load plugin which raises unrecoverable error in #start' do
568
- test 'failed to start' do
569
- script = "require 'fluent/plugin/input'\n"
570
- script << "require 'fluent/error'\n"
571
- script << "module Fluent::Plugin\n"
572
- script << " class CrashingInput < Input\n"
573
- script << " Fluent::Plugin.register_input('crashing', self)\n"
574
- script << " def start\n"
575
- script << " raise Fluent::UnrecoverableError"
576
- script << " end\n"
577
- script << " end\n"
578
- script << "end\n"
579
- plugin_path = create_plugin_file('in_crashing.rb', script)
580
-
581
- conf = <<CONF
582
- <source>
583
- @type crashing
584
- @id dummy
585
- @label @dummydata
586
- tag dummy
587
- dummy {"message": "yay!"}
588
- </source>
589
- <label @dummydata>
590
- <match dummy>
591
- @type null
592
- @id blackhole
593
- </match>
594
- </label>
595
- CONF
596
- conf_path = create_conf_file('crashing_plugin.conf', conf)
597
- assert File.exist?(conf_path)
598
-
599
- assert_fluentd_fails_to_start(
600
- create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
601
- 'unexpected error error_class=Fluent::UnrecoverableError error="an unrecoverable error occurs in Fluentd process"',
602
- )
603
- end
604
- end
605
-
606
- sub_test_case 'configured to run 2 workers' do
607
- setup do
608
- @root_path = File.join(@tmp_dir, "rootpath")
609
- FileUtils.rm_rf(@root_path)
610
- FileUtils.mkdir_p(@root_path)
611
- end
612
-
613
- test 'success to start the number of workers specified in configuration' do
614
- conf = <<'CONF'
615
- <system>
616
- workers 2
617
- root_dir #{@root_path}
618
- </system>
619
- <source>
620
- @type dummy
621
- @id "dummy#{worker_id}" # check worker_id works or not with actual command
622
- @label @dummydata
623
- tag dummy
624
- dummy {"message": "yay!"}
625
- </source>
626
- <label @dummydata>
627
- <match dummy>
628
- @type null
629
- @id blackhole
630
- </match>
631
- </label>
632
- CONF
633
- conf_path = create_conf_file('workers1.conf', conf)
634
- assert Dir.exist?(@root_path)
635
-
636
- assert_log_matches(
637
- create_cmdline(conf_path),
638
- "#0 fluentd worker is now running worker=0",
639
- "#1 fluentd worker is now running worker=1"
640
- )
641
- end
642
-
643
- sub_test_case "YAML config format" do
644
- test 'success to start the number of workers specified in configuration' do
645
- conf = <<'CONF'
646
- system:
647
- workers: 2
648
- root_dir: "#{@root_path}"
649
- config:
650
- - source:
651
- $type: dummy
652
- $id: !fluent/s "dummy.#{worker_id}" # check worker_id works or not with actual command
653
- $label: '@dummydata'
654
- tag: dummy
655
- dummy: !fluent/json {"message": !fluent/s "yay from #{hostname}!"}
656
-
657
- - label:
658
- $name: '@dummydata'
659
- config:
660
- - match:
661
- $tag: dummy
662
- $type: "null"
663
- $id: blackhole
664
- CONF
665
- conf_path = create_conf_file('workers1.yaml', conf)
666
- assert Dir.exist?(@root_path)
667
-
668
- assert_log_matches(
669
- create_cmdline(conf_path),
670
- "#0 fluentd worker is now running worker=0",
671
- "#1 fluentd worker is now running worker=1"
672
- )
673
- end
674
- end
675
-
676
- test 'success to start the number of workers specified by command line option' do
677
- conf = <<CONF
678
- <system>
679
- root_dir #{@root_path}
680
- </system>
681
- <source>
682
- @type dummy
683
- @id dummy
684
- @label @dummydata
685
- tag dummy
686
- dummy {"message": "yay!"}
687
- </source>
688
- <label @dummydata>
689
- <match dummy>
690
- @type null
691
- @id blackhole
692
- </match>
693
- </label>
694
- CONF
695
- conf_path = create_conf_file('workers2.conf', conf)
696
- assert_log_matches(
697
- create_cmdline(conf_path, '--workers', '2'),
698
- "#0 fluentd worker is now running worker=0",
699
- "#1 fluentd worker is now running worker=1"
700
- )
701
- end
702
-
703
- test 'failed to start workers when configured plugins do not support multi worker configuration' do
704
- script = "require 'fluent/plugin/input'\n"
705
- script << "module Fluent::Plugin\n"
706
- script << " class SingleInput < Input\n"
707
- script << " Fluent::Plugin.register_input('single', self)\n"
708
- script << " def multi_workers_ready?\n"
709
- script << " false\n"
710
- script << " end\n"
711
- script << " end\n"
712
- script << "end\n"
713
- plugin_path = create_plugin_file('in_single.rb', script)
714
-
715
- conf = <<CONF
716
- <system>
717
- workers 2
718
- </system>
719
- <source>
720
- @type single
721
- @id single
722
- @label @dummydata
723
- </source>
724
- <label @dummydata>
725
- <match dummy>
726
- @type null
727
- @id blackhole
728
- </match>
729
- </label>
730
- CONF
731
- conf_path = create_conf_file('workers_invalid1.conf', conf)
732
- assert_fluentd_fails_to_start(
733
- create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
734
- "Plugin 'single' does not support multi workers configuration (Fluent::Plugin::SingleInput)",
735
- )
736
- end
737
-
738
- test 'failed to start workers when file buffer is configured in non-workers way' do
739
- conf = <<CONF
740
- <system>
741
- workers 2
742
- </system>
743
- <source>
744
- @type dummy
745
- tag dummy
746
- @id single
747
- @label @dummydata
748
- </source>
749
- <label @dummydata>
750
- <match dummy>
751
- @type null
752
- @id blackhole
753
- <buffer>
754
- @type file
755
- path #{File.join(@root_path, "buf", "file.*.log")}
756
- </buffer>
757
- </match>
758
- </label>
759
- CONF
760
- conf_path = create_conf_file('workers_invalid2.conf', conf)
761
- assert_fluentd_fails_to_start(
762
- create_cmdline(conf_path),
763
- "[blackhole] file buffer with multi workers should be configured to use directory 'path', or system root_dir and plugin id",
764
- "config error file=\"#{conf_path}\" error_class=Fluent::ConfigError error=\"Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)\"",
765
- )
766
- end
767
-
768
- test 'failed to start workers when configured plugins as children of MultiOutput do not support multi worker configuration' do
769
- script = <<-EOC
770
- require 'fluent/plugin/output'
771
- module Fluent::Plugin
772
- class SingleOutput < Output
773
- Fluent::Plugin.register_output('single', self)
774
- def multi_workers_ready?
775
- false
776
- end
777
- def write(chunk)
778
- end
779
- end
780
- end
781
- EOC
782
- plugin_path = create_plugin_file('out_single.rb', script)
783
-
784
- conf = <<CONF
785
- <system>
786
- workers 2
787
- </system>
788
- <source>
789
- @type dummy
790
- tag dummy
791
- @id single
792
- @label @dummydata
793
- </source>
794
- <label @dummydata>
795
- <match dummy>
796
- @type copy
797
- <store>
798
- @type single
799
- </store>
800
- <store>
801
- @type single
802
- </store>
803
- </match>
804
- </label>
805
- CONF
806
- conf_path = create_conf_file('workers_invalid3.conf', conf)
807
- assert_fluentd_fails_to_start(
808
- create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
809
- "Plugin 'single' does not support multi workers configuration (Fluent::Plugin::SingleOutput)",
810
- )
811
- end
812
-
813
- test 'success to start a worker2 with worker specific configuration' do
814
- conf = <<CONF
815
- <system>
816
- root_dir #{@root_path}
817
- dir_permission 0744
818
- </system>
819
- CONF
820
- conf_path = create_conf_file('worker_section0.conf', conf)
821
-
822
- FileUtils.rm_rf(@root_path) rescue nil
823
-
824
- assert_path_not_exist(@root_path)
825
- assert_log_matches(create_cmdline(conf_path), 'spawn command to main') # any message is ok
826
- assert_path_exist(@root_path)
827
- if Fluent.windows?
828
- # In Windows, dir permission is always 755.
829
- assert_equal '755', File.stat(@root_path).mode.to_s(8)[-3, 3]
830
- else
831
- assert_equal '744', File.stat(@root_path).mode.to_s(8)[-3, 3]
832
- end
833
- end
834
-
835
- test 'success to start a worker with worker specific configuration' do
836
- conf = <<CONF
837
- <system>
838
- workers 2
839
- root_dir #{@root_path}
840
- </system>
841
- <source>
842
- @type dummy
843
- @id dummy
844
- @label @dummydata
845
- tag dummy
846
- dummy {"message": "yay!"}
847
- </source>
848
- <worker 1>
849
- <source>
850
- @type dummy
851
- @id dummy_in_worker
852
- @label @dummydata
853
- tag dummy
854
- dummy {"message": "yay!"}
855
- </source>
856
- </worker>
857
- <label @dummydata>
858
- <match dummy>
859
- @type null
860
- @id blackhole
861
- </match>
862
- </label>
863
- CONF
864
- conf_path = create_conf_file('worker_section0.conf', conf)
865
- assert Dir.exist?(@root_path)
866
-
867
- assert_log_matches(
868
- create_cmdline(conf_path),
869
- "#0 fluentd worker is now running worker=0",
870
- "#1 fluentd worker is now running worker=1",
871
- /(?!#\d) adding source type="dummy"/,
872
- '#1 adding source type="dummy"'
873
- )
874
- end
875
-
876
- test 'success to start workers when configured plugins only for specific worker do not support multi worker configuration' do
877
- script = <<-EOC
878
- require 'fluent/plugin/input'
879
- module Fluent::Plugin
880
- class SingleInput < Input
881
- Fluent::Plugin.register_input('single', self)
882
- def multi_workers_ready?
883
- false
884
- end
885
- end
886
- end
887
- EOC
888
- plugin_path = create_plugin_file('in_single.rb', script)
889
-
890
- conf = <<CONF
891
- <system>
892
- workers 2
893
- </system>
894
- <worker 1>
895
- <source>
896
- @type single
897
- @id single
898
- @label @dummydata
899
- </source>
900
- </worker>
901
- <label @dummydata>
902
- <match dummy>
903
- @type null
904
- @id blackhole
905
- </match>
906
- </label>
907
- CONF
908
- conf_path = create_conf_file('worker_section1.conf', conf)
909
- assert Dir.exist?(@root_path)
910
-
911
- assert_log_matches(
912
- create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
913
- "#0 fluentd worker is now running worker=0",
914
- "#1 fluentd worker is now running worker=1",
915
- '#1 adding source type="single"'
916
- )
917
- end
918
-
919
- test "multiple values are set to RUBYOPT" do
920
- conf = <<CONF
921
- <source>
922
- @type dummy
923
- tag dummy
924
- </source>
925
- <match>
926
- @type null
927
- </match>
928
- CONF
929
- conf_path = create_conf_file('rubyopt_test.conf', conf)
930
- assert_log_matches(
931
- create_cmdline(conf_path),
932
- '#0 fluentd worker is now running worker=0',
933
- patterns_not_match: ['(LoadError)'],
934
- env: { 'RUBYOPT' => '-rtest-unit -rbundler/setup' },
935
- )
936
- end
937
-
938
- data(
939
- '-E' => '-Eutf-8',
940
- '-encoding' => '--encoding=utf-8',
941
- '-external-encoding' => '--external-encoding=utf-8',
942
- '-internal-encoding' => '--internal-encoding=utf-8',
943
- )
944
- test "-E option is set to RUBYOPT" do |base_opt|
945
- conf = <<CONF
946
- <source>
947
- @type dummy
948
- tag dummy
949
- </source>
950
- <match>
951
- @type null
952
- </match>
953
- CONF
954
- conf_path = create_conf_file('rubyopt_test.conf', conf)
955
- opt = base_opt.dup
956
- opt << " #{ENV['RUBYOPT']}" if ENV['RUBYOPT']
957
- assert_log_matches(
958
- create_cmdline(conf_path),
959
- *opt.split(' '),
960
- patterns_not_match: ['-Eascii-8bit:ascii-8bit'],
961
- env: { 'RUBYOPT' => opt },
962
- )
963
- end
964
-
965
- test "without RUBYOPT" do
966
- saved_ruby_opt = ENV["RUBYOPT"]
967
- ENV["RUBYOPT"] = nil
968
- conf = <<CONF
969
- <source>
970
- @type dummy
971
- tag dummy
972
- </source>
973
- <match>
974
- @type null
975
- </match>
976
- CONF
977
- conf_path = create_conf_file('rubyopt_test.conf', conf)
978
- assert_log_matches(create_cmdline(conf_path), '-Eascii-8bit:ascii-8bit')
979
- ensure
980
- ENV["RUBYOPT"] = saved_ruby_opt
981
- end
982
-
983
- test 'invalid values are set to RUBYOPT' do
984
- omit "hard to run correctly because RUBYOPT=-r/path/to/bundler/setup is required on Windows while this test set invalid RUBYOPT" if Fluent.windows?
985
- conf = <<CONF
986
- <source>
987
- @type dummy
988
- tag dummy
989
- </source>
990
- <match>
991
- @type null
992
- </match>
993
- CONF
994
- conf_path = create_conf_file('rubyopt_invalid_test.conf', conf)
995
- if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
996
- expected_phrase = 'ruby: invalid switch in RUBYOPT'
997
- else
998
- expected_phrase = 'Invalid option is passed to RUBYOPT'
999
- end
1000
- assert_log_matches(
1001
- create_cmdline(conf_path),
1002
- expected_phrase,
1003
- env: { 'RUBYOPT' => 'a' },
1004
- )
1005
- end
1006
-
1007
- # https://github.com/fluent/fluentd/issues/2915
1008
- test "ruby path contains spaces" do
1009
- saved_ruby_opt = ENV["RUBYOPT"]
1010
- ENV["RUBYOPT"] = nil
1011
- conf = <<CONF
1012
- <source>
1013
- @type dummy
1014
- tag dummy
1015
- </source>
1016
- <match>
1017
- @type null
1018
- </match>
1019
- CONF
1020
- ruby_path = ServerEngine.ruby_bin_path
1021
- tmp_ruby_path = File.join(@tmp_dir, "ruby with spaces")
1022
- if Fluent.windows?
1023
- tmp_ruby_path << ".bat"
1024
- Fluent::FileWrapper.open(tmp_ruby_path, "w") do |file|
1025
- file.write "#{ruby_path} %*"
1026
- end
1027
- else
1028
- FileUtils.ln_sf(ruby_path, tmp_ruby_path)
1029
- end
1030
- ENV["TEST_RUBY_PATH"] = tmp_ruby_path
1031
- cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd")
1032
- conf_path = create_conf_file('space_mixed_ruby_path_test.conf', conf)
1033
- args = ["bundle", "exec", tmp_ruby_path, cmd_path, "-c", conf_path]
1034
- assert_log_matches(
1035
- args,
1036
- 'spawn command to main:',
1037
- '-Eascii-8bit:ascii-8bit'
1038
- )
1039
- ensure
1040
- ENV["RUBYOPT"] = saved_ruby_opt
1041
- end
1042
-
1043
- test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
1044
- conf = <<CONF
1045
- <system>
1046
- workers 2
1047
- </system>
1048
- <source>
1049
- @type dummy
1050
- @id dummy
1051
- tag dummy
1052
- dummy {"message": "yay!"}
1053
- </source>
1054
- <worker 1>
1055
- <match dummy>
1056
- @type null
1057
- @id blackhole
1058
- <buffer>
1059
- @type file
1060
- path #{File.join(@root_path, "buf")}
1061
- </buffer>
1062
- </match>
1063
- </worker>
1064
- CONF
1065
- conf_path = create_conf_file('worker_section2.conf', conf)
1066
- assert_log_matches(
1067
- create_cmdline(conf_path),
1068
- "#0 fluentd worker is now running worker=0",
1069
- "#1 fluentd worker is now running worker=1",
1070
- '#1 adding match pattern="dummy" type="null"'
1071
- )
1072
- end
1073
-
1074
- test 'success to start workers when configured plugins as a children of MultiOutput only for specific worker do not support multi worker configuration' do
1075
- script = <<-EOC
1076
- require 'fluent/plugin/output'
1077
- module Fluent::Plugin
1078
- class SingleOutput < Output
1079
- Fluent::Plugin.register_output('single', self)
1080
- def multi_workers_ready?
1081
- false
1082
- end
1083
- def write(chunk)
1084
- end
1085
- end
1086
- end
1087
- EOC
1088
- plugin_path = create_plugin_file('out_single.rb', script)
1089
-
1090
- conf = <<CONF
1091
- <system>
1092
- workers 2
1093
- </system>
1094
- <source>
1095
- @type dummy
1096
- @id dummy
1097
- tag dummy
1098
- dummy {"message": "yay!"}
1099
- </source>
1100
- <worker 1>
1101
- <match dummy>
1102
- @type copy
1103
- <store>
1104
- @type single
1105
- </store>
1106
- <store>
1107
- @type single
1108
- </store>
1109
- </match>
1110
- </worker>
1111
- CONF
1112
- conf_path = create_conf_file('worker_section3.conf', conf)
1113
- assert_log_matches(
1114
- create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
1115
- "#0 fluentd worker is now running worker=0",
1116
- "#1 fluentd worker is now running worker=1",
1117
- '#1 adding match pattern="dummy" type="copy"'
1118
- )
1119
- end
1120
- end
1121
-
1122
- sub_test_case 'config dump' do
1123
- test 'all secret parameters in worker section is sealed' do
1124
- script = <<-EOC
1125
- require 'fluent/plugin/input'
1126
- module Fluent::Plugin
1127
- class FakeInput < Input
1128
- Fluent::Plugin.register_input('fake', self)
1129
- config_param :secret, :string, secret: true
1130
- def multi_workers_ready?; true; end
1131
- end
1132
- end
1133
- EOC
1134
- plugin_path = create_plugin_file('in_fake.rb', script)
1135
-
1136
- conf = <<CONF
1137
- <system>
1138
- workers 2
1139
- </system>
1140
- <worker 0>
1141
- <source>
1142
- @type fake
1143
- secret secret0
1144
- </source>
1145
- <match>
1146
- @type null
1147
- </match>
1148
- </worker>
1149
- <worker 1>
1150
- <source>
1151
- @type fake
1152
- secret secret1
1153
- </source>
1154
- <match>
1155
- @type null
1156
- </match>
1157
- </worker>
1158
- CONF
1159
- conf_path = create_conf_file('secret_in_worker.conf', conf)
1160
- assert File.exist?(conf_path)
1161
-
1162
- assert_log_matches(create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
1163
- "secret xxxxxx", patterns_not_match: ["secret secret0", "secret secret1"])
1164
- end
1165
- end
1166
-
1167
- sub_test_case 'sahred socket options' do
1168
- test 'enable shared socket by default' do
1169
- conf = ""
1170
- conf_path = create_conf_file('empty.conf', conf)
1171
- assert File.exist?(conf_path)
1172
- assert_log_matches(create_cmdline(conf_path),
1173
- patterns_not_match: ["shared socket for multiple workers is disabled"])
1174
- end
1175
-
1176
- test 'disable shared socket by command line option' do
1177
- conf = ""
1178
- conf_path = create_conf_file('empty.conf', conf)
1179
- assert File.exist?(conf_path)
1180
- assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"),
1181
- "shared socket for multiple workers is disabled",)
1182
- end
1183
-
1184
- test 'disable shared socket by system config' do
1185
- conf = <<CONF
1186
- <system>
1187
- disable_shared_socket
1188
- </system>
1189
- CONF
1190
- conf_path = create_conf_file('empty.conf', conf)
1191
- assert File.exist?(conf_path)
1192
- assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"),
1193
- "shared socket for multiple workers is disabled",)
1194
- end
1195
- end
1196
-
1197
- # TODO: `patterns_not_match` can test only logs up to `pattern_list`,
1198
- # so we need to fix some meaningless `patterns_not_match` conditions.
1199
- sub_test_case 'log_level by command line option' do
1200
- test 'info' do
1201
- conf = ""
1202
- conf_path = create_conf_file('empty.conf', conf)
1203
- assert File.exist?(conf_path)
1204
- assert_log_matches(create_cmdline(conf_path),
1205
- "[info]",
1206
- patterns_not_match: ["[debug]"])
1207
- end
1208
-
1209
- test 'debug' do
1210
- conf = ""
1211
- conf_path = create_conf_file('empty.conf', conf)
1212
- assert File.exist?(conf_path)
1213
- assert_log_matches(create_cmdline(conf_path, "-v"),
1214
- "[debug]",
1215
- patterns_not_match: ["[trace]"])
1216
- end
1217
-
1218
- data("Trace" => "-vv")
1219
- data("Invalid low level should be treated as Trace level": "-vvv")
1220
- test 'trace' do |option|
1221
- conf = <<CONF
1222
- <source>
1223
- @type sample
1224
- tag test
1225
- </source>
1226
- CONF
1227
- conf_path = create_conf_file('sample.conf', conf)
1228
- assert File.exist?(conf_path)
1229
- assert_log_matches(create_cmdline(conf_path, option),
1230
- "[trace]",)
1231
- end
1232
-
1233
- test 'warn' do
1234
- omit "Can't run on Windows since there is no way to take pid of the supervisor." if Fluent.windows?
1235
- conf = <<CONF
1236
- <source>
1237
- @type sample
1238
- tag test
1239
- </source>
1240
- CONF
1241
- conf_path = create_conf_file('sample.conf', conf)
1242
- assert File.exist?(conf_path)
1243
- assert_log_matches(create_cmdline(conf_path, "-q"),
1244
- "[warn]",
1245
- patterns_not_match: ["[info]"])
1246
- end
1247
-
1248
- data("Error" => "-qq")
1249
- data("Fatal should be treated as Error level" => "-qqq")
1250
- data("Invalid high level should be treated as Error level": "-qqqq")
1251
- test 'error' do |option|
1252
- # This test can run on Windows correctly,
1253
- # since the process will stop automatically with an error.
1254
- conf = <<CONF
1255
- <source>
1256
- @type plugin_not_found
1257
- tag test
1258
- </source>
1259
- CONF
1260
- conf_path = create_conf_file('plugin_not_found.conf', conf)
1261
- assert File.exist?(conf_path)
1262
- assert_log_matches(create_cmdline(conf_path, option),
1263
- "[error]",
1264
- patterns_not_match: ["[warn]"])
1265
- end
1266
-
1267
- test 'system config one should not be overwritten when cmd line one is not specified' do
1268
- conf = <<CONF
1269
- <system>
1270
- log_level debug
1271
- </system>
1272
- CONF
1273
- conf_path = create_conf_file('debug.conf', conf)
1274
- assert File.exist?(conf_path)
1275
- assert_log_matches(create_cmdline(conf_path),
1276
- "[debug]")
1277
- end
1278
- end
1279
-
1280
- sub_test_case "inline_config" do
1281
- test "can change log_level by --inline-config" do
1282
- # Since we can't define multiple `<system>` directives, this use-case is not recommended.
1283
- # This is just for this test.
1284
- inline_conf = '<system>\nlog_level debug\n</system>'
1285
- conf_path = create_conf_file('test.conf', "")
1286
- assert File.exist?(conf_path)
1287
- assert_log_matches(create_cmdline(conf_path, "--inline-config", inline_conf),
1288
- "[debug]")
1289
- end
1290
- end
1291
- end