fluentd 1.17.0-x64-mingw32 → 1.18.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (278) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +100 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/command/fluentd.rb +7 -1
  8. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  9. data/lib/fluent/compat/propagate_default.rb +4 -4
  10. data/lib/fluent/config/literal_parser.rb +9 -2
  11. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  12. data/lib/fluent/engine.rb +49 -33
  13. data/lib/fluent/env.rb +3 -0
  14. data/lib/fluent/event_router.rb +2 -2
  15. data/lib/fluent/log/console_adapter.rb +4 -2
  16. data/lib/fluent/plugin/filter_parser.rb +27 -51
  17. data/lib/fluent/plugin/in_exec.rb +14 -2
  18. data/lib/fluent/plugin/in_http.rb +6 -1
  19. data/lib/fluent/plugin/in_sample.rb +13 -7
  20. data/lib/fluent/plugin/in_syslog.rb +4 -0
  21. data/lib/fluent/plugin/in_tail.rb +65 -23
  22. data/lib/fluent/plugin/in_tcp.rb +4 -0
  23. data/lib/fluent/plugin/in_udp.rb +10 -1
  24. data/lib/fluent/plugin/input.rb +4 -0
  25. data/lib/fluent/plugin/out_buffer.rb +40 -0
  26. data/lib/fluent/plugin/out_copy.rb +1 -1
  27. data/lib/fluent/plugin/out_file.rb +8 -0
  28. data/lib/fluent/plugin/out_http.rb +12 -0
  29. data/lib/fluent/plugin/output.rb +2 -0
  30. data/lib/fluent/plugin/parser_json.rb +4 -12
  31. data/lib/fluent/plugin_helper/cert_option.rb +8 -0
  32. data/lib/fluent/plugin_helper/event_emitter.rb +12 -0
  33. data/lib/fluent/plugin_helper/http_server/server.rb +24 -8
  34. data/lib/fluent/plugin_helper/server.rb +9 -0
  35. data/lib/fluent/root_agent.rb +114 -19
  36. data/lib/fluent/source_only_buffer_agent.rb +102 -0
  37. data/lib/fluent/supervisor.rb +207 -34
  38. data/lib/fluent/system_config.rb +15 -3
  39. data/lib/fluent/version.rb +1 -1
  40. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  41. metadata +24 -483
  42. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  43. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  44. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  45. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  46. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  47. data/.github/ISSUE_TEMPLATE.md +0 -17
  48. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  49. data/.github/workflows/stale-actions.yml +0 -24
  50. data/.github/workflows/test-ruby-head.yml +0 -31
  51. data/.github/workflows/test.yml +0 -32
  52. data/.gitignore +0 -30
  53. data/Gemfile +0 -9
  54. data/fluentd.gemspec +0 -62
  55. data/test/command/test_binlog_reader.rb +0 -362
  56. data/test/command/test_ca_generate.rb +0 -70
  57. data/test/command/test_cap_ctl.rb +0 -100
  58. data/test/command/test_cat.rb +0 -128
  59. data/test/command/test_ctl.rb +0 -56
  60. data/test/command/test_fluentd.rb +0 -1291
  61. data/test/command/test_plugin_config_formatter.rb +0 -397
  62. data/test/command/test_plugin_generator.rb +0 -109
  63. data/test/compat/test_calls_super.rb +0 -166
  64. data/test/compat/test_parser.rb +0 -92
  65. data/test/config/assertions.rb +0 -42
  66. data/test/config/test_config_parser.rb +0 -551
  67. data/test/config/test_configurable.rb +0 -1784
  68. data/test/config/test_configure_proxy.rb +0 -604
  69. data/test/config/test_dsl.rb +0 -415
  70. data/test/config/test_element.rb +0 -518
  71. data/test/config/test_literal_parser.rb +0 -309
  72. data/test/config/test_plugin_configuration.rb +0 -56
  73. data/test/config/test_section.rb +0 -191
  74. data/test/config/test_system_config.rb +0 -195
  75. data/test/config/test_types.rb +0 -408
  76. data/test/counter/test_client.rb +0 -563
  77. data/test/counter/test_error.rb +0 -44
  78. data/test/counter/test_mutex_hash.rb +0 -179
  79. data/test/counter/test_server.rb +0 -589
  80. data/test/counter/test_store.rb +0 -258
  81. data/test/counter/test_validator.rb +0 -137
  82. data/test/helper.rb +0 -155
  83. data/test/helpers/fuzzy_assert.rb +0 -89
  84. data/test/helpers/process_extenstion.rb +0 -33
  85. data/test/log/test_console_adapter.rb +0 -117
  86. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  87. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  88. data/test/plugin/data/2010/01/20100102.log +0 -0
  89. data/test/plugin/data/log/bar +0 -0
  90. data/test/plugin/data/log/foo/bar.log +0 -0
  91. data/test/plugin/data/log/foo/bar2 +0 -0
  92. data/test/plugin/data/log/test.log +0 -0
  93. data/test/plugin/data/log_numeric/01.log +0 -0
  94. data/test/plugin/data/log_numeric/02.log +0 -0
  95. data/test/plugin/data/log_numeric/12.log +0 -0
  96. data/test/plugin/data/log_numeric/14.log +0 -0
  97. data/test/plugin/data/sd_file/config +0 -11
  98. data/test/plugin/data/sd_file/config.json +0 -17
  99. data/test/plugin/data/sd_file/config.yaml +0 -11
  100. data/test/plugin/data/sd_file/config.yml +0 -11
  101. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  102. data/test/plugin/in_tail/test_fifo.rb +0 -121
  103. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  104. data/test/plugin/in_tail/test_position_file.rb +0 -346
  105. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  106. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  107. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  108. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  109. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  110. data/test/plugin/test_bare_output.rb +0 -131
  111. data/test/plugin/test_base.rb +0 -247
  112. data/test/plugin/test_buf_file.rb +0 -1314
  113. data/test/plugin/test_buf_file_single.rb +0 -898
  114. data/test/plugin/test_buf_memory.rb +0 -42
  115. data/test/plugin/test_buffer.rb +0 -1493
  116. data/test/plugin/test_buffer_chunk.rb +0 -209
  117. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  118. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  119. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  120. data/test/plugin/test_compressable.rb +0 -87
  121. data/test/plugin/test_file_util.rb +0 -96
  122. data/test/plugin/test_filter.rb +0 -368
  123. data/test/plugin/test_filter_grep.rb +0 -697
  124. data/test/plugin/test_filter_parser.rb +0 -731
  125. data/test/plugin/test_filter_record_transformer.rb +0 -577
  126. data/test/plugin/test_filter_stdout.rb +0 -207
  127. data/test/plugin/test_formatter_csv.rb +0 -136
  128. data/test/plugin/test_formatter_hash.rb +0 -38
  129. data/test/plugin/test_formatter_json.rb +0 -61
  130. data/test/plugin/test_formatter_ltsv.rb +0 -70
  131. data/test/plugin/test_formatter_msgpack.rb +0 -28
  132. data/test/plugin/test_formatter_out_file.rb +0 -116
  133. data/test/plugin/test_formatter_single_value.rb +0 -44
  134. data/test/plugin/test_formatter_tsv.rb +0 -76
  135. data/test/plugin/test_in_debug_agent.rb +0 -49
  136. data/test/plugin/test_in_exec.rb +0 -261
  137. data/test/plugin/test_in_forward.rb +0 -1178
  138. data/test/plugin/test_in_gc_stat.rb +0 -62
  139. data/test/plugin/test_in_http.rb +0 -1124
  140. data/test/plugin/test_in_monitor_agent.rb +0 -922
  141. data/test/plugin/test_in_object_space.rb +0 -66
  142. data/test/plugin/test_in_sample.rb +0 -190
  143. data/test/plugin/test_in_syslog.rb +0 -505
  144. data/test/plugin/test_in_tail.rb +0 -3429
  145. data/test/plugin/test_in_tcp.rb +0 -328
  146. data/test/plugin/test_in_udp.rb +0 -296
  147. data/test/plugin/test_in_unix.rb +0 -181
  148. data/test/plugin/test_input.rb +0 -137
  149. data/test/plugin/test_metadata.rb +0 -89
  150. data/test/plugin/test_metrics.rb +0 -294
  151. data/test/plugin/test_metrics_local.rb +0 -96
  152. data/test/plugin/test_multi_output.rb +0 -204
  153. data/test/plugin/test_out_copy.rb +0 -308
  154. data/test/plugin/test_out_exec.rb +0 -312
  155. data/test/plugin/test_out_exec_filter.rb +0 -606
  156. data/test/plugin/test_out_file.rb +0 -1038
  157. data/test/plugin/test_out_forward.rb +0 -1349
  158. data/test/plugin/test_out_http.rb +0 -557
  159. data/test/plugin/test_out_null.rb +0 -105
  160. data/test/plugin/test_out_relabel.rb +0 -28
  161. data/test/plugin/test_out_roundrobin.rb +0 -146
  162. data/test/plugin/test_out_secondary_file.rb +0 -458
  163. data/test/plugin/test_out_stdout.rb +0 -205
  164. data/test/plugin/test_out_stream.rb +0 -103
  165. data/test/plugin/test_output.rb +0 -1334
  166. data/test/plugin/test_output_as_buffered.rb +0 -2024
  167. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  168. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  169. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  170. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  171. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  172. data/test/plugin/test_output_as_standard.rb +0 -374
  173. data/test/plugin/test_owned_by.rb +0 -34
  174. data/test/plugin/test_parser.rb +0 -399
  175. data/test/plugin/test_parser_apache.rb +0 -42
  176. data/test/plugin/test_parser_apache2.rb +0 -47
  177. data/test/plugin/test_parser_apache_error.rb +0 -45
  178. data/test/plugin/test_parser_csv.rb +0 -200
  179. data/test/plugin/test_parser_json.rb +0 -244
  180. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  181. data/test/plugin/test_parser_msgpack.rb +0 -127
  182. data/test/plugin/test_parser_multiline.rb +0 -111
  183. data/test/plugin/test_parser_nginx.rb +0 -88
  184. data/test/plugin/test_parser_none.rb +0 -52
  185. data/test/plugin/test_parser_regexp.rb +0 -284
  186. data/test/plugin/test_parser_syslog.rb +0 -650
  187. data/test/plugin/test_parser_tsv.rb +0 -122
  188. data/test/plugin/test_sd_file.rb +0 -228
  189. data/test/plugin/test_sd_srv.rb +0 -230
  190. data/test/plugin/test_storage.rb +0 -166
  191. data/test/plugin/test_storage_local.rb +0 -335
  192. data/test/plugin/test_string_util.rb +0 -26
  193. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  195. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  196. data/test/plugin_helper/data/cert/cert.pem +0 -19
  197. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  198. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  199. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  200. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  201. data/test/plugin_helper/data/cert/empty.pem +0 -0
  202. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  203. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  204. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  205. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  206. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  207. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  208. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  209. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  210. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  211. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  212. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  213. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  214. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  215. data/test/plugin_helper/http_server/test_app.rb +0 -65
  216. data/test/plugin_helper/http_server/test_route.rb +0 -32
  217. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  218. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  219. data/test/plugin_helper/test_cert_option.rb +0 -25
  220. data/test/plugin_helper/test_child_process.rb +0 -862
  221. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  222. data/test/plugin_helper/test_event_emitter.rb +0 -80
  223. data/test/plugin_helper/test_event_loop.rb +0 -52
  224. data/test/plugin_helper/test_extract.rb +0 -194
  225. data/test/plugin_helper/test_formatter.rb +0 -255
  226. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  227. data/test/plugin_helper/test_inject.rb +0 -561
  228. data/test/plugin_helper/test_metrics.rb +0 -137
  229. data/test/plugin_helper/test_parser.rb +0 -264
  230. data/test/plugin_helper/test_record_accessor.rb +0 -238
  231. data/test/plugin_helper/test_retry_state.rb +0 -1006
  232. data/test/plugin_helper/test_server.rb +0 -1895
  233. data/test/plugin_helper/test_service_discovery.rb +0 -165
  234. data/test/plugin_helper/test_socket.rb +0 -146
  235. data/test/plugin_helper/test_storage.rb +0 -542
  236. data/test/plugin_helper/test_thread.rb +0 -164
  237. data/test/plugin_helper/test_timer.rb +0 -130
  238. data/test/scripts/exec_script.rb +0 -32
  239. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  240. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  241. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  242. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  243. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  244. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  245. data/test/test_capability.rb +0 -74
  246. data/test/test_clock.rb +0 -164
  247. data/test/test_config.rb +0 -369
  248. data/test/test_configdsl.rb +0 -148
  249. data/test/test_daemonizer.rb +0 -91
  250. data/test/test_engine.rb +0 -203
  251. data/test/test_event.rb +0 -531
  252. data/test/test_event_router.rb +0 -348
  253. data/test/test_event_time.rb +0 -199
  254. data/test/test_file_wrapper.rb +0 -53
  255. data/test/test_filter.rb +0 -121
  256. data/test/test_fluent_log_event_router.rb +0 -99
  257. data/test/test_formatter.rb +0 -369
  258. data/test/test_input.rb +0 -31
  259. data/test/test_log.rb +0 -1076
  260. data/test/test_match.rb +0 -148
  261. data/test/test_mixin.rb +0 -351
  262. data/test/test_msgpack_factory.rb +0 -50
  263. data/test/test_oj_options.rb +0 -55
  264. data/test/test_output.rb +0 -278
  265. data/test/test_plugin.rb +0 -251
  266. data/test/test_plugin_classes.rb +0 -370
  267. data/test/test_plugin_helper.rb +0 -81
  268. data/test/test_plugin_id.rb +0 -119
  269. data/test/test_process.rb +0 -14
  270. data/test/test_root_agent.rb +0 -951
  271. data/test/test_static_config_analysis.rb +0 -177
  272. data/test/test_supervisor.rb +0 -821
  273. data/test/test_test_drivers.rb +0 -136
  274. data/test/test_time_formatter.rb +0 -301
  275. data/test/test_time_parser.rb +0 -362
  276. data/test/test_tls.rb +0 -65
  277. data/test/test_unique_id.rb +0 -47
  278. data/test/test_variable_store.rb +0 -65
@@ -1,146 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/multi_output'
3
- require 'fluent/plugin/out_roundrobin'
4
-
5
- class RoundRobinOutputTest < Test::Unit::TestCase
6
- class << self
7
- def startup
8
- $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))
9
- require 'fluent/plugin/out_test'
10
- require 'fluent/plugin/out_test2'
11
- end
12
-
13
- def shutdown
14
- $LOAD_PATH.shift
15
- end
16
- end
17
-
18
- def setup
19
- Fluent::Test.setup
20
- end
21
-
22
- CONFIG = %[
23
- <store>
24
- @type test
25
- name c0
26
- </store>
27
- <store>
28
- @type test2
29
- name c1
30
- </store>
31
- <store>
32
- @type test
33
- name c2
34
- </store>
35
- ]
36
- CONFIG_WITH_WEIGHT = %[
37
- <store>
38
- @type test
39
- name c0
40
- weight 3
41
- </store>
42
- <store>
43
- @type test2
44
- name c1
45
- weight 3
46
- </store>
47
- <store>
48
- @type test
49
- name c2
50
- </store>
51
- ]
52
-
53
- def create_driver(conf = CONFIG)
54
- Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::RoundRobinOutput).configure(conf)
55
- end
56
-
57
- def test_configure
58
- d = create_driver
59
-
60
- outputs = d.instance.outputs
61
- assert_equal 3, outputs.size
62
-
63
- assert_equal Fluent::Plugin::TestOutput, outputs[0].class
64
- assert_equal Fluent::Plugin::Test2Output, outputs[1].class
65
- assert_equal Fluent::Plugin::TestOutput, outputs[2].class
66
-
67
- assert !outputs[0].has_router?
68
- assert outputs[1].has_router?
69
- assert outputs[1].router
70
- assert !outputs[2].has_router?
71
-
72
- assert_equal "c0", outputs[0].name
73
- assert_equal "c1", outputs[1].name
74
- assert_equal "c2", outputs[2].name
75
-
76
- weights = d.instance.weights
77
- assert_equal 3, weights.size
78
- assert_equal 1, weights[0]
79
- assert_equal 1, weights[1]
80
- assert_equal 1, weights[2]
81
-
82
- d = create_driver(CONFIG_WITH_WEIGHT)
83
-
84
- outputs = d.instance.outputs
85
- assert_equal 3, outputs.size
86
-
87
- assert_equal Fluent::Plugin::TestOutput, outputs[0].class
88
- assert_equal Fluent::Plugin::Test2Output, outputs[1].class
89
- assert_equal Fluent::Plugin::TestOutput, outputs[2].class
90
-
91
- assert_equal "c0", outputs[0].name
92
- assert_equal "c1", outputs[1].name
93
- assert_equal "c2", outputs[2].name
94
-
95
- weights = d.instance.weights
96
- assert_equal 3, weights.size
97
- assert_equal 3, weights[0]
98
- assert_equal 3, weights[1]
99
- assert_equal 1, weights[2]
100
- end
101
-
102
- def test_events_feeded_to_plugins_by_roundrobin
103
- d = create_driver
104
-
105
- time = event_time("2011-01-02 13:14:15 UTC")
106
- d.run(default_tag: 'test') do
107
- d.feed(time, {"a" => 1})
108
- d.feed(time, {"a" => 2})
109
- d.feed(time, {"a" => 3})
110
- d.feed(time, {"a" => 4})
111
- end
112
-
113
- os = d.instance.outputs
114
-
115
- assert_equal [
116
- [time, {"a"=>1}],
117
- [time, {"a"=>4}],
118
- ], os[0].events
119
-
120
- assert_equal [
121
- [time, {"a"=>2}],
122
- ], os[1].events
123
-
124
- assert_equal [
125
- [time, {"a"=>3}],
126
- ], os[2].events
127
- end
128
-
129
- def test_events_feeded_with_specified_weights
130
- d = create_driver(CONFIG_WITH_WEIGHT)
131
-
132
- time = event_time("2011-01-02 13:14:15 UTC")
133
- d.run(default_tag: 'test') do
134
- 14.times do |i|
135
- d.feed(time, {"a" => i})
136
- end
137
- end
138
-
139
- os = d.instance.outputs
140
-
141
- assert_equal 6, os[0].events.size # weight=3
142
- assert_equal 6, os[1].events.size # weight=3
143
- assert_equal 2, os[2].events.size # weight=1
144
- end
145
- end
146
-
@@ -1,458 +0,0 @@
1
- require_relative '../helper'
2
- require 'time'
3
- require 'fileutils'
4
- require 'fluent/event'
5
- require 'fluent/unique_id'
6
- require 'fluent/plugin/buffer'
7
- require 'fluent/plugin/out_secondary_file'
8
- require 'fluent/plugin/buffer/memory_chunk'
9
- require 'fluent/test/driver/output'
10
-
11
- class FileOutputSecondaryTest < Test::Unit::TestCase
12
- include Fluent::UniqueId::Mixin
13
-
14
- def setup
15
- Fluent::Test.setup
16
- FileUtils.rm_rf(TMP_DIR)
17
- FileUtils.mkdir_p(TMP_DIR)
18
- end
19
-
20
- TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/out_secondary_file#{ENV['TEST_ENV_NUMBER']}")
21
-
22
- CONFIG = %[
23
- directory #{TMP_DIR}
24
- basename out_file_test
25
- compress gzip
26
- ]
27
-
28
- class DummyOutput < Fluent::Plugin::Output
29
- def write(chunk); end
30
- end
31
-
32
- def create_primary(buffer_cofig = config_element('buffer'))
33
- DummyOutput.new.configure(config_element('ROOT','',{}, [buffer_cofig]))
34
- end
35
-
36
- def create_driver(conf = CONFIG, primary = create_primary)
37
- c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)
38
- c.instance.acts_as_secondary(primary)
39
- c.configure(conf)
40
- end
41
-
42
- sub_test_case 'configure' do
43
- test 'default configuration' do
44
- d = create_driver %[directory #{TMP_DIR}]
45
- assert_equal 'dump.bin', d.instance.basename
46
- assert_equal TMP_DIR, d.instance.directory
47
- assert_equal :text, d.instance.compress
48
- assert_equal false, d.instance.append
49
- end
50
-
51
- test 'should be configurable' do
52
- d = create_driver %[
53
- directory #{TMP_DIR}
54
- basename out_file_test
55
- compress gzip
56
- append true
57
- ]
58
- assert_equal 'out_file_test', d.instance.basename
59
- assert_equal TMP_DIR, d.instance.directory
60
- assert_equal :gzip, d.instance.compress
61
- assert_equal true, d.instance.append
62
- end
63
-
64
- test 'should only use in secondary' do
65
- c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)
66
- assert_raise Fluent::ConfigError.new("This plugin can only be used in the <secondary> section") do
67
- c.configure(CONFIG)
68
- end
69
- end
70
-
71
- test 'basename should not include `/`' do
72
- assert_raise Fluent::ConfigError.new("basename should not include `/`") do
73
- create_driver %[
74
- directory #{TMP_DIR}
75
- basename out/file
76
- ]
77
- end
78
- end
79
-
80
- test 'directory should be writable' do
81
- assert_nothing_raised do
82
- create_driver %[directory #{TMP_DIR}/test_dir/foo/bar/]
83
- end
84
-
85
- assert_nothing_raised do
86
- FileUtils.mkdir_p("#{TMP_DIR}/test_dir")
87
- File.chmod(0777, "#{TMP_DIR}/test_dir")
88
- create_driver %[directory #{TMP_DIR}/test_dir/foo/bar/]
89
- end
90
-
91
- if Process.uid.nonzero?
92
- assert_raise Fluent::ConfigError.new("out_secondary_file: `#{TMP_DIR}/test_dir/foo/bar/` should be writable") do
93
- FileUtils.mkdir_p("#{TMP_DIR}/test_dir")
94
- File.chmod(0555, "#{TMP_DIR}/test_dir")
95
- create_driver %[directory #{TMP_DIR}/test_dir/foo/bar/]
96
- end
97
- end
98
- end
99
-
100
- test 'should be passed directory' do
101
- assert_raise Fluent::ConfigError do
102
- i = Fluent::Plugin::SecondaryFileOutput.new
103
- i.acts_as_secondary(create_primary)
104
- i.configure(config_element())
105
- end
106
-
107
- assert_nothing_raised do
108
- create_driver %[directory #{TMP_DIR}/test_dir/foo/bar/]
109
- end
110
- end
111
- end
112
-
113
- def check_gzipped_result(path, expect)
114
- # Zlib::GzipReader has a bug of concatenated file: https://bugs.ruby-lang.org/issues/9790
115
- # Following code from https://www.ruby-forum.com/topic/971591#979520
116
- result = ""
117
- waiting(10) do
118
- # we can expect that GzipReader#read can wait unflushed raw data of `io` on disk
119
- File.open(path, "rb") { |io|
120
- loop do
121
- gzr = Zlib::GzipReader.new(io)
122
- result << gzr.read
123
- unused = gzr.unused
124
- gzr.finish
125
- break if unused.nil?
126
- io.pos -= unused.length
127
- end
128
- }
129
- end
130
-
131
- assert_equal expect, result
132
- end
133
-
134
- def create_chunk(primary, metadata, es)
135
- primary.buffer.generate_chunk(metadata).tap do |c|
136
- c.concat(es.to_msgpack_stream, es.size) # to_msgpack_stream is standard_format
137
- c.commit
138
- end
139
- end
140
-
141
- sub_test_case 'write' do
142
- setup do
143
- @record = { 'key' => 'value' }
144
- @time = event_time
145
- @es = Fluent::OneEventStream.new(@time, @record)
146
- @primary = create_primary
147
- metadata = @primary.buffer.new_metadata
148
- @chunk = create_chunk(@primary, metadata, @es)
149
- end
150
-
151
- test 'should output compressed file when compress option is gzip' do
152
- d = create_driver(CONFIG, @primary)
153
- path = d.instance.write(@chunk)
154
-
155
- assert_equal "#{TMP_DIR}/out_file_test.0.gz", path
156
- check_gzipped_result(path, @es.to_msgpack_stream.force_encoding('ASCII-8BIT'))
157
- end
158
-
159
- test 'should output plain text when compress option is default(text)' do
160
- d = create_driver(%[
161
- directory #{TMP_DIR}/
162
- basename out_file_test
163
- ], @primary)
164
-
165
- msgpack_binary = @es.to_msgpack_stream.force_encoding('ASCII-8BIT')
166
-
167
- path = d.instance.write(@chunk)
168
- assert_equal "#{TMP_DIR}/out_file_test.0", path
169
- waiting(5) do
170
- sleep 0.1 until File.stat(path).size == msgpack_binary.size
171
- end
172
-
173
- assert_equal msgpack_binary, File.binread(path)
174
- end
175
-
176
- test 'path should be incremental when append option is false' do
177
- d = create_driver(CONFIG, @primary)
178
- packed_value = @es.to_msgpack_stream.force_encoding('ASCII-8BIT')
179
-
180
- 5.times do |i|
181
- path = d.instance.write(@chunk)
182
- assert_equal "#{TMP_DIR}/out_file_test.#{i}.gz", path
183
- check_gzipped_result(path, packed_value)
184
- end
185
- end
186
-
187
- test 'path should be unchanged when append option is true' do
188
- d = create_driver(CONFIG + %[append true], @primary)
189
- packed_value = @es.to_msgpack_stream.force_encoding('ASCII-8BIT')
190
-
191
- [*1..5].each do |i|
192
- path = d.instance.write(@chunk)
193
- assert_equal "#{TMP_DIR}/out_file_test.gz", path
194
- check_gzipped_result(path, packed_value * i)
195
- end
196
- end
197
- end
198
-
199
- sub_test_case 'Syntax of placeholders' do
200
- data(
201
- tag: '${tag}',
202
- tag_index: '${tag[0]}',
203
- tag_index1: '${tag[10]}',
204
- variable: '${key1}',
205
- variable2: '${key@value}',
206
- variable3: '${key_value}',
207
- variable4: '${key.value}',
208
- variable5: '${key-value}',
209
- variable6: '${KEYVALUE}',
210
- variable7: '${tags}',
211
- variable8: '${tag${key}', # matched ${key}
212
- )
213
- test 'matches with a valid placeholder' do |path|
214
- assert Fluent::Plugin::SecondaryFileOutput::PLACEHOLDER_REGEX.match(path)
215
- end
216
-
217
- data(
218
- invalid_tag: 'tag',
219
- invalid_tag2: '{tag}',
220
- invalid_tag3: '${tag',
221
- invalid_tag4: '${tag0]}',
222
- invalid_tag5: '${tag[]]}',
223
- invalid_variable: '${key[0]}',
224
- invalid_variable2: '${key{key2}}',
225
- )
226
- test "doesn't match with an invalid placeholder" do |path|
227
- assert !Fluent::Plugin::SecondaryFileOutput::PLACEHOLDER_REGEX.match(path)
228
- end
229
- end
230
-
231
- sub_test_case 'path' do
232
- setup do
233
- @record = { 'key' => 'value' }
234
- @time = event_time
235
- @es = Fluent::OneEventStream.new(@time, @record)
236
- primary = create_primary
237
- m = primary.buffer.new_metadata
238
- @c = create_chunk(primary, m, @es)
239
- end
240
-
241
- test 'normal path when compress option is gzip' do
242
- d = create_driver
243
- path = d.instance.write(@c)
244
- assert_equal "#{TMP_DIR}/out_file_test.0.gz", path
245
- end
246
-
247
- test 'normal path when compress option is default' do
248
- d = create_driver %[
249
- directory #{TMP_DIR}
250
- basename out_file_test
251
- ]
252
- path = d.instance.write(@c)
253
- assert_equal "#{TMP_DIR}/out_file_test.0", path
254
- end
255
-
256
- test 'normal path when append option is true' do
257
- d = create_driver %[
258
- directory #{TMP_DIR}
259
- append true
260
- ]
261
- path = d.instance.write(@c)
262
- assert_equal "#{TMP_DIR}/dump.bin", path
263
- end
264
-
265
- test 'path with ${chunk_id}' do
266
- d = create_driver %[
267
- directory #{TMP_DIR}
268
- basename out_file_chunk_id_${chunk_id}
269
- ]
270
- path = d.instance.write(@c)
271
- if File.basename(path) =~ /out_file_chunk_id_([-_.@a-zA-Z0-9].*).0/
272
- unique_id = Fluent::UniqueId.hex(Fluent::UniqueId.generate)
273
- assert_equal unique_id.size, $1.size, "chunk_id size is mismatched"
274
- else
275
- flunk "chunk_id is not included in the path"
276
- end
277
- end
278
-
279
- data(
280
- invalid_tag: [/tag/, '${tag}'],
281
- invalid_tag0: [/tag\[0\]/, '${tag[0]}'],
282
- invalid_variable: [/dummy/, '${dummy}'],
283
- invalid_timeformat: [/time/, '%Y%m%d'],
284
- )
285
- test 'raise an error when basename includes incompatible placeholder' do |(expected_message, invalid_basename)|
286
- c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)
287
- c.instance.acts_as_secondary(DummyOutput.new)
288
-
289
- assert_raise_message(expected_message) do
290
- c.configure %[
291
- directory #{TMP_DIR}/
292
- basename #{invalid_basename}
293
- compress gzip
294
- ]
295
- end
296
- end
297
-
298
- data(
299
- invalid_tag: [/tag/, '${tag}'],
300
- invalid_tag0: [/tag\[0\]/, '${tag[0]}'],
301
- invalid_variable: [/dummy/, '${dummy}'],
302
- invalid_timeformat: [/time/, '%Y%m%d'],
303
- )
304
- test 'raise an error when directory includes incompatible placeholder' do |(expected_message, invalid_directory)|
305
- c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)
306
- c.instance.acts_as_secondary(DummyOutput.new)
307
-
308
- assert_raise_message(expected_message) do
309
- c.configure %[
310
- directory #{invalid_directory}/
311
- compress gzip
312
- ]
313
- end
314
- end
315
-
316
- test 'basename includes tag' do
317
- primary = create_primary(config_element('buffer', 'tag'))
318
-
319
- d = create_driver(%[
320
- directory #{TMP_DIR}/
321
- basename cool_${tag}
322
- compress gzip
323
- ], primary)
324
-
325
- m = primary.buffer.new_metadata(tag: 'test.dummy')
326
- c = create_chunk(primary, m, @es)
327
-
328
- path = d.instance.write(c)
329
- assert_equal "#{TMP_DIR}/cool_test.dummy.0.gz", path
330
- end
331
-
332
- test 'basename includes /tag[\d+]/' do
333
- primary = create_primary(config_element('buffer', 'tag'))
334
-
335
- d = create_driver(%[
336
- directory #{TMP_DIR}/
337
- basename cool_${tag[0]}_${tag[1]}
338
- compress gzip
339
- ], primary)
340
-
341
- m = primary.buffer.new_metadata(tag: 'test.dummy')
342
- c = create_chunk(primary, m, @es)
343
-
344
- path = d.instance.write(c)
345
- assert_equal "#{TMP_DIR}/cool_test_dummy.0.gz", path
346
- end
347
-
348
- test 'basename includes time format' do
349
- primary = create_primary(
350
- config_element('buffer', 'time', { 'timekey_zone' => '+0900', 'timekey' => 1 })
351
- )
352
-
353
- d = create_driver(%[
354
- directory #{TMP_DIR}/
355
- basename cool_%Y%m%d%H
356
- compress gzip
357
- ], primary)
358
-
359
- m = primary.buffer.new_metadata(timekey: event_time("2011-01-02 13:14:15 UTC"))
360
- c = create_chunk(primary, m, @es)
361
-
362
- path = d.instance.write(c)
363
- assert_equal "#{TMP_DIR}/cool_2011010222.0.gz", path
364
- end
365
-
366
- test 'basename includes time format with timekey_use_utc option' do
367
- primary = create_primary(
368
- config_element('buffer', 'time', { 'timekey_zone' => '+0900', 'timekey' => 1, 'timekey_use_utc' => true })
369
- )
370
-
371
- d = create_driver(%[
372
- directory #{TMP_DIR}/
373
- basename cool_%Y%m%d%H
374
- compress gzip
375
- ], primary)
376
-
377
- m = primary.buffer.new_metadata(timekey: event_time("2011-01-02 13:14:15 UTC"))
378
- c = create_chunk(primary, m, @es)
379
-
380
- path = d.instance.write(c)
381
- assert_equal "#{TMP_DIR}/cool_2011010213.0.gz", path
382
- end
383
-
384
- test 'basename includes variable' do
385
- primary = create_primary(config_element('buffer', 'test1'))
386
-
387
- d = create_driver(%[
388
- directory #{TMP_DIR}/
389
- basename cool_${test1}
390
- compress gzip
391
- ], primary)
392
-
393
- m = primary.buffer.new_metadata(variables: { "test1".to_sym => "dummy" })
394
- c = create_chunk(primary, m, @es)
395
-
396
- path = d.instance.write(c)
397
- assert_equal "#{TMP_DIR}/cool_dummy.0.gz", path
398
- end
399
-
400
- test 'basename includes unnecessary variable' do
401
- primary = create_primary(config_element('buffer', 'test1'))
402
- c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)
403
- c.instance.acts_as_secondary(primary)
404
-
405
- assert_raise_message(/test2/) do
406
- c.configure %[
407
- directory #{TMP_DIR}/
408
- basename ${test1}_${test2}
409
- compress gzip
410
- ]
411
- end
412
- end
413
-
414
- test 'basename includes tag, time format, and variables' do
415
- primary = create_primary(
416
- config_element('buffer', 'time,tag,test1', { 'timekey_zone' => '+0000', 'timekey' => 1 })
417
- )
418
-
419
- d = create_driver(%[
420
- directory #{TMP_DIR}/
421
- basename cool_%Y%m%d%H_${tag}_${test1}
422
- compress gzip
423
- ], primary)
424
-
425
- m = primary.buffer.new_metadata(
426
- timekey: event_time("2011-01-02 13:14:15 UTC"),
427
- tag: 'test.tag',
428
- variables: { "test1".to_sym => "dummy" }
429
- )
430
-
431
- c = create_chunk(primary, m, @es)
432
-
433
- path = d.instance.write(c)
434
- assert_equal "#{TMP_DIR}/cool_2011010213_test.tag_dummy.0.gz", path
435
- end
436
-
437
- test 'directory includes tag, time format, and variables' do
438
- primary = create_primary(
439
- config_element('buffer', 'time,tag,test1', { 'timekey_zone' => '+0000', 'timekey' => 1 })
440
- )
441
-
442
- d = create_driver(%[
443
- directory #{TMP_DIR}/%Y%m%d%H/${tag}/${test1}
444
- compress gzip
445
- ], primary)
446
-
447
- m = primary.buffer.new_metadata(
448
- timekey: event_time("2011-01-02 13:14:15 UTC"),
449
- tag: 'test.tag',
450
- variables: { "test1".to_sym => "dummy" }
451
- )
452
- c = create_chunk(primary, m, @es)
453
-
454
- path = d.instance.write(c)
455
- assert_equal "#{TMP_DIR}/2011010213/test.tag/dummy/dump.bin.0.gz", path
456
- end
457
- end
458
- end