fluentd 1.17.0 → 1.17.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
@@ -1,363 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/plugin/output'
3
- require 'fluent/plugin/buffer'
4
- require 'fluent/event'
5
- require 'fluent/error'
6
-
7
- require 'json'
8
- require 'time'
9
- require 'timeout'
10
- require 'timecop'
11
-
12
-
13
- class BufferedOutputBackupTest < Test::Unit::TestCase
14
- class BareOutput < Fluent::Plugin::Output
15
- def register(name, &block)
16
- instance_variable_set("@#{name}", block)
17
- end
18
- end
19
- class DummyOutput < BareOutput
20
- def initialize
21
- super
22
- @process = nil
23
- @format = nil
24
- @write = nil
25
- @try_write = nil
26
- end
27
- def prefer_buffered_processing
28
- true
29
- end
30
- def prefer_delayed_commit
31
- false
32
- end
33
- def process(tag, es)
34
- @process ? @process.call(tag, es) : nil
35
- end
36
- def format(tag, time, record)
37
- [tag, time.to_i, record].to_json + "\n"
38
- end
39
- def write(chunk)
40
- @write ? @write.call(chunk) : nil
41
- end
42
- def try_write(chunk)
43
- @try_write ? @try_write.call(chunk) : nil
44
- end
45
- end
46
- class DummyOutputForSecondary < BareOutput
47
- def initialize
48
- super
49
- @process = nil
50
- @format = nil
51
- @write = nil
52
- @try_write = nil
53
- end
54
- def prefer_buffered_processing
55
- true
56
- end
57
- def prefer_delayed_commit
58
- false
59
- end
60
- def process(tag, es)
61
- @process ? @process.call(tag, es) : nil
62
- end
63
- def format(tag, time, record)
64
- [tag, time.to_i, record].to_json + "\n"
65
- end
66
- def write(chunk)
67
- @write ? @write.call(chunk) : nil
68
- end
69
- def try_write(chunk)
70
- @try_write ? @try_write.call(chunk) : nil
71
- end
72
- end
73
- class DummyAsyncOutputForSecondary < BareOutput
74
- def initialize
75
- super
76
- @process = nil
77
- @format = nil
78
- @write = nil
79
- @try_write = nil
80
- end
81
- def prefer_buffered_processing
82
- true
83
- end
84
- def prefer_delayed_commit
85
- true
86
- end
87
- def process(tag, es)
88
- @process ? @process.call(tag, es) : nil
89
- end
90
- def format(tag, time, record)
91
- [tag, time.to_i, record].to_json + "\n"
92
- end
93
- def write(chunk)
94
- @write ? @write.call(chunk) : nil
95
- end
96
- def try_write(chunk)
97
- @try_write ? @try_write.call(chunk) : nil
98
- end
99
- end
100
-
101
- TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/bu#{ENV['TEST_ENV_NUMBER']}")
102
-
103
- def create_output
104
- DummyOutput.new
105
- end
106
- def create_metadata(timekey: nil, tag: nil, variables: nil)
107
- Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
108
- end
109
- def waiting(seconds)
110
- begin
111
- Timeout.timeout(seconds) do
112
- yield
113
- end
114
- rescue Timeout::Error
115
- STDERR.print(*@i.log.out.logs)
116
- raise
117
- end
118
- end
119
-
120
- def dummy_event_stream
121
- Fluent::ArrayEventStream.new([
122
- [ event_time('2016-04-13 18:33:00'), {"name" => "moris", "age" => 36, "message" => "data1"} ],
123
- [ event_time('2016-04-13 18:33:13'), {"name" => "moris", "age" => 36, "message" => "data2"} ],
124
- [ event_time('2016-04-13 18:33:32'), {"name" => "moris", "age" => 36, "message" => "data3"} ],
125
- ])
126
- end
127
-
128
- setup do
129
- @i = create_output
130
- FileUtils.rm_rf(TMP_DIR)
131
- FileUtils.mkdir_p(TMP_DIR)
132
-
133
- Fluent::Plugin.register_output('backup_output', DummyOutput)
134
- Fluent::Plugin.register_output('backup_output2', DummyOutputForSecondary)
135
- Fluent::Plugin.register_output('backup_async_output', DummyAsyncOutputForSecondary)
136
- end
137
-
138
- teardown do
139
- if @i
140
- @i.stop unless @i.stopped?
141
- @i.before_shutdown unless @i.before_shutdown?
142
- @i.shutdown unless @i.shutdown?
143
- @i.after_shutdown unless @i.after_shutdown?
144
- @i.close unless @i.closed?
145
- @i.terminate unless @i.terminated?
146
- end
147
- Timecop.return
148
- end
149
-
150
- sub_test_case 'buffered output for broken chunks' do
151
- def flush_chunks
152
- @i.start
153
- @i.after_start
154
-
155
- @i.interrupt_flushes
156
-
157
- now = Time.parse('2016-04-13 18:33:30 -0700')
158
- Timecop.freeze(now)
159
- @i.emit_events("test.tag.1", dummy_event_stream())
160
- now = Time.parse('2016-04-13 18:33:32 -0700')
161
- Timecop.freeze(now)
162
-
163
- @i.enqueue_thread_wait
164
- @i.flush_thread_wakeup
165
- waiting(4) { Thread.pass until @i.write_count > 0 }
166
-
167
- assert { @i.write_count > 0 }
168
- Timecop.freeze(now)
169
- @i.flush_thread_wakeup
170
- end
171
-
172
- def wait_flush(target_file)
173
- waiting(5) {
174
- target_dir = File.join(File.dirname(target_file), "*")
175
- while Dir.glob(target_dir).size.zero?
176
- # Avoid to lose globbed entries on Windows in busy loop
177
- sleep 0.1 if Fluent.windows?
178
- end
179
- }
180
- end
181
-
182
- data('unrecoverable error' => Fluent::UnrecoverableError,
183
- 'type error' => TypeError,
184
- 'argument error' => ArgumentError,
185
- 'no method error' => NoMethodError,
186
- 'msgpack unpack error' => MessagePack::UnpackError,
187
- 'encoding error' => EncodingError)
188
- test 'backup chunk without secondary' do |error_class|
189
- Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
190
- id = 'backup_test'
191
- hash = {
192
- 'flush_interval' => 1,
193
- 'flush_thread_burst_interval' => 0.1,
194
- }
195
- chunk_id = nil
196
- @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))
197
- @i.register(:write) { |chunk|
198
- chunk_id = chunk.unique_id;
199
- raise error_class, "yay, your #write must fail"
200
- }
201
-
202
- flush_chunks
203
-
204
- target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
205
- wait_flush(target)
206
- assert_true File.exist?(target)
207
- logs = @i.log.out.logs
208
- assert { logs.any? { |l| l.include?("got unrecoverable error in primary and no secondary") } }
209
- end
210
- end
211
-
212
- test 'backup chunk with same type secondary' do
213
- Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
214
- id = 'backup_test_with_same_secondary'
215
- hash = {
216
- 'flush_interval' => 1,
217
- 'flush_thread_burst_interval' => 0.1,
218
- }
219
- chunk_id = nil
220
- secconf = config_element('secondary','',{'@type' => 'backup_output'})
221
- @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
222
- @i.register(:write) { |chunk|
223
- chunk_id = chunk.unique_id;
224
- raise Fluent::UnrecoverableError, "yay, your #write must fail"
225
- }
226
-
227
- flush_chunks
228
-
229
- target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
230
- wait_flush(target)
231
- assert_true File.exist?(target)
232
- logs = @i.log.out.logs
233
- assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary type is same as primary") } }
234
- end
235
- end
236
-
237
- test 'create directory' do
238
- Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
239
- id = 'backup_test_with_same_secondary'
240
- hash = { 'flush_interval' => 1, 'flush_thread_burst_interval' => 0.1 }
241
- chunk_id = nil
242
- secconf = config_element('secondary', '', { '@type' => 'backup_output' })
243
- @i.configure(config_element('ROOT', '', { '@id' => id }, [config_element('buffer', 'tag', hash), secconf]))
244
- @i.register(:write) { |chunk|
245
- chunk_id = chunk.unique_id;
246
- raise Fluent::UnrecoverableError, "yay, your #write must fail"
247
- }
248
-
249
- flush_chunks
250
-
251
- target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
252
- target_dir = File.dirname(target)
253
- wait_flush(target)
254
-
255
- assert_path_exist(target_dir)
256
- assert_equal '755', File.stat(target_dir).mode.to_s(8)[-3, 3]
257
- end
258
- end
259
-
260
- test 'create directory with specific mode' do
261
- omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
262
-
263
- Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR, 'dir_permission' => '744') do
264
- id = 'backup_test_with_same_secondary'
265
- hash = { 'flush_interval' => 1, 'flush_thread_burst_interval' => 0.1 }
266
- chunk_id = nil
267
- secconf = config_element('secondary', '', { '@type' => 'backup_output' })
268
- @i.configure(config_element('ROOT', '', { '@id' => id }, [config_element('buffer', 'tag', hash), secconf]))
269
- @i.register(:write) { |chunk|
270
- chunk_id = chunk.unique_id;
271
- raise Fluent::UnrecoverableError, "yay, your #write must fail"
272
- }
273
-
274
- flush_chunks
275
-
276
- target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
277
- target_dir = File.dirname(target)
278
- wait_flush(target)
279
-
280
- assert_path_exist(target_dir)
281
- assert_equal '744', File.stat(target_dir).mode.to_s(8)[-3, 3]
282
- end
283
- end
284
-
285
- test 'backup chunk with different type secondary' do
286
- Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
287
- id = 'backup_test_with_diff_secondary'
288
- hash = {
289
- 'flush_interval' => 1,
290
- 'flush_thread_burst_interval' => 0.1,
291
- }
292
- chunk_id = nil
293
- secconf = config_element('secondary','',{'@type' => 'backup_output2'})
294
- @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
295
- @i.register(:write) { |chunk|
296
- chunk_id = chunk.unique_id;
297
- raise Fluent::UnrecoverableError, "yay, your #write must fail"
298
- }
299
- @i.secondary.register(:write) { |chunk|
300
- raise Fluent::UnrecoverableError, "yay, your secondary #write must fail"
301
- }
302
-
303
- flush_chunks
304
-
305
- target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
306
- wait_flush(target)
307
- assert_true File.exist?(target)
308
- logs = @i.log.out.logs
309
- assert { logs.any? { |l| l.include?("got unrecoverable error in primary. Skip retry and flush chunk to secondary") } }
310
- assert { logs.any? { |l| l.include?("got an error in secondary for unrecoverable error") } }
311
- end
312
- end
313
-
314
- test 'backup chunk with async secondary' do
315
- Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
316
- id = 'backup_test_with_diff_secondary'
317
- hash = {
318
- 'flush_interval' => 1,
319
- 'flush_thread_burst_interval' => 0.1,
320
- }
321
- chunk_id = nil
322
- secconf = config_element('secondary','',{'@type' => 'backup_async_output'})
323
- @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
324
- @i.register(:write) { |chunk|
325
- chunk_id = chunk.unique_id;
326
- raise Fluent::UnrecoverableError, "yay, your #write must fail"
327
- }
328
-
329
- flush_chunks
330
-
331
- target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
332
- wait_flush(target)
333
- assert_true File.exist?(target)
334
- logs = @i.log.out.logs
335
- assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary is async output") } }
336
- end
337
- end
338
-
339
- test 'chunk is thrown away when disable_chunk_backup is true' do
340
- Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
341
- id = 'backup_test'
342
- hash = {
343
- 'flush_interval' => 1,
344
- 'flush_thread_burst_interval' => 0.1,
345
- 'disable_chunk_backup' => true
346
- }
347
- chunk_id = nil
348
- @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))
349
- @i.register(:write) { |chunk|
350
- chunk_id = chunk.unique_id;
351
- raise Fluent::UnrecoverableError, "yay, your #write must fail"
352
- }
353
-
354
- flush_chunks
355
-
356
- target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
357
- assert_false File.exist?(target)
358
- logs = @i.log.out.logs
359
- assert { logs.any? { |l| l.include?("disable_chunk_backup is true") } }
360
- end
361
- end
362
- end
363
- end
@@ -1,179 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/plugin/output'
3
- require 'fluent/plugin/buffer'
4
- require 'fluent/plugin/compressable'
5
- require 'fluent/event'
6
-
7
- require 'timeout'
8
-
9
- module FluentPluginOutputAsBufferedCompressTest
10
- class DummyBareOutput < Fluent::Plugin::Output
11
- def register(name, &block)
12
- instance_variable_set("@#{name}", block)
13
- end
14
- end
15
-
16
- class DummyAsyncOutput < DummyBareOutput
17
- def initialize
18
- super
19
- @format = @write = nil
20
- end
21
- def write(chunk)
22
- @write ? @write.call(chunk) : nil
23
- end
24
- end
25
-
26
- class DummyAsyncOutputWithFormat < DummyBareOutput
27
- def initialize
28
- super
29
- @format = nil
30
- end
31
- def write(chunk)
32
- @write ? @write.call(chunk) : nil
33
- end
34
- def format(tag, time, record)
35
- @format ? @format.call(tag, time, record) : [tag, time, record].to_json
36
- end
37
- end
38
-
39
- def self.dummy_event_stream
40
- Fluent::ArrayEventStream.new(
41
- [
42
- [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
43
- [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
44
- [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
45
- ]
46
- )
47
- end
48
- end
49
-
50
- class BufferedOutputCompressTest < Test::Unit::TestCase
51
- include Fluent::Plugin::Compressable
52
-
53
- def create_output(type=:async)
54
- case type
55
- when :async then FluentPluginOutputAsBufferedCompressTest::DummyAsyncOutput.new
56
- when :async_with_format then FluentPluginOutputAsBufferedCompressTest::DummyAsyncOutputWithFormat.new
57
- else
58
- raise ArgumentError, "unknown type: #{type}"
59
- end
60
- end
61
-
62
- def waiting(seconds)
63
- begin
64
- Timeout.timeout(seconds) do
65
- yield
66
- end
67
- rescue Timeout::Error
68
- STDERR.print(*@i.log.out.logs)
69
- raise
70
- end
71
- end
72
-
73
- TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
74
-
75
- setup do
76
- FileUtils.rm_r TMP_DIR rescue nil
77
- FileUtils.mkdir_p TMP_DIR
78
- end
79
-
80
- teardown do
81
- if @i
82
- @i.stop unless @i.stopped?
83
- @i.before_shutdown unless @i.before_shutdown?
84
- @i.shutdown unless @i.shutdown?
85
- @i.after_shutdown unless @i.after_shutdown?
86
- @i.close unless @i.closed?
87
- @i.terminate unless @i.terminated?
88
- end
89
- end
90
-
91
- data(
92
- :buffer_config,
93
- [
94
- config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
95
- config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
96
- config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
97
- config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
98
- ],
99
- )
100
- data(
101
- :input_es,
102
- [
103
- FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
104
- # If already compressed data is incoming, it must be written as is (i.e. without decompressed).
105
- # https://github.com/fluent/fluentd/issues/4146
106
- Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
107
- ],
108
- )
109
- test 'call a standard format when output plugin adds data to chunk' do |data|
110
- buffer_config = data[:buffer_config]
111
- es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
112
-
113
- @i = create_output(:async)
114
- @i.configure(config_element('ROOT','', {}, [buffer_config]))
115
- @i.start
116
- @i.after_start
117
-
118
- io = StringIO.new
119
- expected = es.dup.map { |t, r| [t, r] }
120
- compressed_data = ''
121
-
122
- assert_equal :gzip, @i.buffer.compress
123
-
124
- @i.register(:write) do |c|
125
- compressed_data = c.instance_variable_get(:@chunk)
126
- if compressed_data.is_a?(File)
127
- compressed_data.seek(0, IO::SEEK_SET)
128
- compressed_data = compressed_data.read
129
- end
130
- c.write_to(io)
131
- end
132
-
133
- @i.emit_events('tag', es)
134
- @i.enqueue_thread_wait
135
- @i.flush_thread_wakeup
136
- waiting(4) { Thread.pass until io.size > 0 }
137
-
138
- assert_equal expected, Fluent::MessagePackEventStream.new(decompress(compressed_data)).map { |t, r| [t, r] }
139
- assert_equal expected, Fluent::MessagePackEventStream.new(io.string).map { |t, r| [t, r] }
140
- end
141
-
142
- data(
143
- handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
144
- handle_stream_with_custom_format: config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
145
- handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
146
- handle_stream_with_custom_format_and_file_chunk: config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
147
- )
148
- test 'call a custom format when output plugin adds data to chunk' do |buffer_config|
149
- @i = create_output(:async_with_format)
150
- @i.configure(config_element('ROOT','', {}, [buffer_config]))
151
- @i.start
152
- @i.after_start
153
-
154
- io = StringIO.new
155
- es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
156
- expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
157
- compressed_data = ''
158
-
159
- assert_equal :gzip, @i.buffer.compress
160
-
161
- @i.register(:format) { |tag, time, record| "#{record}\n" }
162
- @i.register(:write) { |c|
163
- compressed_data = c.instance_variable_get(:@chunk)
164
- if compressed_data.is_a?(File)
165
- compressed_data.seek(0, IO::SEEK_SET)
166
- compressed_data = compressed_data.read
167
- end
168
- c.write_to(io)
169
- }
170
-
171
- @i.emit_events('tag', es)
172
- @i.enqueue_thread_wait
173
- @i.flush_thread_wakeup
174
- waiting(4) { sleep 0.1 until io.size > 0 }
175
-
176
- assert_equal expected, decompress(compressed_data)
177
- assert_equal expected, io.string
178
- end
179
- end