fluentd 1.17.0-x64-mingw-ucrt → 1.18.0-x64-mingw-ucrt

Sign up to get free protection for your applications and to get access to all the features.
Files changed (278) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +100 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/command/fluentd.rb +7 -1
  8. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  9. data/lib/fluent/compat/propagate_default.rb +4 -4
  10. data/lib/fluent/config/literal_parser.rb +9 -2
  11. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  12. data/lib/fluent/engine.rb +49 -33
  13. data/lib/fluent/env.rb +3 -0
  14. data/lib/fluent/event_router.rb +2 -2
  15. data/lib/fluent/log/console_adapter.rb +4 -2
  16. data/lib/fluent/plugin/filter_parser.rb +27 -51
  17. data/lib/fluent/plugin/in_exec.rb +14 -2
  18. data/lib/fluent/plugin/in_http.rb +6 -1
  19. data/lib/fluent/plugin/in_sample.rb +13 -7
  20. data/lib/fluent/plugin/in_syslog.rb +4 -0
  21. data/lib/fluent/plugin/in_tail.rb +65 -23
  22. data/lib/fluent/plugin/in_tcp.rb +4 -0
  23. data/lib/fluent/plugin/in_udp.rb +10 -1
  24. data/lib/fluent/plugin/input.rb +4 -0
  25. data/lib/fluent/plugin/out_buffer.rb +40 -0
  26. data/lib/fluent/plugin/out_copy.rb +1 -1
  27. data/lib/fluent/plugin/out_file.rb +8 -0
  28. data/lib/fluent/plugin/out_http.rb +12 -0
  29. data/lib/fluent/plugin/output.rb +2 -0
  30. data/lib/fluent/plugin/parser_json.rb +4 -12
  31. data/lib/fluent/plugin_helper/cert_option.rb +8 -0
  32. data/lib/fluent/plugin_helper/event_emitter.rb +12 -0
  33. data/lib/fluent/plugin_helper/http_server/server.rb +24 -8
  34. data/lib/fluent/plugin_helper/server.rb +9 -0
  35. data/lib/fluent/root_agent.rb +114 -19
  36. data/lib/fluent/source_only_buffer_agent.rb +102 -0
  37. data/lib/fluent/supervisor.rb +207 -34
  38. data/lib/fluent/system_config.rb +15 -3
  39. data/lib/fluent/version.rb +1 -1
  40. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  41. metadata +24 -483
  42. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  43. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  44. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  45. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  46. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  47. data/.github/ISSUE_TEMPLATE.md +0 -17
  48. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  49. data/.github/workflows/stale-actions.yml +0 -24
  50. data/.github/workflows/test-ruby-head.yml +0 -31
  51. data/.github/workflows/test.yml +0 -32
  52. data/.gitignore +0 -30
  53. data/Gemfile +0 -9
  54. data/fluentd.gemspec +0 -62
  55. data/test/command/test_binlog_reader.rb +0 -362
  56. data/test/command/test_ca_generate.rb +0 -70
  57. data/test/command/test_cap_ctl.rb +0 -100
  58. data/test/command/test_cat.rb +0 -128
  59. data/test/command/test_ctl.rb +0 -56
  60. data/test/command/test_fluentd.rb +0 -1291
  61. data/test/command/test_plugin_config_formatter.rb +0 -397
  62. data/test/command/test_plugin_generator.rb +0 -109
  63. data/test/compat/test_calls_super.rb +0 -166
  64. data/test/compat/test_parser.rb +0 -92
  65. data/test/config/assertions.rb +0 -42
  66. data/test/config/test_config_parser.rb +0 -551
  67. data/test/config/test_configurable.rb +0 -1784
  68. data/test/config/test_configure_proxy.rb +0 -604
  69. data/test/config/test_dsl.rb +0 -415
  70. data/test/config/test_element.rb +0 -518
  71. data/test/config/test_literal_parser.rb +0 -309
  72. data/test/config/test_plugin_configuration.rb +0 -56
  73. data/test/config/test_section.rb +0 -191
  74. data/test/config/test_system_config.rb +0 -195
  75. data/test/config/test_types.rb +0 -408
  76. data/test/counter/test_client.rb +0 -563
  77. data/test/counter/test_error.rb +0 -44
  78. data/test/counter/test_mutex_hash.rb +0 -179
  79. data/test/counter/test_server.rb +0 -589
  80. data/test/counter/test_store.rb +0 -258
  81. data/test/counter/test_validator.rb +0 -137
  82. data/test/helper.rb +0 -155
  83. data/test/helpers/fuzzy_assert.rb +0 -89
  84. data/test/helpers/process_extenstion.rb +0 -33
  85. data/test/log/test_console_adapter.rb +0 -117
  86. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  87. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  88. data/test/plugin/data/2010/01/20100102.log +0 -0
  89. data/test/plugin/data/log/bar +0 -0
  90. data/test/plugin/data/log/foo/bar.log +0 -0
  91. data/test/plugin/data/log/foo/bar2 +0 -0
  92. data/test/plugin/data/log/test.log +0 -0
  93. data/test/plugin/data/log_numeric/01.log +0 -0
  94. data/test/plugin/data/log_numeric/02.log +0 -0
  95. data/test/plugin/data/log_numeric/12.log +0 -0
  96. data/test/plugin/data/log_numeric/14.log +0 -0
  97. data/test/plugin/data/sd_file/config +0 -11
  98. data/test/plugin/data/sd_file/config.json +0 -17
  99. data/test/plugin/data/sd_file/config.yaml +0 -11
  100. data/test/plugin/data/sd_file/config.yml +0 -11
  101. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  102. data/test/plugin/in_tail/test_fifo.rb +0 -121
  103. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  104. data/test/plugin/in_tail/test_position_file.rb +0 -346
  105. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  106. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  107. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  108. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  109. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  110. data/test/plugin/test_bare_output.rb +0 -131
  111. data/test/plugin/test_base.rb +0 -247
  112. data/test/plugin/test_buf_file.rb +0 -1314
  113. data/test/plugin/test_buf_file_single.rb +0 -898
  114. data/test/plugin/test_buf_memory.rb +0 -42
  115. data/test/plugin/test_buffer.rb +0 -1493
  116. data/test/plugin/test_buffer_chunk.rb +0 -209
  117. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  118. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  119. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  120. data/test/plugin/test_compressable.rb +0 -87
  121. data/test/plugin/test_file_util.rb +0 -96
  122. data/test/plugin/test_filter.rb +0 -368
  123. data/test/plugin/test_filter_grep.rb +0 -697
  124. data/test/plugin/test_filter_parser.rb +0 -731
  125. data/test/plugin/test_filter_record_transformer.rb +0 -577
  126. data/test/plugin/test_filter_stdout.rb +0 -207
  127. data/test/plugin/test_formatter_csv.rb +0 -136
  128. data/test/plugin/test_formatter_hash.rb +0 -38
  129. data/test/plugin/test_formatter_json.rb +0 -61
  130. data/test/plugin/test_formatter_ltsv.rb +0 -70
  131. data/test/plugin/test_formatter_msgpack.rb +0 -28
  132. data/test/plugin/test_formatter_out_file.rb +0 -116
  133. data/test/plugin/test_formatter_single_value.rb +0 -44
  134. data/test/plugin/test_formatter_tsv.rb +0 -76
  135. data/test/plugin/test_in_debug_agent.rb +0 -49
  136. data/test/plugin/test_in_exec.rb +0 -261
  137. data/test/plugin/test_in_forward.rb +0 -1178
  138. data/test/plugin/test_in_gc_stat.rb +0 -62
  139. data/test/plugin/test_in_http.rb +0 -1124
  140. data/test/plugin/test_in_monitor_agent.rb +0 -922
  141. data/test/plugin/test_in_object_space.rb +0 -66
  142. data/test/plugin/test_in_sample.rb +0 -190
  143. data/test/plugin/test_in_syslog.rb +0 -505
  144. data/test/plugin/test_in_tail.rb +0 -3429
  145. data/test/plugin/test_in_tcp.rb +0 -328
  146. data/test/plugin/test_in_udp.rb +0 -296
  147. data/test/plugin/test_in_unix.rb +0 -181
  148. data/test/plugin/test_input.rb +0 -137
  149. data/test/plugin/test_metadata.rb +0 -89
  150. data/test/plugin/test_metrics.rb +0 -294
  151. data/test/plugin/test_metrics_local.rb +0 -96
  152. data/test/plugin/test_multi_output.rb +0 -204
  153. data/test/plugin/test_out_copy.rb +0 -308
  154. data/test/plugin/test_out_exec.rb +0 -312
  155. data/test/plugin/test_out_exec_filter.rb +0 -606
  156. data/test/plugin/test_out_file.rb +0 -1038
  157. data/test/plugin/test_out_forward.rb +0 -1349
  158. data/test/plugin/test_out_http.rb +0 -557
  159. data/test/plugin/test_out_null.rb +0 -105
  160. data/test/plugin/test_out_relabel.rb +0 -28
  161. data/test/plugin/test_out_roundrobin.rb +0 -146
  162. data/test/plugin/test_out_secondary_file.rb +0 -458
  163. data/test/plugin/test_out_stdout.rb +0 -205
  164. data/test/plugin/test_out_stream.rb +0 -103
  165. data/test/plugin/test_output.rb +0 -1334
  166. data/test/plugin/test_output_as_buffered.rb +0 -2024
  167. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  168. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  169. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  170. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  171. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  172. data/test/plugin/test_output_as_standard.rb +0 -374
  173. data/test/plugin/test_owned_by.rb +0 -34
  174. data/test/plugin/test_parser.rb +0 -399
  175. data/test/plugin/test_parser_apache.rb +0 -42
  176. data/test/plugin/test_parser_apache2.rb +0 -47
  177. data/test/plugin/test_parser_apache_error.rb +0 -45
  178. data/test/plugin/test_parser_csv.rb +0 -200
  179. data/test/plugin/test_parser_json.rb +0 -244
  180. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  181. data/test/plugin/test_parser_msgpack.rb +0 -127
  182. data/test/plugin/test_parser_multiline.rb +0 -111
  183. data/test/plugin/test_parser_nginx.rb +0 -88
  184. data/test/plugin/test_parser_none.rb +0 -52
  185. data/test/plugin/test_parser_regexp.rb +0 -284
  186. data/test/plugin/test_parser_syslog.rb +0 -650
  187. data/test/plugin/test_parser_tsv.rb +0 -122
  188. data/test/plugin/test_sd_file.rb +0 -228
  189. data/test/plugin/test_sd_srv.rb +0 -230
  190. data/test/plugin/test_storage.rb +0 -166
  191. data/test/plugin/test_storage_local.rb +0 -335
  192. data/test/plugin/test_string_util.rb +0 -26
  193. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  195. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  196. data/test/plugin_helper/data/cert/cert.pem +0 -19
  197. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  198. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  199. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  200. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  201. data/test/plugin_helper/data/cert/empty.pem +0 -0
  202. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  203. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  204. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  205. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  206. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  207. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  208. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  209. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  210. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  211. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  212. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  213. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  214. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  215. data/test/plugin_helper/http_server/test_app.rb +0 -65
  216. data/test/plugin_helper/http_server/test_route.rb +0 -32
  217. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  218. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  219. data/test/plugin_helper/test_cert_option.rb +0 -25
  220. data/test/plugin_helper/test_child_process.rb +0 -862
  221. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  222. data/test/plugin_helper/test_event_emitter.rb +0 -80
  223. data/test/plugin_helper/test_event_loop.rb +0 -52
  224. data/test/plugin_helper/test_extract.rb +0 -194
  225. data/test/plugin_helper/test_formatter.rb +0 -255
  226. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  227. data/test/plugin_helper/test_inject.rb +0 -561
  228. data/test/plugin_helper/test_metrics.rb +0 -137
  229. data/test/plugin_helper/test_parser.rb +0 -264
  230. data/test/plugin_helper/test_record_accessor.rb +0 -238
  231. data/test/plugin_helper/test_retry_state.rb +0 -1006
  232. data/test/plugin_helper/test_server.rb +0 -1895
  233. data/test/plugin_helper/test_service_discovery.rb +0 -165
  234. data/test/plugin_helper/test_socket.rb +0 -146
  235. data/test/plugin_helper/test_storage.rb +0 -542
  236. data/test/plugin_helper/test_thread.rb +0 -164
  237. data/test/plugin_helper/test_timer.rb +0 -130
  238. data/test/scripts/exec_script.rb +0 -32
  239. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  240. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  241. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  242. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  243. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  244. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  245. data/test/test_capability.rb +0 -74
  246. data/test/test_clock.rb +0 -164
  247. data/test/test_config.rb +0 -369
  248. data/test/test_configdsl.rb +0 -148
  249. data/test/test_daemonizer.rb +0 -91
  250. data/test/test_engine.rb +0 -203
  251. data/test/test_event.rb +0 -531
  252. data/test/test_event_router.rb +0 -348
  253. data/test/test_event_time.rb +0 -199
  254. data/test/test_file_wrapper.rb +0 -53
  255. data/test/test_filter.rb +0 -121
  256. data/test/test_fluent_log_event_router.rb +0 -99
  257. data/test/test_formatter.rb +0 -369
  258. data/test/test_input.rb +0 -31
  259. data/test/test_log.rb +0 -1076
  260. data/test/test_match.rb +0 -148
  261. data/test/test_mixin.rb +0 -351
  262. data/test/test_msgpack_factory.rb +0 -50
  263. data/test/test_oj_options.rb +0 -55
  264. data/test/test_output.rb +0 -278
  265. data/test/test_plugin.rb +0 -251
  266. data/test/test_plugin_classes.rb +0 -370
  267. data/test/test_plugin_helper.rb +0 -81
  268. data/test/test_plugin_id.rb +0 -119
  269. data/test/test_process.rb +0 -14
  270. data/test/test_root_agent.rb +0 -951
  271. data/test/test_static_config_analysis.rb +0 -177
  272. data/test/test_supervisor.rb +0 -821
  273. data/test/test_test_drivers.rb +0 -136
  274. data/test/test_time_formatter.rb +0 -301
  275. data/test/test_time_parser.rb +0 -362
  276. data/test/test_tls.rb +0 -65
  277. data/test/test_unique_id.rb +0 -47
  278. data/test/test_variable_store.rb +0 -65
@@ -1,2024 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/plugin/output'
3
- require 'fluent/plugin/buffer'
4
- require 'fluent/output'
5
- require 'fluent/event'
6
-
7
- require 'json'
8
- require 'time'
9
- require 'timeout'
10
- require 'timecop'
11
-
12
- module FluentPluginOutputAsBufferedTest
13
- class DummyBareOutput < Fluent::Plugin::Output
14
- def register(name, &block)
15
- instance_variable_set("@#{name}", block)
16
- end
17
- end
18
- class DummySyncOutput < DummyBareOutput
19
- def initialize
20
- super
21
- @process = nil
22
- end
23
- def process(tag, es)
24
- @process ? @process.call(tag, es) : nil
25
- end
26
- end
27
- class DummyAsyncOutput < DummyBareOutput
28
- def initialize
29
- super
30
- @format = nil
31
- @write = nil
32
- end
33
- def format(tag, time, record)
34
- @format ? @format.call(tag, time, record) : [tag, time, record].to_json
35
- end
36
- def write(chunk)
37
- @write ? @write.call(chunk) : nil
38
- end
39
- end
40
- class DummyDelayedOutput < DummyBareOutput
41
- def initialize
42
- super
43
- @format = nil
44
- @try_write = nil
45
- @shutdown_hook = nil
46
- end
47
- def format(tag, time, record)
48
- @format ? @format.call(tag, time, record) : [tag, time, record].to_json
49
- end
50
- def try_write(chunk)
51
- @try_write ? @try_write.call(chunk) : nil
52
- end
53
- def shutdown
54
- if @shutdown_hook
55
- @shutdown_hook.call
56
- end
57
- super
58
- end
59
- end
60
- class DummyStandardBufferedOutput < DummyBareOutput
61
- def initialize
62
- super
63
- @prefer_delayed_commit = nil
64
- @write = nil
65
- @try_write = nil
66
- end
67
- def prefer_delayed_commit
68
- @prefer_delayed_commit ? @prefer_delayed_commit.call : false
69
- end
70
- def write(chunk)
71
- @write ? @write.call(chunk) : nil
72
- end
73
- def try_write(chunk)
74
- @try_write ? @try_write.call(chunk) : nil
75
- end
76
- end
77
- class DummyCustomFormatBufferedOutput < DummyBareOutput
78
- def initialize
79
- super
80
- @format_type_is_msgpack = nil
81
- @prefer_delayed_commit = nil
82
- @write = nil
83
- @try_write = nil
84
- end
85
- def format(tag, time, record)
86
- @format ? @format.call(tag, time, record) : [tag, time, record].to_json
87
- end
88
- def formatted_to_msgpack_binary?
89
- @format_type_is_msgpack ? @format_type_is_msgpack.call : false
90
- end
91
- def prefer_delayed_commit
92
- @prefer_delayed_commit ? @prefer_delayed_commit.call : false
93
- end
94
- def write(chunk)
95
- @write ? @write.call(chunk) : nil
96
- end
97
- def try_write(chunk)
98
- @try_write ? @try_write.call(chunk) : nil
99
- end
100
- end
101
- # check for formatted_to_msgpack_binary compatibility
102
- class DummyOldCustomFormatBufferedOutput < DummyBareOutput
103
- def initialize
104
- super
105
- @format_type_is_msgpack = nil
106
- @prefer_delayed_commit = nil
107
- @write = nil
108
- @try_write = nil
109
- end
110
- def format(tag, time, record)
111
- @format ? @format.call(tag, time, record) : [tag, time, record].to_json
112
- end
113
- def formatted_to_msgpack_binary
114
- @format_type_is_msgpack ? @format_type_is_msgpack.call : false
115
- end
116
- def prefer_delayed_commit
117
- @prefer_delayed_commit ? @prefer_delayed_commit.call : false
118
- end
119
- def write(chunk)
120
- @write ? @write.call(chunk) : nil
121
- end
122
- def try_write(chunk)
123
- @try_write ? @try_write.call(chunk) : nil
124
- end
125
- end
126
- class DummyFullFeatureOutput < DummyBareOutput
127
- def initialize
128
- super
129
- @prefer_buffered_processing = nil
130
- @prefer_delayed_commit = nil
131
- @process = nil
132
- @format = nil
133
- @write = nil
134
- @try_write = nil
135
- end
136
- def prefer_buffered_processing
137
- @prefer_buffered_processing ? @prefer_buffered_processing.call : false
138
- end
139
- def prefer_delayed_commit
140
- @prefer_delayed_commit ? @prefer_delayed_commit.call : false
141
- end
142
- def process(tag, es)
143
- @process ? @process.call(tag, es) : nil
144
- end
145
- def format(tag, time, record)
146
- @format ? @format.call(tag, time, record) : [tag, time, record].to_json
147
- end
148
- def write(chunk)
149
- @write ? @write.call(chunk) : nil
150
- end
151
- def try_write(chunk)
152
- @try_write ? @try_write.call(chunk) : nil
153
- end
154
- end
155
- module OldPluginMethodMixin
156
- def initialize
157
- super
158
- @format = nil
159
- @write = nil
160
- end
161
- def register(name, &block)
162
- instance_variable_set("@#{name}", block)
163
- end
164
- def format(tag, time, record)
165
- @format ? @format.call(tag, time, record) : [tag, time, record].to_json
166
- end
167
- def write(chunk)
168
- @write ? @write.call(chunk) : nil
169
- end
170
- end
171
- class DummyOldBufferedOutput < Fluent::BufferedOutput
172
- include OldPluginMethodMixin
173
- end
174
- class DummyOldObjectBufferedOutput < Fluent::ObjectBufferedOutput
175
- include OldPluginMethodMixin
176
- end
177
- end
178
-
179
- class BufferedOutputTest < Test::Unit::TestCase
180
- def create_output(type=:full)
181
- case type
182
- when :bare then FluentPluginOutputAsBufferedTest::DummyBareOutput.new
183
- when :sync then FluentPluginOutputAsBufferedTest::DummySyncOutput.new
184
- when :buffered then FluentPluginOutputAsBufferedTest::DummyAsyncOutput.new
185
- when :delayed then FluentPluginOutputAsBufferedTest::DummyDelayedOutput.new
186
- when :standard then FluentPluginOutputAsBufferedTest::DummyStandardBufferedOutput.new
187
- when :custom then FluentPluginOutputAsBufferedTest::DummyCustomFormatBufferedOutput.new
188
- when :full then FluentPluginOutputAsBufferedTest::DummyFullFeatureOutput.new
189
- when :old_buf then FluentPluginOutputAsBufferedTest::DummyOldBufferedOutput.new
190
- when :old_obj then FluentPluginOutputAsBufferedTest::DummyOldObjectBufferedOutput.new
191
- when :old_custom then FluentPluginOutputAsBufferedTest::DummyOldCustomFormatBufferedOutput.new
192
- else
193
- raise ArgumentError, "unknown type: #{type}"
194
- end
195
- end
196
- def create_metadata(timekey: nil, tag: nil, variables: nil)
197
- Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
198
- end
199
- def waiting(seconds)
200
- begin
201
- Timeout.timeout(seconds) do
202
- yield
203
- end
204
- rescue Timeout::Error
205
- STDERR.print(*@i.log.out.logs)
206
- raise
207
- end
208
- end
209
-
210
- setup do
211
- @i = nil
212
- end
213
-
214
- teardown do
215
- if @i
216
- @i.stop unless @i.stopped?
217
- @i.before_shutdown unless @i.before_shutdown?
218
- @i.shutdown unless @i.shutdown?
219
- @i.after_shutdown unless @i.after_shutdown?
220
- @i.close unless @i.closed?
221
- @i.terminate unless @i.terminated?
222
- end
223
- Timecop.return
224
- end
225
-
226
- test 'queued_chunks_limit_size is same as flush_thread_count by default' do
227
- hash = {'flush_thread_count' => 4}
228
- i = create_output
229
- i.register(:prefer_buffered_processing) { true }
230
- i.configure(config_element('ROOT', '', {}, [config_element('buffer','tag',hash)]))
231
-
232
- assert_equal 4, i.buffer.queued_chunks_limit_size
233
- end
234
-
235
- test 'prefer queued_chunks_limit_size parameter than flush_thread_count' do
236
- hash = {'flush_thread_count' => 4, 'queued_chunks_limit_size' => 2}
237
- i = create_output
238
- i.register(:prefer_buffered_processing) { true }
239
- i.configure(config_element('ROOT', '', {}, [config_element('buffer','tag',hash)]))
240
-
241
- assert_equal 2, i.buffer.queued_chunks_limit_size
242
- end
243
-
244
- sub_test_case 'chunk feature in #write for output plugins' do
245
- setup do
246
- @stored_global_logger = $log
247
- $log = Fluent::Test::TestLogger.new
248
- @hash = {
249
- 'flush_mode' => 'immediate',
250
- 'flush_thread_interval' => '0.01',
251
- 'flush_thread_burst_interval' => '0.01',
252
- }
253
- end
254
-
255
- teardown do
256
- $log = @stored_global_logger
257
- end
258
-
259
- test 'plugin using standard format can iterate chunk for time, record in #write' do
260
- events_from_chunk = []
261
- @i = create_output(:standard)
262
- @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
263
- @i.register(:prefer_delayed_commit){ false }
264
- @i.register(:write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|t,r| e << [t,r]}; events_from_chunk << [:write, e] }
265
- @i.register(:try_write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|t,r| e << [t,r]}; events_from_chunk << [:try_write, e] }
266
- @i.start
267
- @i.after_start
268
-
269
- events = [
270
- [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
271
- [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
272
- ]
273
-
274
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
275
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
276
-
277
- waiting(5){ sleep 0.1 until events_from_chunk.size == 2 }
278
-
279
- assert_equal 2, events_from_chunk.size
280
- 2.times.each do |i|
281
- assert_equal :write, events_from_chunk[i][0]
282
- assert_equal events, events_from_chunk[i][1]
283
- end
284
- end
285
-
286
- test 'plugin using standard format can iterate chunk for time, record in #try_write' do
287
- events_from_chunk = []
288
- @i = create_output(:standard)
289
- @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
290
- @i.register(:prefer_delayed_commit){ true }
291
- @i.register(:write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|t,r| e << [t,r]}; events_from_chunk << [:write, e] }
292
- @i.register(:try_write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|t,r| e << [t,r]}; events_from_chunk << [:try_write, e] }
293
- @i.start
294
- @i.after_start
295
-
296
- events = [
297
- [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
298
- [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
299
- ]
300
-
301
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
302
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
303
-
304
- waiting(5){ sleep 0.1 until events_from_chunk.size == 2 }
305
-
306
- assert_equal 2, events_from_chunk.size
307
- 2.times.each do |i|
308
- assert_equal :try_write, events_from_chunk[i][0]
309
- assert_equal events, events_from_chunk[i][1]
310
- end
311
- end
312
-
313
- test 'plugin using custom format cannot iterate chunk in #write' do
314
- events_from_chunk = []
315
- @i = create_output(:custom)
316
- @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
317
- @i.register(:prefer_delayed_commit){ false }
318
- @i.register(:format){ |tag, time, record| [tag,time,record].to_json }
319
- @i.register(:format_type_is_msgpack){ false }
320
- @i.register(:write){ |chunk| assert !(chunk.respond_to?(:each)) }
321
- @i.register(:try_write){ |chunk| assert !(chunk.respond_to?(:each)) }
322
- @i.start
323
- @i.after_start
324
-
325
- events = [
326
- [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
327
- [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
328
- ]
329
-
330
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
331
-
332
- assert_equal 0, events_from_chunk.size
333
- end
334
-
335
- test 'plugin using custom format cannot iterate chunk in #try_write' do
336
- events_from_chunk = []
337
- @i = create_output(:custom)
338
- @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
339
- @i.register(:prefer_delayed_commit){ true }
340
- @i.register(:format){ |tag, time, record| [tag,time,record].to_json }
341
- @i.register(:format_type_is_msgpack){ false }
342
- @i.register(:write){ |chunk| assert !(chunk.respond_to?(:each)) }
343
- @i.register(:try_write){ |chunk| assert !(chunk.respond_to?(:each)) }
344
- @i.start
345
- @i.after_start
346
-
347
- events = [
348
- [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
349
- [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
350
- ]
351
-
352
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
353
-
354
- assert_equal 0, events_from_chunk.size
355
- end
356
-
357
- data('formatted_to_msgpack_binary?' => :custom,
358
- 'formatted_to_msgpack_binary' => :old_custom)
359
- test 'plugin using custom format can iterate chunk in #write if #format returns msgpack' do |out_type|
360
- events_from_chunk = []
361
- @i = create_output(out_type)
362
- @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
363
- @i.register(:prefer_delayed_commit){ false }
364
- @i.register(:format){ |tag, time, record| [tag,time,record].to_msgpack }
365
- @i.register(:format_type_is_msgpack){ true }
366
- @i.register(:write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|ta,t,r| e << [ta,t,r]}; events_from_chunk << [:write, e] }
367
- @i.register(:try_write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|ta,t,r| e << [ta,t,r]}; events_from_chunk << [:try_write, e] }
368
- @i.start
369
- @i.after_start
370
-
371
- events = [
372
- [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
373
- [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
374
- ]
375
-
376
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
377
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
378
-
379
- waiting(5){ sleep 0.1 until events_from_chunk.size == 2 }
380
-
381
- assert_equal 2, events_from_chunk.size
382
- 2.times.each do |i|
383
- assert_equal :write, events_from_chunk[i][0]
384
- each_pushed = events_from_chunk[i][1]
385
- assert_equal 2, each_pushed.size
386
- assert_equal 'test.tag', each_pushed[0][0]
387
- assert_equal 'test.tag', each_pushed[1][0]
388
- assert_equal events, each_pushed.map{|tag,time,record| [time,record]}
389
- end
390
- end
391
-
392
- data(:handle_stream_simple => '',
393
- :handle_stream_with_custom_format => 'tag,message')
394
- test 'plugin using custom format can skip record chunk when format return nil' do |chunk_keys|
395
- events_from_chunk = []
396
- @i = create_output(:custom)
397
- @i.configure(config_element('ROOT', '', {}, [config_element('buffer', chunk_keys, @hash)]))
398
- @i.register(:prefer_delayed_commit) { false }
399
- @i.register(:format) { |tag, time, record|
400
- if record['message'] == 'test1'
401
- nil
402
- else
403
- [tag,time,record].to_msgpack
404
- end
405
- }
406
- @i.register(:format_type_is_msgpack) { true }
407
- @i.register(:write){ |chunk| e = []; chunk.each { |ta, t, r| e << [ta, t, r] }; events_from_chunk << [:write, e] }
408
- @i.start
409
- @i.after_start
410
-
411
- events = [
412
- [event_time('2016-10-05 16:16:16 -0700'), {"message" => "test1"}],
413
- [event_time('2016-10-05 16:16:17 -0700'), {"message" => "test2"}],
414
- ]
415
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
416
-
417
- waiting(5) { sleep 0.1 until events_from_chunk.size == 1 }
418
-
419
- assert_equal 1, events_from_chunk.size
420
- assert_equal :write, events_from_chunk[0][0]
421
- each_pushed = events_from_chunk[0][1]
422
- assert_equal 1, each_pushed.size
423
- assert_equal 'test.tag', each_pushed[0][0]
424
- assert_equal "test2", each_pushed[0][2]['message']
425
- end
426
-
427
- test 'plugin using custom format can iterate chunk in #try_write if #format returns msgpack' do
428
- events_from_chunk = []
429
- @i = create_output(:custom)
430
- @i.configure(config_element('ROOT','',{},[config_element('buffer','',@hash)]))
431
- @i.register(:prefer_delayed_commit){ true }
432
- @i.register(:format){ |tag, time, record| [tag,time,record].to_msgpack }
433
- @i.register(:format_type_is_msgpack){ true }
434
- @i.register(:write){ |chunk| events_from_chunk = []; assert chunk.respond_to?(:each); chunk.each{|ta,t,r| e << [ta,t,r]}; events_from_chunk << [:write, e] }
435
- @i.register(:try_write){ |chunk| e = []; assert chunk.respond_to?(:each); chunk.each{|ta,t,r| e << [ta,t,r]}; events_from_chunk << [:try_write, e] }
436
- @i.start
437
- @i.after_start
438
-
439
- events = [
440
- [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
441
- [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
442
- ]
443
-
444
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
445
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
446
-
447
- waiting(5){ sleep 0.1 until events_from_chunk.size == 2 }
448
-
449
- assert_equal 2, events_from_chunk.size
450
- 2.times.each do |i|
451
- assert_equal :try_write, events_from_chunk[i][0]
452
- each_pushed = events_from_chunk[i][1]
453
- assert_equal 2, each_pushed.size
454
- assert_equal 'test.tag', each_pushed[0][0]
455
- assert_equal 'test.tag', each_pushed[1][0]
456
- assert_equal events, each_pushed.map{|tag,time,record| [time,record]}
457
- end
458
- end
459
-
460
- data(:BufferedOutput => :old_buf,
461
- :ObjectBufferedOutput => :old_obj)
462
- test 'old plugin types can iterate chunk by msgpack_each in #write' do |plugin_type|
463
- events_from_chunk = []
464
- # event_emitter helper requires Engine.root_agent for routing
465
- ra = Fluent::RootAgent.new(log: $log)
466
- stub(Fluent::Engine).root_agent { ra }
467
- @i = create_output(plugin_type)
468
- @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', @hash)]))
469
- @i.register(:format) { |tag, time, record| [time, record].to_msgpack }
470
- @i.register(:write) { |chunk| e = []; chunk.msgpack_each { |t, r| e << [t, r] }; events_from_chunk << [:write, e]; }
471
- @i.start
472
- @i.after_start
473
-
474
- events = [
475
- [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
476
- [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
477
- ]
478
-
479
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
480
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
481
-
482
- waiting(5) { sleep 0.1 until events_from_chunk.size == 2 }
483
-
484
- assert_equal 2, events_from_chunk.size
485
- 2.times.each do |i|
486
- assert_equal :write, events_from_chunk[i][0]
487
- assert_equal events, events_from_chunk[i][1]
488
- end
489
- end
490
- end
491
-
492
- sub_test_case 'buffered output configured with many chunk keys' do
493
- setup do
494
- @stored_global_logger = $log
495
- $log = Fluent::Test::TestLogger.new
496
- @hash = {
497
- 'flush_mode' => 'interval',
498
- 'flush_thread_burst_interval' => 0.01,
499
- 'chunk_limit_size' => 1024,
500
- 'timekey' => 60,
501
- }
502
- @i = create_output(:buffered)
503
- end
504
- teardown do
505
- $log = @stored_global_logger
506
- end
507
- test 'nothing are warned with less chunk keys' do
508
- chunk_keys = 'time,key1,key2,key3'
509
- @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))
510
- logs = @i.log.out.logs.dup
511
- @i.start
512
- @i.after_start
513
- assert{ logs.count{|log| log.include?('[warn]') } == 0 }
514
- end
515
-
516
- test 'a warning reported with 4 chunk keys' do
517
- chunk_keys = 'key1,key2,key3,key4'
518
- @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))
519
- logs = @i.log.out.logs.dup
520
-
521
- @i.start # this calls `log.reset`... capturing logs about configure must be done before this line
522
- @i.after_start
523
- assert_equal ['key1', 'key2', 'key3', 'key4'], @i.chunk_keys
524
-
525
- assert{ logs.count{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') } == 1 }
526
- end
527
-
528
- test 'a warning reported with 4 chunk keys including "tag"' do
529
- chunk_keys = 'tag,key1,key2,key3'
530
- @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))
531
- logs = @i.log.out.logs.dup
532
- @i.start # this calls `log.reset`... capturing logs about configure must be done before this line
533
- @i.after_start
534
- assert{ logs.count{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') } == 1 }
535
- end
536
-
537
- test 'time key is not included for warned chunk keys' do
538
- chunk_keys = 'time,key1,key2,key3'
539
- @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_keys,@hash)]))
540
- logs = @i.log.out.logs.dup
541
- @i.start
542
- @i.after_start
543
- assert{ logs.count{|log| log.include?('[warn]') } == 0 }
544
- end
545
- end
546
-
547
- sub_test_case 'buffered output feature without any buffer key, flush_mode: lazy' do
548
- setup do
549
- hash = {
550
- 'flush_mode' => 'lazy',
551
- 'flush_thread_burst_interval' => 0.01,
552
- 'flush_thread_count' => 2,
553
- 'chunk_limit_size' => 1024,
554
- }
555
- @i = create_output(:buffered)
556
- @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)]))
557
- @i.start
558
- @i.after_start
559
- end
560
-
561
- test '#start does not create enqueue thread, but creates flush threads' do
562
- @i.thread_wait_until_start
563
-
564
- assert @i.thread_exist?(:flush_thread_0)
565
- assert @i.thread_exist?(:flush_thread_1)
566
- assert !@i.thread_exist?(:enqueue_thread)
567
- end
568
-
569
- test '#format is called for each events' do
570
- ary = []
571
- @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }
572
-
573
- t = event_time()
574
- es = Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ])
575
-
576
- 4.times do
577
- @i.emit_events('tag.test', es)
578
- end
579
-
580
- assert_equal 8, ary.size
581
- 4.times do |i|
582
- assert_equal ["tag.test", t, {"key" => "value1"}], ary[i*2]
583
- assert_equal ["tag.test", t, {"key" => "value2"}], ary[i*2+1]
584
- end
585
- end
586
-
587
- test '#write is called only when chunk bytes limit exceeded, and buffer chunk is purged' do
588
- ary = []
589
- @i.register(:write){|chunk| ary << chunk.read }
590
-
591
- tag = "test.tag"
592
- t = event_time()
593
- r = {}
594
- (0...10).each do |i|
595
- r["key#{i}"] = "value #{i}"
596
- end
597
- event_size = [tag, t, r].to_json.size # 195
598
-
599
- (1024 * 0.9 / event_size).to_i.times do |i|
600
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new([ [t, r] ]))
601
- end
602
- assert{ @i.buffer.queue.size == 0 && ary.size == 0 }
603
-
604
- staged_chunk = @i.buffer.stage[@i.buffer.stage.keys.first]
605
- assert{ staged_chunk.size != 0 }
606
-
607
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new([ [t, r] ]))
608
-
609
- assert{ @i.buffer.queue.size > 0 || @i.buffer.dequeued.size > 0 || ary.size > 0 }
610
-
611
- waiting(10) do
612
- Thread.pass until @i.buffer.queue.size == 0 && @i.buffer.dequeued.size == 0
613
- Thread.pass until staged_chunk.size == 0
614
- end
615
-
616
- assert_equal 1, ary.size
617
- assert_equal [tag,t,r].to_json * (1024 / event_size), ary.first
618
- end
619
-
620
- test 'flush_at_shutdown work well when plugin is shutdown' do
621
- ary = []
622
- @i.register(:write){|chunk| ary << chunk.read }
623
-
624
- tag = "test.tag"
625
- t = event_time()
626
- r = {}
627
- (0...10).each do |i|
628
- r["key#{i}"] = "value #{i}"
629
- end
630
- event_size = [tag, t, r].to_json.size # 195
631
-
632
- (1024 * 0.9 / event_size).to_i.times do |i|
633
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new([ [t, r] ]))
634
- end
635
- assert{ @i.buffer.queue.size == 0 && ary.size == 0 }
636
-
637
- @i.stop
638
- @i.before_shutdown
639
- @i.shutdown
640
- @i.after_shutdown
641
-
642
- waiting(10) do
643
- Thread.pass until ary.size == 1
644
- end
645
- assert_equal [tag,t,r].to_json * (1024 * 0.9 / event_size), ary.first
646
- end
647
- end
648
-
649
- sub_test_case 'buffered output feature without any buffer key, flush_mode: interval' do
650
- setup do
651
- hash = {
652
- 'flush_mode' => 'interval',
653
- 'flush_interval' => 1,
654
- 'flush_thread_count' => 1,
655
- 'flush_thread_burst_interval' => 0.01,
656
- 'chunk_limit_size' => 1024,
657
- }
658
- @i = create_output(:buffered)
659
- @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)]))
660
- @i.start
661
- @i.after_start
662
- end
663
-
664
- test '#start creates enqueue thread and flush threads' do
665
- @i.thread_wait_until_start
666
-
667
- assert @i.thread_exist?(:flush_thread_0)
668
- assert @i.thread_exist?(:enqueue_thread)
669
- end
670
-
671
- test '#format is called for each event streams' do
672
- ary = []
673
- @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }
674
-
675
- t = event_time()
676
- es = Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ])
677
-
678
- 4.times do
679
- @i.emit_events('tag.test', es)
680
- end
681
-
682
- assert_equal 8, ary.size
683
- 4.times do |i|
684
- assert_equal ["tag.test", t, {"key" => "value1"}], ary[i*2]
685
- assert_equal ["tag.test", t, {"key" => "value2"}], ary[i*2+1]
686
- end
687
- end
688
-
689
- test '#write is called per flush_interval, and buffer chunk is purged' do
690
- @i.thread_wait_until_start
691
-
692
- ary = []
693
- @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" }
694
- @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| ary << data } }
695
-
696
- t = event_time()
697
- r = {}
698
- (0...10).each do |i|
699
- r["key#{i}"] = "value #{i}"
700
- end
701
-
702
- 2.times do |i|
703
- rand_records = rand(1..4)
704
- es = Fluent::ArrayEventStream.new([ [t, r] ] * rand_records)
705
- assert_equal rand_records, es.size
706
-
707
- @i.interrupt_flushes
708
-
709
- assert{ @i.buffer.queue.size == 0 }
710
-
711
- @i.emit_events("test.tag", es)
712
-
713
- assert{ @i.buffer.queue.size == 0 }
714
- assert{ @i.buffer.stage.size == 1 }
715
-
716
- staged_chunk = @i.instance_eval{ @buffer.stage[@buffer.stage.keys.first] }
717
- assert{ staged_chunk.size != 0 }
718
-
719
- @i.enqueue_thread_wait
720
-
721
- waiting(10) do
722
- Thread.pass until @i.buffer.queue.size == 0 && @i.buffer.dequeued.size == 0
723
- Thread.pass until staged_chunk.size == 0
724
- end
725
-
726
- assert_equal rand_records, ary.size
727
- ary.reject!{|e| true }
728
- end
729
- end
730
- end
731
-
732
- sub_test_case 'with much longer flush_interval' do
733
- setup do
734
- hash = {
735
- 'flush_mode' => 'interval',
736
- 'flush_interval' => 3000,
737
- 'flush_thread_count' => 1,
738
- 'flush_thread_burst_interval' => 0.01,
739
- 'chunk_limit_size' => 1024,
740
- }
741
- @i = create_output(:buffered)
742
- @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)]))
743
- @i.start
744
- @i.after_start
745
- end
746
-
747
- test 'flush_at_shutdown work well when plugin is shutdown' do
748
- ary = []
749
- @i.register(:write){|chunk| ary << chunk.read }
750
-
751
- tag = "test.tag"
752
- t = event_time()
753
- r = {}
754
- (0...10).each do |i|
755
- r["key#{i}"] = "value #{i}"
756
- end
757
- event_size = [tag, t, r].to_json.size # 195
758
-
759
- (1024 * 0.9 / event_size).to_i.times do |i|
760
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new([ [t, r] ]))
761
- end
762
- queue_size = @i.buffer.queue.size
763
- assert{ queue_size == 0 && ary.size == 0 }
764
-
765
- @i.stop
766
- @i.before_shutdown
767
- @i.shutdown
768
- @i.after_shutdown
769
-
770
- waiting(10){ sleep 0.1 until ary.size == 1 }
771
- assert_equal [tag,t,r].to_json * (1024 * 0.9 / event_size), ary.first
772
- end
773
- end
774
-
775
- sub_test_case 'buffered output feature without any buffer key, flush_mode: immediate' do
776
- setup do
777
- hash = {
778
- 'flush_mode' => 'immediate',
779
- 'flush_thread_count' => 1,
780
- 'flush_thread_burst_interval' => 0.01,
781
- 'chunk_limit_size' => 1024,
782
- }
783
- @i = create_output(:buffered)
784
- @i.configure(config_element('ROOT','',{},[config_element('buffer','',hash)]))
785
- @i.start
786
- @i.after_start
787
- end
788
-
789
- test '#start does not create enqueue thread, but creates flush threads' do
790
- @i.thread_wait_until_start
791
-
792
- assert @i.thread_exist?(:flush_thread_0)
793
- assert !@i.thread_exist?(:enqueue_thread)
794
- end
795
-
796
- test '#format is called for each event streams' do
797
- ary = []
798
- @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }
799
-
800
- t = event_time()
801
- es = Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ])
802
-
803
- 4.times do
804
- @i.emit_events('tag.test', es)
805
- end
806
-
807
- assert_equal 8, ary.size
808
- 4.times do |i|
809
- assert_equal ["tag.test", t, {"key" => "value1"}], ary[i*2]
810
- assert_equal ["tag.test", t, {"key" => "value2"}], ary[i*2+1]
811
- end
812
- end
813
-
814
- test '#write is called every time for each emits, and buffer chunk is purged' do
815
- @i.thread_wait_until_start
816
-
817
- ary = []
818
- @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" }
819
- @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| ary << data } }
820
-
821
- t = event_time()
822
- r = {}
823
- (0...10).each do |i|
824
- r["key#{i}"] = "value #{i}"
825
- end
826
-
827
- 3.times do |i|
828
- rand_records = rand(1..5)
829
- es = Fluent::ArrayEventStream.new([ [t, r] ] * rand_records)
830
- assert_equal rand_records, es.size
831
- @i.emit_events("test.tag", es)
832
-
833
- waiting(10){ sleep 0.1 until @i.buffer.stage.size == 0 } # make sure that the emitted es is enqueued by "flush_mode immediate"
834
- waiting(10){ sleep 0.1 until @i.buffer.queue.size == 0 && @i.buffer.dequeued.size == 0 }
835
- waiting(10){ sleep 0.1 until ary.size == rand_records }
836
-
837
- assert_equal rand_records, ary.size
838
- ary.reject!{|e| true }
839
- end
840
- end
841
-
842
- test 'flush_at_shutdown work well when plugin is shutdown' do
843
- ary = []
844
- @i.register(:write){|chunk| ary << chunk.read }
845
-
846
- tag = "test.tag"
847
- t = event_time()
848
- r = {}
849
- (0...10).each do |i|
850
- r["key#{i}"] = "value #{i}"
851
- end
852
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new([ [t, r] ]))
853
-
854
- @i.stop
855
- @i.before_shutdown
856
- @i.shutdown
857
- @i.after_shutdown
858
-
859
- waiting(10) do
860
- Thread.pass until ary.size == 1
861
- end
862
- assert_equal [tag,t,r].to_json, ary.first
863
- end
864
- end
865
-
866
- sub_test_case 'buffered output feature with timekey and range' do
867
- setup do
868
- chunk_key = 'time'
869
- hash = {
870
- 'timekey' => 30, # per 30seconds
871
- 'timekey_wait' => 5, # 5 second delay for flush
872
- 'flush_thread_count' => 1,
873
- 'flush_thread_burst_interval' => 0.01,
874
- }
875
- @i = create_output(:buffered)
876
- @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
877
- @i.start
878
- @i.after_start
879
- end
880
-
881
- test '#configure raises config error if timekey is not specified' do
882
- i = create_output(:buffered)
883
- assert_raise Fluent::ConfigError do
884
- i.configure(config_element('ROOT','',{},[config_element('buffer','time',)]))
885
- end
886
- end
887
-
888
- test 'default flush_mode is set to :lazy' do
889
- assert_equal :lazy, @i.instance_eval{ @flush_mode }
890
- end
891
-
892
- test '#start creates enqueue thread and flush threads' do
893
- @i.thread_wait_until_start
894
-
895
- assert @i.thread_exist?(:flush_thread_0)
896
- assert @i.thread_exist?(:enqueue_thread)
897
- end
898
-
899
- test '#format is called for each event streams' do
900
- ary = []
901
- @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }
902
-
903
- t = event_time()
904
- es = Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ])
905
-
906
- 5.times do
907
- @i.emit_events('tag.test', es)
908
- end
909
-
910
- assert_equal 10, ary.size
911
- 5.times do |i|
912
- assert_equal ["tag.test", t, {"key" => "value1"}], ary[i*2]
913
- assert_equal ["tag.test", t, {"key" => "value2"}], ary[i*2+1]
914
- end
915
- end
916
-
917
- test '#write is called per time ranges after timekey_wait, and buffer chunk is purged' do
918
- Timecop.freeze( Time.parse('2016-04-13 14:04:00 +0900') )
919
-
920
- @i.thread_wait_until_start
921
-
922
- ary = []
923
- metachecks = []
924
-
925
- @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" }
926
- @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (chunk.metadata.timekey.to_i <= e[1].to_i && e[1].to_i < chunk.metadata.timekey.to_i + 30) } }
927
-
928
- r = {}
929
- (0...10).each do |i|
930
- r["key#{i}"] = "value #{i}"
931
- end
932
- ts = [
933
- Fluent::EventTime.parse('2016-04-13 14:03:21 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:23 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:29 +0900'),
934
- Fluent::EventTime.parse('2016-04-13 14:03:30 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:33 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:38 +0900'),
935
- Fluent::EventTime.parse('2016-04-13 14:03:43 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:49 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:51 +0900'),
936
- Fluent::EventTime.parse('2016-04-13 14:04:00 +0900'), Fluent::EventTime.parse('2016-04-13 14:04:01 +0900'),
937
- ]
938
- events = [
939
- ["test.tag.1", ts[0], r], # range 14:03:00 - 03:29
940
- ["test.tag.2", ts[1], r],
941
- ["test.tag.1", ts[2], r],
942
- ["test.tag.1", ts[3], r], # range 14:03:30 - 04:00
943
- ["test.tag.1", ts[4], r],
944
- ["test.tag.1", ts[5], r],
945
- ["test.tag.1", ts[6], r],
946
- ["test.tag.1", ts[7], r],
947
- ["test.tag.2", ts[8], r],
948
- ["test.tag.1", ts[9], r], # range 14:04:00 - 04:29
949
- ["test.tag.2", ts[10], r],
950
- ]
951
-
952
- assert_equal 0, @i.write_count
953
-
954
- @i.interrupt_flushes
955
-
956
- events.shuffle.each do |tag, time, record|
957
- @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))
958
- end
959
- assert{ @i.buffer.stage.size == 3 }
960
- assert{ @i.write_count == 0 }
961
-
962
- @i.enqueue_thread_wait
963
-
964
- waiting(4){ sleep 0.1 until @i.write_count > 0 }
965
-
966
- assert{ @i.buffer.stage.size == 2 && @i.write_count == 1 }
967
-
968
- waiting(4){ sleep 0.1 until ary.size == 3 }
969
-
970
- assert_equal 3, ary.size
971
- assert_equal 2, ary.count{|e| e[0] == "test.tag.1" }
972
- assert_equal 1, ary.count{|e| e[0] == "test.tag.2" }
973
-
974
- Timecop.freeze( Time.parse('2016-04-13 14:04:04 +0900') )
975
-
976
- @i.enqueue_thread_wait
977
-
978
- assert{ @i.buffer.stage.size == 2 && @i.write_count == 1 }
979
-
980
- Timecop.freeze( Time.parse('2016-04-13 14:04:06 +0900') )
981
-
982
- @i.enqueue_thread_wait
983
- waiting(4){ sleep 0.1 until @i.write_count > 1 }
984
-
985
- assert{ @i.buffer.stage.size == 1 && @i.write_count == 2 }
986
-
987
- assert_equal 9, ary.size
988
- assert_equal 7, ary.count{|e| e[0] == "test.tag.1" }
989
- assert_equal 2, ary.count{|e| e[0] == "test.tag.2" }
990
-
991
- assert metachecks.all?{|e| e }
992
- end
993
-
994
- test 'flush_at_shutdown work well when plugin is shutdown' do
995
- Timecop.freeze( Time.parse('2016-04-13 14:04:00 +0900') )
996
-
997
- @i.thread_wait_until_start
998
-
999
- ary = []
1000
- metachecks = []
1001
-
1002
- @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" }
1003
- @i.register(:write){|chunk|
1004
- chunk.read.split("\n").reject{|l| l.empty? }.each{|data|
1005
- e = JSON.parse(data)
1006
- ary << e
1007
- metachecks << (chunk.metadata.timekey.to_i <= e[1].to_i && e[1].to_i < chunk.metadata.timekey.to_i + 30)
1008
- }
1009
- }
1010
-
1011
- r = {}
1012
- (0...10).each do |i|
1013
- r["key#{i}"] = "value #{i}"
1014
- end
1015
- ts = [
1016
- Fluent::EventTime.parse('2016-04-13 14:03:21 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:23 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:29 +0900'),
1017
- Fluent::EventTime.parse('2016-04-13 14:03:30 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:33 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:38 +0900'),
1018
- Fluent::EventTime.parse('2016-04-13 14:03:43 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:49 +0900'), Fluent::EventTime.parse('2016-04-13 14:03:51 +0900'),
1019
- Fluent::EventTime.parse('2016-04-13 14:04:00 +0900'), Fluent::EventTime.parse('2016-04-13 14:04:01 +0900'),
1020
- ]
1021
- events = [
1022
- ["test.tag.1", ts[0], r], # range 14:03:00 - 03:29
1023
- ["test.tag.2", ts[1], r],
1024
- ["test.tag.1", ts[2], r],
1025
- ["test.tag.1", ts[3], r], # range 14:03:30 - 04:00
1026
- ["test.tag.1", ts[4], r],
1027
- ["test.tag.1", ts[5], r],
1028
- ["test.tag.1", ts[6], r],
1029
- ["test.tag.1", ts[7], r],
1030
- ["test.tag.2", ts[8], r],
1031
- ["test.tag.1", ts[9], r], # range 14:04:00 - 04:29
1032
- ["test.tag.2", ts[10], r],
1033
- ]
1034
-
1035
- assert_equal 0, @i.write_count
1036
-
1037
- @i.interrupt_flushes
1038
-
1039
- events.shuffle.each do |tag, time, record|
1040
- @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))
1041
- end
1042
- assert{ @i.buffer.stage.size == 3 }
1043
- assert{ @i.write_count == 0 }
1044
-
1045
- @i.enqueue_thread_wait
1046
-
1047
- waiting(4){ sleep 0.1 until @i.write_count > 0 }
1048
-
1049
- assert{ @i.buffer.stage.size == 2 && @i.write_count == 1 }
1050
-
1051
- Timecop.freeze( Time.parse('2016-04-13 14:04:04 +0900') )
1052
-
1053
- @i.enqueue_thread_wait
1054
-
1055
- assert{ @i.buffer.stage.size == 2 && @i.write_count == 1 }
1056
-
1057
- Timecop.freeze( Time.parse('2016-04-13 14:04:06 +0900') )
1058
-
1059
- @i.enqueue_thread_wait
1060
- waiting(4){ sleep 0.1 until @i.write_count > 1 }
1061
-
1062
- assert{ @i.buffer.stage.size == 1 && @i.write_count == 2 }
1063
-
1064
- Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') )
1065
-
1066
- waiting(4){ sleep 0.1 until ary.size == 9 }
1067
- assert_equal 9, ary.size
1068
-
1069
- @i.stop
1070
- @i.before_shutdown
1071
- @i.shutdown
1072
- @i.after_shutdown
1073
-
1074
- waiting(4){ sleep 0.1 until @i.write_count > 2 && ary.size == 11 }
1075
-
1076
- assert_equal 11, ary.size
1077
- assert metachecks.all?{|e| e }
1078
- end
1079
- end
1080
-
1081
- sub_test_case 'buffered output with large timekey and small timekey_wait' do
1082
- test 'writes event in proper interval' do
1083
- chunk_key = 'time'
1084
- hash = {
1085
- 'timekey_zone' => '+0900',
1086
- 'timekey' => 86400, # per 1 day
1087
- 'timekey_wait' => 10, # 10 seconds delay for flush
1088
- 'flush_thread_count' => 1,
1089
- 'flush_thread_burst_interval' => 0.01,
1090
- }
1091
-
1092
- with_timezone("UTC-9") do
1093
- Timecop.freeze(Time.parse('2019-02-08 00:01:00 +0900'))
1094
- @i = create_output(:buffered)
1095
- # timezone is set
1096
- @i.configure(config_element('ROOT', '', {}, [config_element('buffer',chunk_key,hash)]))
1097
- @i.start
1098
- @i.after_start
1099
- @i.thread_wait_until_start
1100
- assert_equal(0, @i.write_count)
1101
- @i.interrupt_flushes
1102
-
1103
- events = [
1104
- [event_time('2019-02-08 00:02:00 +0900'), { "message" => "foobar" }]
1105
- ]
1106
- @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
1107
- @i.enqueue_thread_wait
1108
- assert_equal(0, @i.write_count)
1109
-
1110
- Timecop.freeze(Time.parse('2019-02-09 00:00:08 +0900'))
1111
- @i.enqueue_thread_wait
1112
- assert_equal(0, @i.write_count)
1113
-
1114
- Timecop.freeze(Time.parse('2019-02-09 00:00:12 +0900'))
1115
- # write should be called in few seconds since
1116
- # running interval of enque thread is timekey_wait / 11.0.
1117
- waiting(5){ sleep 0.1 until @i.write_count == 1 }
1118
- end
1119
- end
1120
- end
1121
-
1122
- sub_test_case 'buffered output feature with tag key' do
1123
- setup do
1124
- chunk_key = 'tag'
1125
- hash = {
1126
- 'flush_interval' => 10,
1127
- 'flush_thread_count' => 1,
1128
- 'flush_thread_burst_interval' => 0.1,
1129
- 'chunk_limit_size' => 1024,
1130
- 'queued_chunks_limit_size' => 100
1131
- }
1132
- @i = create_output(:buffered)
1133
- @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
1134
- @i.start
1135
- @i.after_start
1136
- end
1137
-
1138
- test 'default flush_mode is set to :interval' do
1139
- assert_equal :interval, @i.instance_eval{ @flush_mode }
1140
- end
1141
-
1142
- test '#start creates enqueue thread and flush threads' do
1143
- @i.thread_wait_until_start
1144
-
1145
- assert @i.thread_exist?(:flush_thread_0)
1146
- assert @i.thread_exist?(:enqueue_thread)
1147
- end
1148
-
1149
- test '#format is called for each event streams' do
1150
- ary = []
1151
- @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }
1152
-
1153
- t = event_time()
1154
- es = Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ])
1155
-
1156
- 5.times do
1157
- @i.emit_events('tag.test', es)
1158
- end
1159
-
1160
- assert_equal 10, ary.size
1161
- 5.times do |i|
1162
- assert_equal ["tag.test", t, {"key" => "value1"}], ary[i*2]
1163
- assert_equal ["tag.test", t, {"key" => "value2"}], ary[i*2+1]
1164
- end
1165
- end
1166
-
1167
- test '#write is called per tags, per flush_interval & chunk sizes, and buffer chunk is purged' do
1168
- Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )
1169
-
1170
- ary = []
1171
- metachecks = []
1172
-
1173
- @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" }
1174
- @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (chunk.metadata.tag == e[0]) } }
1175
-
1176
- @i.thread_wait_until_start
1177
-
1178
- r = {}
1179
- (0...10).each do |i|
1180
- r["key#{i}"] = "value #{i}"
1181
- end
1182
- ts = [
1183
- event_time('2016-04-13 14:03:21 +0900'), event_time('2016-04-13 14:03:23 +0900'), event_time('2016-04-13 14:03:29 +0900'),
1184
- event_time('2016-04-13 14:03:30 +0900'), event_time('2016-04-13 14:03:33 +0900'), event_time('2016-04-13 14:03:38 +0900'),
1185
- event_time('2016-04-13 14:03:43 +0900'), event_time('2016-04-13 14:03:49 +0900'), event_time('2016-04-13 14:03:51 +0900'),
1186
- event_time('2016-04-13 14:04:00 +0900'), event_time('2016-04-13 14:04:01 +0900'),
1187
- ]
1188
- # size of a event is 197
1189
- events = [
1190
- ["test.tag.1", ts[0], r],
1191
- ["test.tag.2", ts[1], r],
1192
- ["test.tag.1", ts[2], r],
1193
- ["test.tag.1", ts[3], r],
1194
- ["test.tag.1", ts[4], r],
1195
- ["test.tag.1", ts[5], r],
1196
- ["test.tag.1", ts[6], r],
1197
- ["test.tag.1", ts[7], r],
1198
- ["test.tag.2", ts[8], r],
1199
- ["test.tag.1", ts[9], r],
1200
- ["test.tag.2", ts[10], r],
1201
- ]
1202
-
1203
- assert_equal 0, @i.write_count
1204
-
1205
- @i.interrupt_flushes
1206
-
1207
- events.shuffle.each do |tag, time, record|
1208
- @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))
1209
- end
1210
- assert{ @i.buffer.stage.size == 2 } # test.tag.1 x1, test.tag.2 x1
1211
-
1212
- Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )
1213
-
1214
- @i.enqueue_thread_wait
1215
- @i.flush_thread_wakeup
1216
-
1217
- waiting(4) do
1218
- Thread.pass until @i.write_count > 0
1219
- end
1220
-
1221
- assert{ @i.buffer.stage.size == 2 }
1222
- assert{ @i.write_count == 1 }
1223
- assert{ @i.buffer.queue.size == 0 }
1224
-
1225
- # events fulfills a chunk (and queued immediately)
1226
- assert_equal 5, ary.size
1227
- assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1228
- assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1229
-
1230
- Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
1231
-
1232
- @i.enqueue_thread_wait
1233
-
1234
- assert{ @i.buffer.stage.size == 2 }
1235
-
1236
- # to trigger try_flush with flush_thread_burst_interval
1237
- Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )
1238
- @i.enqueue_thread_wait
1239
- Timecop.freeze( Time.parse('2016-04-13 14:04:15 +0900') )
1240
- @i.enqueue_thread_wait
1241
- @i.flush_thread_wakeup
1242
-
1243
- assert{ @i.buffer.stage.size == 0 }
1244
-
1245
- waiting(4) do
1246
- Thread.pass until @i.write_count > 2
1247
- end
1248
-
1249
- assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 }
1250
-
1251
- assert_equal 11, ary.size
1252
- assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1253
- assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1254
-
1255
- assert metachecks.all?{|e| e }
1256
- end
1257
-
1258
- test 'flush_at_shutdown work well when plugin is shutdown' do
1259
- Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )
1260
-
1261
- ary = []
1262
- metachecks = []
1263
-
1264
- @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" }
1265
- @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (chunk.metadata.tag == e[0]) } }
1266
-
1267
- @i.thread_wait_until_start
1268
-
1269
- r = {}
1270
- (0...10).each do |i|
1271
- r["key#{i}"] = "value #{i}"
1272
- end
1273
- ts = [
1274
- event_time('2016-04-13 14:03:21 +0900'), event_time('2016-04-13 14:03:23 +0900'), event_time('2016-04-13 14:03:29 +0900'),
1275
- event_time('2016-04-13 14:03:30 +0900'), event_time('2016-04-13 14:03:33 +0900'), event_time('2016-04-13 14:03:38 +0900'),
1276
- event_time('2016-04-13 14:03:43 +0900'), event_time('2016-04-13 14:03:49 +0900'), event_time('2016-04-13 14:03:51 +0900'),
1277
- event_time('2016-04-13 14:04:00 +0900'), event_time('2016-04-13 14:04:01 +0900'),
1278
- ]
1279
- # size of a event is 197
1280
- events = [
1281
- ["test.tag.1", ts[0], r],
1282
- ["test.tag.2", ts[1], r],
1283
- ["test.tag.1", ts[2], r],
1284
- ["test.tag.1", ts[3], r],
1285
- ["test.tag.1", ts[4], r],
1286
- ["test.tag.1", ts[5], r],
1287
- ["test.tag.1", ts[6], r],
1288
- ["test.tag.1", ts[7], r],
1289
- ["test.tag.2", ts[8], r],
1290
- ["test.tag.1", ts[9], r],
1291
- ["test.tag.2", ts[10], r],
1292
- ]
1293
-
1294
- assert_equal 0, @i.write_count
1295
-
1296
- @i.interrupt_flushes
1297
-
1298
- events.shuffle.each do |tag, time, record|
1299
- @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))
1300
- end
1301
- assert{ @i.buffer.stage.size == 2 } # test.tag.1 x1, test.tag.2 x1
1302
-
1303
- Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )
1304
-
1305
- @i.enqueue_thread_wait
1306
- @i.flush_thread_wakeup
1307
-
1308
- waiting(4) do
1309
- Thread.pass until @i.write_count > 0
1310
- end
1311
-
1312
- assert{ @i.buffer.stage.size == 2 }
1313
- assert{ @i.write_count == 1 }
1314
- assert{ @i.buffer.queue.size == 0 }
1315
-
1316
- # events fulfills a chunk (and queued immediately)
1317
- assert_equal 5, ary.size
1318
- assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1319
- assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1320
-
1321
- @i.stop
1322
- @i.before_shutdown
1323
- @i.shutdown
1324
- @i.after_shutdown
1325
-
1326
- waiting(4) do
1327
- Thread.pass until @i.write_count > 1
1328
- end
1329
-
1330
- assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 3 }
1331
-
1332
- assert_equal 11, ary.size
1333
- assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1334
- assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1335
-
1336
- assert metachecks.all?{|e| e }
1337
- end
1338
- end
1339
-
1340
- sub_test_case 'buffered output feature with variables' do
1341
- setup do
1342
- chunk_key = 'name,service'
1343
- hash = {
1344
- 'flush_interval' => 10,
1345
- 'flush_thread_count' => 1,
1346
- 'flush_thread_burst_interval' => 0.1,
1347
- 'chunk_limit_size' => 1024,
1348
- }
1349
- @i = create_output(:buffered)
1350
- @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
1351
- @i.start
1352
- @i.after_start
1353
- end
1354
-
1355
- test 'default flush_mode is set to :interval' do
1356
- assert_equal :interval, @i.instance_eval{ @flush_mode }
1357
- end
1358
-
1359
- test '#start creates enqueue thread and flush threads' do
1360
- @i.thread_wait_until_start
1361
-
1362
- assert @i.thread_exist?(:flush_thread_0)
1363
- assert @i.thread_exist?(:enqueue_thread)
1364
- end
1365
-
1366
- test '#format is called for each event streams' do
1367
- ary = []
1368
- @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }
1369
-
1370
- t = event_time()
1371
- es = Fluent::ArrayEventStream.new([
1372
- [t, {"key" => "value1", "name" => "moris", "service" => "a"}],
1373
- [t, {"key" => "value2", "name" => "moris", "service" => "b"}],
1374
- ])
1375
-
1376
- 5.times do
1377
- @i.emit_events('tag.test', es)
1378
- end
1379
-
1380
- assert_equal 10, ary.size
1381
- 5.times do |i|
1382
- assert_equal ["tag.test", t, {"key" => "value1", "name" => "moris", "service" => "a"}], ary[i*2]
1383
- assert_equal ["tag.test", t, {"key" => "value2", "name" => "moris", "service" => "b"}], ary[i*2+1]
1384
- end
1385
- end
1386
-
1387
- test '#write is called per value combination of variables, per flush_interval & chunk sizes, and buffer chunk is purged' do
1388
- Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )
1389
-
1390
- ary = []
1391
- metachecks = []
1392
-
1393
- @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" }
1394
- @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (e[2]["name"] == chunk.metadata.variables[:name] && e[2]["service"] == chunk.metadata.variables[:service]) } }
1395
-
1396
- @i.thread_wait_until_start
1397
-
1398
- # size of a event is 195
1399
- dummy_data = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
1400
- events = [
1401
- ["test.tag.1", event_time('2016-04-13 14:03:21 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) xxx-a (6 events)
1402
- ["test.tag.2", event_time('2016-04-13 14:03:23 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], #(2) yyy-a (3 events)
1403
- ["test.tag.1", event_time('2016-04-13 14:03:29 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1)
1404
- ["test.tag.1", event_time('2016-04-13 14:03:30 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1)
1405
- ["test.tag.1", event_time('2016-04-13 14:03:33 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1)
1406
- ["test.tag.1", event_time('2016-04-13 14:03:38 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], #(3) xxx-b (2 events)
1407
- ["test.tag.1", event_time('2016-04-13 14:03:43 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1)
1408
- ["test.tag.1", event_time('2016-04-13 14:03:49 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], #(3)
1409
- ["test.tag.2", event_time('2016-04-13 14:03:51 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], #(2)
1410
- ["test.tag.1", event_time('2016-04-13 14:04:00 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1)
1411
- ["test.tag.2", event_time('2016-04-13 14:04:01 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], #(2)
1412
- ]
1413
-
1414
- assert_equal 0, @i.write_count
1415
-
1416
- @i.interrupt_flushes
1417
-
1418
- events.shuffle.each do |tag, time, record|
1419
- @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))
1420
- end
1421
- assert{ @i.buffer.stage.size == 3 }
1422
-
1423
- Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )
1424
-
1425
- @i.enqueue_thread_wait
1426
- @i.flush_thread_wakeup
1427
-
1428
- waiting(4) do
1429
- Thread.pass until @i.write_count > 0
1430
- end
1431
-
1432
- assert{ @i.buffer.stage.size == 3 }
1433
- assert{ @i.write_count == 1 }
1434
- assert{ @i.buffer.queue.size == 0 }
1435
-
1436
- # events fulfills a chunk (and queued immediately)
1437
- assert_equal 5, ary.size
1438
- assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1439
- assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1440
- assert ary[0...5].all?{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
1441
-
1442
- Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
1443
-
1444
- @i.enqueue_thread_wait
1445
-
1446
- assert{ @i.buffer.stage.size == 3 }
1447
-
1448
- # to trigger try_flush with flush_thread_burst_interval
1449
- Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )
1450
- @i.enqueue_thread_wait
1451
- Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') )
1452
- @i.enqueue_thread_wait
1453
- Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') )
1454
- @i.enqueue_thread_wait
1455
- Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') )
1456
- @i.enqueue_thread_wait
1457
- @i.flush_thread_wakeup
1458
-
1459
- assert{ @i.buffer.stage.size == 0 }
1460
-
1461
- waiting(4) do
1462
- Thread.pass until @i.write_count > 1
1463
- end
1464
-
1465
- assert{ @i.buffer.stage.size == 0 && @i.write_count == 4 }
1466
-
1467
- assert_equal 11, ary.size
1468
- assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1469
- assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1470
- assert_equal 6, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
1471
- assert_equal 3, ary.count{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }
1472
- assert_equal 2, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }
1473
-
1474
- assert metachecks.all?{|e| e }
1475
- end
1476
-
1477
- test 'flush_at_shutdown work well when plugin is shutdown' do
1478
- Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )
1479
-
1480
- ary = []
1481
- metachecks = []
1482
-
1483
- @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" }
1484
- @i.register(:write){|chunk| chunk.read.split("\n").reject{|l| l.empty? }.each{|data| e = JSON.parse(data); ary << e; metachecks << (e[2]["name"] == chunk.metadata.variables[:name] && e[2]["service"] == chunk.metadata.variables[:service]) } }
1485
-
1486
- @i.thread_wait_until_start
1487
-
1488
- # size of a event is 195
1489
- dummy_data = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
1490
- events = [
1491
- ["test.tag.1", event_time('2016-04-13 14:03:21 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1) xxx-a (6 events)
1492
- ["test.tag.2", event_time('2016-04-13 14:03:23 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], #(2) yyy-a (3 events)
1493
- ["test.tag.1", event_time('2016-04-13 14:03:29 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1)
1494
- ["test.tag.1", event_time('2016-04-13 14:03:30 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1)
1495
- ["test.tag.1", event_time('2016-04-13 14:03:33 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1)
1496
- ["test.tag.1", event_time('2016-04-13 14:03:38 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], #(3) xxx-b (2 events)
1497
- ["test.tag.1", event_time('2016-04-13 14:03:43 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1)
1498
- ["test.tag.1", event_time('2016-04-13 14:03:49 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}], #(3)
1499
- ["test.tag.2", event_time('2016-04-13 14:03:51 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], #(2)
1500
- ["test.tag.1", event_time('2016-04-13 14:04:00 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}], #(1)
1501
- ["test.tag.2", event_time('2016-04-13 14:04:01 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}], #(2)
1502
- ]
1503
-
1504
- assert_equal 0, @i.write_count
1505
-
1506
- @i.interrupt_flushes
1507
-
1508
- events.shuffle.each do |tag, time, record|
1509
- @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))
1510
- end
1511
- assert{ @i.buffer.stage.size == 3 }
1512
-
1513
- Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )
1514
-
1515
- @i.enqueue_thread_wait
1516
- @i.flush_thread_wakeup
1517
-
1518
- waiting(4) do
1519
- Thread.pass until @i.write_count > 0
1520
- end
1521
-
1522
- assert{ @i.buffer.stage.size == 3 }
1523
- assert{ @i.write_count == 1 }
1524
- assert{ @i.buffer.queue.size == 0 }
1525
-
1526
- # events fulfills a chunk (and queued immediately)
1527
- assert_equal 5, ary.size
1528
- assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1529
- assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1530
-
1531
- @i.stop
1532
- @i.before_shutdown
1533
- @i.shutdown
1534
- @i.after_shutdown
1535
-
1536
- waiting(4) do
1537
- Thread.pass until @i.write_count > 1
1538
- end
1539
-
1540
- assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 4 }
1541
-
1542
- assert_equal 11, ary.size
1543
- assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1544
- assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1545
- assert_equal 6, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
1546
- assert_equal 3, ary.count{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }
1547
- assert_equal 2, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }
1548
-
1549
- assert metachecks.all?{|e| e }
1550
- end
1551
- end
1552
-
1553
- sub_test_case 'buffered output feature with many keys' do
1554
- test 'default flush mode is set to :interval if keys does not include time' do
1555
- chunk_key = 'name,service,tag'
1556
- hash = {
1557
- 'flush_interval' => 10,
1558
- 'flush_thread_count' => 1,
1559
- 'flush_thread_burst_interval' => 0.1,
1560
- 'chunk_limit_size' => 1024,
1561
- }
1562
- @i = create_output(:buffered)
1563
- @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
1564
- @i.start
1565
- @i.after_start
1566
-
1567
- assert_equal :interval, @i.instance_eval{ @flush_mode }
1568
- end
1569
-
1570
- test 'default flush mode is set to :lazy if keys includes time' do
1571
- chunk_key = 'name,service,tag,time'
1572
- hash = {
1573
- 'timekey' => 60,
1574
- 'flush_interval' => 10,
1575
- 'flush_thread_count' => 1,
1576
- 'flush_thread_burst_interval' => 0.1,
1577
- 'chunk_limit_size' => 1024,
1578
- }
1579
- @i = create_output(:buffered)
1580
- @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
1581
- @i.start
1582
- @i.after_start
1583
-
1584
- assert_equal :lazy, @i.instance_eval{ @flush_mode }
1585
- end
1586
- end
1587
-
1588
- sub_test_case 'buffered output feature with delayed commit' do
1589
- setup do
1590
- chunk_key = 'tag'
1591
- hash = {
1592
- 'flush_interval' => 10,
1593
- 'flush_thread_count' => 1,
1594
- 'flush_thread_burst_interval' => 0.1,
1595
- 'delayed_commit_timeout' => 30,
1596
- 'chunk_limit_size' => 1024,
1597
- }
1598
- @i = create_output(:delayed)
1599
- @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
1600
- @i.start
1601
- @i.after_start
1602
- @i.log = Fluent::Test::TestLogger.new
1603
- end
1604
-
1605
- test '#format is called for each event streams' do
1606
- ary = []
1607
- @i.register(:format){|tag, time, record| ary << [tag, time, record]; '' }
1608
-
1609
- t = event_time()
1610
- es = Fluent::ArrayEventStream.new([
1611
- [t, {"key" => "value1", "name" => "moris", "service" => "a"}],
1612
- [t, {"key" => "value2", "name" => "moris", "service" => "b"}],
1613
- ])
1614
-
1615
- 5.times do
1616
- @i.emit_events('tag.test', es)
1617
- end
1618
-
1619
- assert_equal 10, ary.size
1620
- 5.times do |i|
1621
- assert_equal ["tag.test", t, {"key" => "value1", "name" => "moris", "service" => "a"}], ary[i*2]
1622
- assert_equal ["tag.test", t, {"key" => "value2", "name" => "moris", "service" => "b"}], ary[i*2+1]
1623
- end
1624
- end
1625
-
1626
- test '#try_write is called per flush, buffer chunk is not purged until #commit_write is called' do
1627
- Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )
1628
-
1629
- ary = []
1630
- metachecks = []
1631
- chunks = []
1632
-
1633
- @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" }
1634
- @i.register(:try_write) do |chunk|
1635
- chunks << chunk
1636
- chunk.read.split("\n").reject{|l| l.empty? }.each do |data|
1637
- e = JSON.parse(data)
1638
- ary << e
1639
- metachecks << (e[0] == chunk.metadata.tag)
1640
- end
1641
- end
1642
-
1643
- @i.thread_wait_until_start
1644
-
1645
- # size of a event is 195
1646
- dummy_data = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
1647
- events = [
1648
- ["test.tag.1", event_time('2016-04-13 14:03:21 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1649
- ["test.tag.2", event_time('2016-04-13 14:03:23 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}],
1650
- ["test.tag.1", event_time('2016-04-13 14:03:29 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1651
- ["test.tag.1", event_time('2016-04-13 14:03:30 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1652
- ["test.tag.1", event_time('2016-04-13 14:03:33 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1653
- ["test.tag.1", event_time('2016-04-13 14:03:38 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}],
1654
- ["test.tag.1", event_time('2016-04-13 14:03:43 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1655
- ["test.tag.1", event_time('2016-04-13 14:03:49 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}],
1656
- ["test.tag.2", event_time('2016-04-13 14:03:51 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}],
1657
- ["test.tag.1", event_time('2016-04-13 14:04:00 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1658
- ["test.tag.2", event_time('2016-04-13 14:04:01 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}],
1659
- ]
1660
-
1661
- assert_equal 0, @i.write_count
1662
-
1663
- @i.interrupt_flushes
1664
-
1665
- events.shuffle.each do |tag, time, record|
1666
- @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))
1667
- end
1668
- assert{ @i.buffer.stage.size == 2 }
1669
-
1670
- Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )
1671
-
1672
- @i.enqueue_thread_wait
1673
- @i.flush_thread_wakeup
1674
-
1675
- waiting(4) do
1676
- Thread.pass until @i.write_count > 0
1677
- end
1678
-
1679
- assert{ @i.buffer.stage.size == 2 }
1680
- assert{ @i.write_count == 1 }
1681
- assert{ @i.buffer.queue.size == 0 }
1682
- assert{ @i.buffer.dequeued.size == 1 }
1683
-
1684
- # events fulfills a chunk (and queued immediately)
1685
- assert_equal 5, ary.size
1686
- assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1687
- assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1688
-
1689
- assert_equal 1, chunks.size
1690
- assert !chunks.first.empty?
1691
-
1692
- Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
1693
-
1694
- @i.enqueue_thread_wait
1695
-
1696
- assert{ @i.buffer.stage.size == 2 }
1697
-
1698
- # to trigger try_flush with flush_thread_burst_interval
1699
- Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )
1700
- @i.enqueue_thread_wait
1701
- Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') )
1702
- @i.enqueue_thread_wait
1703
- Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') )
1704
- @i.enqueue_thread_wait
1705
- Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') )
1706
- @i.enqueue_thread_wait
1707
- @i.flush_thread_wakeup
1708
-
1709
- assert{ @i.buffer.stage.size == 0 }
1710
-
1711
- waiting(4) do
1712
- Thread.pass until @i.write_count > 1
1713
- end
1714
-
1715
- assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 }
1716
- assert{ @i.buffer.dequeued.size == 3 }
1717
-
1718
- assert_equal 11, ary.size
1719
- assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1720
- assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1721
-
1722
- assert_equal 3, chunks.size
1723
- assert chunks.all?{|c| !c.empty? }
1724
-
1725
- assert metachecks.all?{|e| e }
1726
-
1727
- @i.commit_write(chunks[0].unique_id)
1728
- assert{ @i.buffer.dequeued.size == 2 }
1729
- assert chunks[0].empty?
1730
-
1731
- @i.commit_write(chunks[1].unique_id)
1732
- assert{ @i.buffer.dequeued.size == 1 }
1733
- assert chunks[1].empty?
1734
-
1735
- @i.commit_write(chunks[2].unique_id)
1736
- assert{ @i.buffer.dequeued.size == 0 }
1737
- assert chunks[2].empty?
1738
-
1739
- # no problem to commit chunks already committed
1740
- assert_nothing_raised do
1741
- @i.commit_write(chunks[2].unique_id)
1742
- end
1743
- end
1744
-
1745
- test '#rollback_write and #try_rollback_write can rollback buffer chunks for delayed commit after timeout, and then be able to write it again' do
1746
- Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )
1747
-
1748
- ary = []
1749
- metachecks = []
1750
- chunks = []
1751
-
1752
- @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" }
1753
- @i.register(:try_write) do |chunk|
1754
- chunks << chunk
1755
- chunk.read.split("\n").reject{|l| l.empty? }.each do |data|
1756
- e = JSON.parse(data)
1757
- ary << e
1758
- metachecks << (e[0] == chunk.metadata.tag)
1759
- end
1760
- end
1761
-
1762
- @i.thread_wait_until_start
1763
-
1764
- # size of a event is 195
1765
- dummy_data = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
1766
- events = [
1767
- ["test.tag.1", event_time('2016-04-13 14:03:21 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1768
- ["test.tag.2", event_time('2016-04-13 14:03:23 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}],
1769
- ["test.tag.1", event_time('2016-04-13 14:03:29 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1770
- ["test.tag.1", event_time('2016-04-13 14:03:30 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1771
- ["test.tag.1", event_time('2016-04-13 14:03:33 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1772
- ["test.tag.1", event_time('2016-04-13 14:03:38 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}],
1773
- ["test.tag.1", event_time('2016-04-13 14:03:43 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1774
- ["test.tag.1", event_time('2016-04-13 14:03:49 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}],
1775
- ["test.tag.2", event_time('2016-04-13 14:03:51 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}],
1776
- ["test.tag.1", event_time('2016-04-13 14:04:00 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1777
- ["test.tag.2", event_time('2016-04-13 14:04:01 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}],
1778
- ]
1779
-
1780
- assert_equal 0, @i.write_count
1781
-
1782
- @i.interrupt_flushes
1783
-
1784
- events.shuffle.each do |tag, time, record|
1785
- @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))
1786
- end
1787
- assert{ @i.buffer.stage.size == 2 }
1788
-
1789
- Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )
1790
-
1791
- @i.enqueue_thread_wait
1792
- @i.flush_thread_wakeup
1793
-
1794
- waiting(4) do
1795
- Thread.pass until @i.write_count > 0
1796
- end
1797
-
1798
- assert{ @i.buffer.stage.size == 2 }
1799
- assert{ @i.write_count == 1 }
1800
- assert{ @i.buffer.queue.size == 0 }
1801
- assert{ @i.buffer.dequeued.size == 1 }
1802
-
1803
- # events fulfills a chunk (and queued immediately)
1804
- assert_equal 5, ary.size
1805
- assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1806
- assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1807
-
1808
- assert_equal 1, chunks.size
1809
- assert !chunks.first.empty?
1810
-
1811
- Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
1812
-
1813
- @i.enqueue_thread_wait
1814
-
1815
- assert{ @i.buffer.stage.size == 2 }
1816
-
1817
- # to trigger try_flush with flush_thread_burst_interval
1818
- Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )
1819
- @i.enqueue_thread_wait
1820
- Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') )
1821
- @i.enqueue_thread_wait
1822
- Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') )
1823
- @i.enqueue_thread_wait
1824
- Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') )
1825
- @i.enqueue_thread_wait
1826
- @i.flush_thread_wakeup
1827
-
1828
- assert{ @i.buffer.stage.size == 0 }
1829
-
1830
- waiting(4) do
1831
- Thread.pass until @i.write_count > 2
1832
- end
1833
-
1834
- assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 }
1835
- assert{ @i.buffer.dequeued.size == 3 }
1836
-
1837
- assert_equal 11, ary.size
1838
- assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1839
- assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1840
-
1841
- assert_equal 3, chunks.size
1842
- assert chunks.all?{|c| !c.empty? }
1843
-
1844
- assert metachecks.all?{|e| e }
1845
-
1846
- @i.interrupt_flushes
1847
-
1848
- @i.rollback_write(chunks[2].unique_id)
1849
-
1850
- assert{ @i.buffer.dequeued.size == 2 }
1851
- assert{ @i.buffer.queue.size == 1 && @i.buffer.queue.first.unique_id == chunks[2].unique_id }
1852
-
1853
- Timecop.freeze( Time.parse('2016-04-13 14:04:15 +0900') )
1854
- @i.enqueue_thread_wait
1855
- @i.flush_thread_wakeup
1856
-
1857
- waiting(4) do
1858
- Thread.pass until @i.write_count > 3
1859
- end
1860
-
1861
- assert{ @i.write_count == 4 }
1862
- assert{ @i.rollback_count == 1 }
1863
- assert{ @i.instance_eval{ @dequeued_chunks.size } == 3 }
1864
- assert{ @i.buffer.dequeued.size == 3 }
1865
- assert{ @i.buffer.queue.size == 0 }
1866
-
1867
- assert_equal 4, chunks.size
1868
- assert chunks[2].unique_id == chunks[3].unique_id
1869
-
1870
- ary.reject!{|e| true }
1871
- chunks.reject!{|e| true }
1872
-
1873
- Timecop.freeze( Time.parse('2016-04-13 14:04:46 +0900') )
1874
- @i.enqueue_thread_wait
1875
- @i.flush_thread_wakeup
1876
-
1877
- waiting(4) do
1878
- Thread.pass until @i.rollback_count == 4
1879
- end
1880
-
1881
- assert{ chunks[0...3].all?{|c| !c.empty? } }
1882
-
1883
- # rollback is in progress, but some may be flushed again in retry state, after rollback
1884
- # retry.next_time is 14:04:49
1885
- Timecop.freeze( Time.parse('2016-04-13 14:04:51 +0900') )
1886
- @i.enqueue_thread_wait
1887
- @i.flush_thread_wakeup
1888
-
1889
- waiting(4) do
1890
- Thread.pass until @i.write_count == 7
1891
- end
1892
-
1893
- assert{ @i.write_count == 7 }
1894
- assert_equal 11, ary.size
1895
- assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1896
- assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1897
- assert{ chunks.size == 3 }
1898
- assert{ chunks.all?{|c| !c.empty? } }
1899
-
1900
- chunks.each{|c| @i.commit_write(c.unique_id) }
1901
- assert{ chunks.all?{|c| c.empty? } }
1902
-
1903
- assert{ @i.buffer.dequeued.size == 0 }
1904
- end
1905
-
1906
- test '#try_rollback_all will be called for all waiting chunks after shutdown' do
1907
- Timecop.freeze( Time.parse('2016-04-13 14:04:01 +0900') )
1908
-
1909
- ary = []
1910
- metachecks = []
1911
- chunks = []
1912
-
1913
- @i.register(:format){|tag,time,record| [tag,time,record].to_json + "\n" }
1914
- @i.register(:try_write) do |chunk|
1915
- chunks << chunk
1916
- chunk.read.split("\n").reject{|l| l.empty? }.each do |data|
1917
- e = JSON.parse(data)
1918
- ary << e
1919
- metachecks << (e[0] == chunk.metadata.tag)
1920
- end
1921
- end
1922
-
1923
- @i.thread_wait_until_start
1924
-
1925
- # size of a event is 195
1926
- dummy_data = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
1927
- events = [
1928
- ["test.tag.1", event_time('2016-04-13 14:03:21 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1929
- ["test.tag.2", event_time('2016-04-13 14:03:23 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}],
1930
- ["test.tag.1", event_time('2016-04-13 14:03:29 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1931
- ["test.tag.1", event_time('2016-04-13 14:03:30 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1932
- ["test.tag.1", event_time('2016-04-13 14:03:33 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1933
- ["test.tag.1", event_time('2016-04-13 14:03:38 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}],
1934
- ["test.tag.1", event_time('2016-04-13 14:03:43 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1935
- ["test.tag.1", event_time('2016-04-13 14:03:49 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "b"}],
1936
- ["test.tag.2", event_time('2016-04-13 14:03:51 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}],
1937
- ["test.tag.1", event_time('2016-04-13 14:04:00 +0900'), {"data" => dummy_data, "name" => "xxx", "service" => "a"}],
1938
- ["test.tag.2", event_time('2016-04-13 14:04:01 +0900'), {"data" => dummy_data, "name" => "yyy", "service" => "a"}],
1939
- ]
1940
-
1941
- assert_equal 0, @i.write_count
1942
-
1943
- @i.interrupt_flushes
1944
-
1945
- events.shuffle.each do |tag, time, record|
1946
- @i.emit_events(tag, Fluent::ArrayEventStream.new([ [time, record] ]))
1947
- end
1948
- assert{ @i.buffer.stage.size == 2 }
1949
-
1950
- Timecop.freeze( Time.parse('2016-04-13 14:04:02 +0900') )
1951
-
1952
- @i.enqueue_thread_wait
1953
- @i.flush_thread_wakeup
1954
-
1955
- waiting(4) do
1956
- Thread.pass until @i.write_count > 0
1957
- end
1958
-
1959
- assert{ @i.buffer.stage.size == 2 }
1960
- assert{ @i.write_count == 1 }
1961
- assert{ @i.buffer.queue.size == 0 }
1962
- assert{ @i.buffer.dequeued.size == 1 }
1963
-
1964
- # events fulfills a chunk (and queued immediately)
1965
- assert_equal 5, ary.size
1966
- assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1967
- assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1968
-
1969
- assert_equal 1, chunks.size
1970
- assert !chunks.first.empty?
1971
-
1972
- Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
1973
-
1974
- @i.enqueue_thread_wait
1975
-
1976
- assert{ @i.buffer.stage.size == 2 }
1977
-
1978
- # to trigger try_flush with flush_thread_burst_interval
1979
- Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )
1980
- @i.enqueue_thread_wait
1981
- Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') )
1982
- @i.enqueue_thread_wait
1983
- Timecop.freeze( Time.parse('2016-04-13 14:04:13 +0900') )
1984
- @i.enqueue_thread_wait
1985
- Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') )
1986
- @i.enqueue_thread_wait
1987
- @i.flush_thread_wakeup
1988
-
1989
- assert{ @i.buffer.stage.size == 0 }
1990
-
1991
- waiting(4) do
1992
- Thread.pass until @i.write_count > 2
1993
- end
1994
-
1995
- assert{ @i.buffer.stage.size == 0 }
1996
- assert{ @i.buffer.queue.size == 0 }
1997
- assert{ @i.buffer.dequeued.size == 3 }
1998
- assert{ @i.write_count == 3 }
1999
- assert{ @i.rollback_count == 0 }
2000
-
2001
- assert_equal 11, ary.size
2002
- assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
2003
- assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
2004
-
2005
- assert{ chunks.size == 3 }
2006
- assert{ chunks.all?{|c| !c.empty? } }
2007
-
2008
- @i.register(:shutdown_hook){ @i.commit_write(chunks[1].unique_id) }
2009
-
2010
- @i.stop
2011
- @i.before_shutdown
2012
- @i.shutdown
2013
-
2014
- assert{ @i.buffer.dequeued.size == 2 }
2015
- assert{ !chunks[0].empty? }
2016
- assert{ chunks[1].empty? }
2017
- assert{ !chunks[2].empty? }
2018
-
2019
- @i.after_shutdown
2020
-
2021
- assert{ @i.rollback_count == 2 }
2022
- end
2023
- end
2024
- end