fluentd 1.17.0 → 1.17.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
@@ -1,1493 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/plugin/buffer'
3
- require 'fluent/plugin/buffer/memory_chunk'
4
- require 'fluent/plugin/compressable'
5
- require 'fluent/plugin/buffer/chunk'
6
- require 'fluent/event'
7
- require 'flexmock/test_unit'
8
-
9
- require 'fluent/log'
10
- require 'fluent/plugin_id'
11
-
12
- require 'time'
13
-
14
- module FluentPluginBufferTest
15
- class DummyOutputPlugin < Fluent::Plugin::Base
16
- include Fluent::PluginId
17
- include Fluent::PluginLoggerMixin
18
- end
19
- class DummyMemoryChunkError < StandardError; end
20
- class DummyMemoryChunk < Fluent::Plugin::Buffer::MemoryChunk
21
- attr_reader :append_count, :rollbacked, :closed, :purged, :chunk
22
- attr_accessor :failing
23
- def initialize(metadata, compress: :text)
24
- super
25
- @append_count = 0
26
- @rollbacked = false
27
- @closed = false
28
- @purged = false
29
- @failing = false
30
- end
31
- def concat(data, size)
32
- @append_count += 1
33
- raise DummyMemoryChunkError if @failing
34
- super
35
- end
36
- def rollback
37
- super
38
- @rollbacked = true
39
- end
40
- def close
41
- super
42
- @closed = true
43
- end
44
- def purge
45
- super
46
- @purged = true
47
- end
48
- end
49
- class DummyPlugin < Fluent::Plugin::Buffer
50
- def create_metadata(timekey=nil, tag=nil, variables=nil)
51
- Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
52
- end
53
- def create_chunk(metadata, data)
54
- c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata)
55
- c.append(data)
56
- c.commit
57
- c
58
- end
59
- def create_chunk_es(metadata, es)
60
- c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata)
61
- c.concat(es.to_msgpack_stream, es.size)
62
- c.commit
63
- c
64
- end
65
- def resume
66
- dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
67
- dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)
68
- dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)
69
- dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil)
70
- staged = {
71
- dm2 => create_chunk(dm2, ["b" * 100]).staged!,
72
- dm3 => create_chunk(dm3, ["c" * 100]).staged!,
73
- }
74
- queued = [
75
- create_chunk(dm0, ["0" * 100]).enqueued!,
76
- create_chunk(dm1, ["a" * 100]).enqueued!,
77
- create_chunk(dm1, ["a" * 3]).enqueued!,
78
- ]
79
- return staged, queued
80
- end
81
- def generate_chunk(metadata)
82
- DummyMemoryChunk.new(metadata, compress: @compress)
83
- end
84
- end
85
- end
86
-
87
- class BufferTest < Test::Unit::TestCase
88
- def create_buffer(hash)
89
- buffer_conf = config_element('buffer', '', hash, [])
90
- owner = FluentPluginBufferTest::DummyOutputPlugin.new
91
- owner.configure(config_element('ROOT', '', {}, [ buffer_conf ]))
92
- p = FluentPluginBufferTest::DummyPlugin.new
93
- p.owner = owner
94
- p.configure(buffer_conf)
95
- p
96
- end
97
-
98
- def create_metadata(timekey=nil, tag=nil, variables=nil)
99
- Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
100
- end
101
-
102
- def create_chunk(metadata, data)
103
- c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata)
104
- c.append(data)
105
- c.commit
106
- c
107
- end
108
-
109
- def create_chunk_es(metadata, es)
110
- c = FluentPluginBufferTest::DummyMemoryChunk.new(metadata)
111
- c.concat(es.to_msgpack_stream, es.size)
112
- c.commit
113
- c
114
- end
115
-
116
- setup do
117
- Fluent::Test.setup
118
- end
119
-
120
- sub_test_case 'using base buffer class' do
121
- setup do
122
- buffer_conf = config_element('buffer', '', {}, [])
123
- owner = FluentPluginBufferTest::DummyOutputPlugin.new
124
- owner.configure(config_element('ROOT', '', {}, [ buffer_conf ]))
125
- p = Fluent::Plugin::Buffer.new
126
- p.owner = owner
127
- p.configure(buffer_conf)
128
- @p = p
129
- end
130
-
131
- test 'default persistency is false' do
132
- assert !@p.persistent?
133
- end
134
-
135
- test 'chunk bytes limit is 8MB, and total bytes limit is 512MB' do
136
- assert_equal 8*1024*1024, @p.chunk_limit_size
137
- assert_equal 512*1024*1024, @p.total_limit_size
138
- end
139
-
140
- test 'chunk records limit is ignored in default' do
141
- assert_nil @p.chunk_limit_records
142
- end
143
-
144
- test '#storable? checks total size of staged and enqueued(includes dequeued chunks) against total_limit_size' do
145
- assert_equal 512*1024*1024, @p.total_limit_size
146
- assert_equal 0, @p.stage_size
147
- assert_equal 0, @p.queue_size
148
- assert @p.storable?
149
-
150
- @p.stage_size = 256 * 1024 * 1024
151
- @p.queue_size = 256 * 1024 * 1024 - 1
152
- assert @p.storable?
153
-
154
- @p.queue_size = 256 * 1024 * 1024
155
- assert !@p.storable?
156
- end
157
-
158
- test '#resume must be implemented by subclass' do
159
- assert_raise NotImplementedError do
160
- @p.resume
161
- end
162
- end
163
-
164
- test '#generate_chunk must be implemented by subclass' do
165
- assert_raise NotImplementedError do
166
- @p.generate_chunk(Object.new)
167
- end
168
- end
169
- end
170
-
171
- sub_test_case 'with default configuration and dummy implementation' do
172
- setup do
173
- @p = create_buffer({'queued_chunks_limit_size' => 100})
174
- @dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
175
- @dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)
176
- @dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)
177
- @dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil)
178
- @p.start
179
- end
180
-
181
- test '#start resumes buffer states and update queued numbers per metadata' do
182
- plugin = create_buffer({})
183
-
184
- assert_equal({}, plugin.stage)
185
- assert_equal([], plugin.queue)
186
- assert_equal({}, plugin.dequeued)
187
- assert_equal({}, plugin.queued_num)
188
-
189
- assert_equal 0, plugin.stage_size
190
- assert_equal 0, plugin.queue_size
191
- assert_equal [], plugin.timekeys
192
-
193
- # @p is started plugin
194
-
195
- assert_equal [@dm2,@dm3], @p.stage.keys
196
- assert_equal "b" * 100, @p.stage[@dm2].read
197
- assert_equal "c" * 100, @p.stage[@dm3].read
198
-
199
- assert_equal 200, @p.stage_size
200
-
201
- assert_equal 3, @p.queue.size
202
- assert_equal "0" * 100, @p.queue[0].read
203
- assert_equal "a" * 100, @p.queue[1].read
204
- assert_equal "a" * 3, @p.queue[2].read
205
-
206
- assert_equal 203, @p.queue_size
207
-
208
- # staged, queued
209
- assert_equal 1, @p.queued_num[@dm0]
210
- assert_equal 2, @p.queued_num[@dm1]
211
- end
212
-
213
- test '#close closes all chunks in dequeued, enqueued and staged' do
214
- dmx = create_metadata(Time.parse('2016-04-11 15:50:00 +0000').to_i, nil, nil)
215
- cx = create_chunk(dmx, ["x" * 1024])
216
- @p.dequeued[cx.unique_id] = cx
217
-
218
- staged_chunks = @p.stage.values.dup
219
- queued_chunks = @p.queue.dup
220
-
221
- @p.close
222
-
223
- assert cx.closed
224
- assert{ staged_chunks.all?{|c| c.closed } }
225
- assert{ queued_chunks.all?{|c| c.closed } }
226
- end
227
-
228
- test '#terminate initializes all internal states' do
229
- dmx = create_metadata(Time.parse('2016-04-11 15:50:00 +0000').to_i, nil, nil)
230
- cx = create_chunk(dmx, ["x" * 1024])
231
- @p.dequeued[cx.unique_id] = cx
232
-
233
- @p.close
234
-
235
- @p.terminate
236
-
237
- assert_nil @p.stage
238
- assert_nil @p.queue
239
- assert_nil @p.dequeued
240
- assert_nil @p.queued_num
241
- assert_nil @p.stage_length_metrics
242
- assert_nil @p.stage_size_metrics
243
- assert_nil @p.queue_length_metrics
244
- assert_nil @p.queue_size_metrics
245
- assert_nil @p.available_buffer_space_ratios_metrics
246
- assert_nil @p.total_queued_size_metrics
247
- assert_nil @p.newest_timekey_metrics
248
- assert_nil @p.oldest_timekey_metrics
249
- assert_equal [], @p.timekeys
250
- end
251
-
252
- test '#queued_records returns total number of size in all chunks in queue' do
253
- assert_equal 3, @p.queue.size
254
-
255
- r0 = @p.queue[0].size
256
- assert_equal 1, r0
257
- r1 = @p.queue[1].size
258
- assert_equal 1, r1
259
- r2 = @p.queue[2].size
260
- assert_equal 1, r2
261
-
262
- assert_equal (r0+r1+r2), @p.queued_records
263
- end
264
-
265
- test '#queued? returns queue has any chunks or not without arguments' do
266
- assert @p.queued?
267
-
268
- @p.queue.reject!{|_c| true }
269
- assert !@p.queued?
270
- end
271
-
272
- test '#queued? returns queue has chunks for specified metadata with an argument' do
273
- assert @p.queued?(@dm0)
274
- assert @p.queued?(@dm1)
275
- assert !@p.queued?(@dm2)
276
- end
277
-
278
- test '#enqueue_chunk enqueues a chunk on stage with specified metadata' do
279
- assert_equal 2, @p.stage.size
280
- assert_equal [@dm2,@dm3], @p.stage.keys
281
- assert_equal 3, @p.queue.size
282
- assert_nil @p.queued_num[@dm2]
283
-
284
- assert_equal 200, @p.stage_size
285
- assert_equal 203, @p.queue_size
286
-
287
- @p.enqueue_chunk(@dm2)
288
-
289
- assert_equal [@dm3], @p.stage.keys
290
- assert_equal @dm2, @p.queue.last.metadata
291
- assert_equal 1, @p.queued_num[@dm2]
292
- assert_equal 100, @p.stage_size
293
- assert_equal 303, @p.queue_size
294
- end
295
-
296
- test '#enqueue_chunk ignores empty chunks' do
297
- assert_equal 3, @p.queue.size
298
-
299
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
300
- c = create_chunk(m, [''])
301
- @p.stage[m] = c
302
- assert @p.stage[m].empty?
303
- assert !c.closed
304
-
305
- @p.enqueue_chunk(m)
306
-
307
- assert_nil @p.stage[m]
308
- assert_equal 3, @p.queue.size
309
- assert_nil @p.queued_num[m]
310
- assert c.closed
311
- end
312
-
313
- test '#enqueue_chunk calls #enqueued! if chunk responds to it' do
314
- assert_equal 3, @p.queue.size
315
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
316
- c = create_chunk(m, ['c' * 256])
317
- callback_called = false
318
- (class << c; self; end).module_eval do
319
- define_method(:enqueued!){ callback_called = true }
320
- end
321
-
322
- @p.stage[m] = c
323
- @p.enqueue_chunk(m)
324
-
325
- assert_equal c, @p.queue.last
326
- assert callback_called
327
- end
328
-
329
- test '#enqueue_all enqueues chunks on stage which given block returns true with' do
330
- m1 = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
331
- c1 = create_chunk(m1, ['c' * 256])
332
- @p.stage[m1] = c1
333
- m2 = @p.metadata(timekey: Time.parse('2016-04-11 16:50:00 +0000').to_i)
334
- c2 = create_chunk(m2, ['c' * 256])
335
- @p.stage[m2] = c2
336
-
337
- assert_equal [@dm2,@dm3,m1,m2], @p.stage.keys
338
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
339
-
340
- @p.enqueue_all{ |m, c| m.timekey < Time.parse('2016-04-11 16:41:00 +0000').to_i }
341
-
342
- assert_equal [m2], @p.stage.keys
343
- assert_equal [@dm0,@dm1,@dm1,@dm2,@dm3,m1], @p.queue.map(&:metadata)
344
- end
345
-
346
- test '#enqueue_all enqueues all chunks on stage without block' do
347
- m1 = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
348
- c1 = create_chunk(m1, ['c' * 256])
349
- @p.stage[m1] = c1
350
- m2 = @p.metadata(timekey: Time.parse('2016-04-11 16:50:00 +0000').to_i)
351
- c2 = create_chunk(m2, ['c' * 256])
352
- @p.stage[m2] = c2
353
-
354
- assert_equal [@dm2,@dm3,m1,m2], @p.stage.keys
355
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
356
-
357
- @p.enqueue_all
358
-
359
- assert_equal [], @p.stage.keys
360
- assert_equal [@dm0,@dm1,@dm1,@dm2,@dm3,m1,m2], @p.queue.map(&:metadata)
361
- end
362
-
363
- test '#dequeue_chunk dequeues a chunk from queue if a chunk exists' do
364
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
365
- assert_equal({}, @p.dequeued)
366
-
367
- m1 = @p.dequeue_chunk
368
- assert_equal @dm0, m1.metadata
369
- assert_equal @dm0, @p.dequeued[m1.unique_id].metadata
370
-
371
- m2 = @p.dequeue_chunk
372
- assert_equal @dm1, m2.metadata
373
- assert_equal @dm1, @p.dequeued[m2.unique_id].metadata
374
-
375
- m3 = @p.dequeue_chunk
376
- assert_equal @dm1, m3.metadata
377
- assert_equal @dm1, @p.dequeued[m3.unique_id].metadata
378
-
379
- m4 = @p.dequeue_chunk
380
- assert_nil m4
381
- end
382
-
383
- test '#takeback_chunk resumes a chunk from dequeued to queued at the head of queue, and returns true' do
384
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
385
- assert_equal({}, @p.dequeued)
386
-
387
- m1 = @p.dequeue_chunk
388
- assert_equal @dm0, m1.metadata
389
- assert_equal @dm0, @p.dequeued[m1.unique_id].metadata
390
- assert_equal [@dm1,@dm1], @p.queue.map(&:metadata)
391
- assert_equal({m1.unique_id => m1}, @p.dequeued)
392
-
393
- assert @p.takeback_chunk(m1.unique_id)
394
-
395
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
396
- assert_equal({}, @p.dequeued)
397
- end
398
-
399
- test '#purge_chunk removes a chunk specified by argument id from dequeued chunks' do
400
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
401
- assert_equal({}, @p.dequeued)
402
-
403
- m0 = @p.dequeue_chunk
404
- m1 = @p.dequeue_chunk
405
-
406
- assert @p.takeback_chunk(m0.unique_id)
407
-
408
- assert_equal [@dm0,@dm1], @p.queue.map(&:metadata)
409
- assert_equal({m1.unique_id => m1}, @p.dequeued)
410
-
411
- assert !m1.purged
412
-
413
- @p.purge_chunk(m1.unique_id)
414
- assert m1.purged
415
-
416
- assert_equal [@dm0,@dm1], @p.queue.map(&:metadata)
417
- assert_equal({}, @p.dequeued)
418
- end
419
-
420
- test '#purge_chunk removes an argument metadata if no chunks exist on stage or in queue' do
421
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
422
- assert_equal({}, @p.dequeued)
423
-
424
- m0 = @p.dequeue_chunk
425
-
426
- assert_equal [@dm1,@dm1], @p.queue.map(&:metadata)
427
- assert_equal({m0.unique_id => m0}, @p.dequeued)
428
-
429
- assert !m0.purged
430
-
431
- @p.purge_chunk(m0.unique_id)
432
- assert m0.purged
433
-
434
- assert_equal [@dm1,@dm1], @p.queue.map(&:metadata)
435
- assert_equal({}, @p.dequeued)
436
- end
437
-
438
- test '#takeback_chunk returns false if specified chunk_id is already purged' do
439
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
440
- assert_equal({}, @p.dequeued)
441
-
442
- m0 = @p.dequeue_chunk
443
-
444
- assert_equal [@dm1,@dm1], @p.queue.map(&:metadata)
445
- assert_equal({m0.unique_id => m0}, @p.dequeued)
446
-
447
- assert !m0.purged
448
-
449
- @p.purge_chunk(m0.unique_id)
450
- assert m0.purged
451
-
452
- assert_equal [@dm1,@dm1], @p.queue.map(&:metadata)
453
- assert_equal({}, @p.dequeued)
454
-
455
- assert !@p.takeback_chunk(m0.unique_id)
456
-
457
- assert_equal [@dm1,@dm1], @p.queue.map(&:metadata)
458
- assert_equal({}, @p.dequeued)
459
- end
460
-
461
- test '#clear_queue! removes all chunks in queue, but leaves staged chunks' do
462
- qchunks = @p.queue.dup
463
-
464
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
465
- assert_equal 2, @p.stage.size
466
- assert_equal({}, @p.dequeued)
467
-
468
- @p.clear_queue!
469
-
470
- assert_equal [], @p.queue
471
- assert_equal 0, @p.queue_size
472
- assert_equal 2, @p.stage.size
473
- assert_equal({}, @p.dequeued)
474
-
475
- assert{ qchunks.all?{ |c| c.purged } }
476
- end
477
-
478
- test '#write returns immediately if argument data is empty array' do
479
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
480
- assert_equal [@dm2,@dm3], @p.stage.keys
481
-
482
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
483
-
484
- @p.write({m => []})
485
-
486
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
487
- assert_equal [@dm2,@dm3], @p.stage.keys
488
- end
489
-
490
- test '#write returns immediately if argument data is empty event stream' do
491
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
492
- assert_equal [@dm2,@dm3], @p.stage.keys
493
-
494
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
495
-
496
- @p.write({m => Fluent::ArrayEventStream.new([])})
497
-
498
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
499
- assert_equal [@dm2,@dm3], @p.stage.keys
500
- end
501
-
502
- test '#write raises BufferOverflowError if buffer is not storable' do
503
- @p.stage_size = 256 * 1024 * 1024
504
- @p.queue_size = 256 * 1024 * 1024
505
-
506
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
507
-
508
- assert_raise Fluent::Plugin::Buffer::BufferOverflowError do
509
- @p.write({m => ["x" * 256]})
510
- end
511
- end
512
-
513
- test '#write stores data into an existing chunk with metadata specified' do
514
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
515
- assert_equal [@dm2,@dm3], @p.stage.keys
516
-
517
- dm3data = @p.stage[@dm3].read.dup
518
- prev_stage_size = @p.stage_size
519
-
520
- assert_equal 1, @p.stage[@dm3].append_count
521
-
522
- @p.write({@dm3 => ["x" * 256, "y" * 256, "z" * 256]})
523
-
524
- assert_equal 2, @p.stage[@dm3].append_count
525
- assert_equal (dm3data + ("x" * 256) + ("y" * 256) + ("z" * 256)), @p.stage[@dm3].read
526
- assert_equal (prev_stage_size + 768), @p.stage_size
527
-
528
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
529
- assert_equal [@dm2,@dm3], @p.stage.keys
530
- end
531
-
532
- test '#write creates new chunk and store data into it if there are no chunks for specified metadata' do
533
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
534
- assert_equal [@dm2,@dm3], @p.stage.keys
535
-
536
- timekey = Time.parse('2016-04-11 16:40:00 +0000').to_i
537
- assert !@p.timekeys.include?(timekey)
538
-
539
- prev_stage_size = @p.stage_size
540
-
541
- m = @p.metadata(timekey: timekey)
542
-
543
- @p.write({m => ["x" * 256, "y" * 256, "z" * 256]})
544
-
545
- assert_equal 1, @p.stage[m].append_count
546
- assert_equal ("x" * 256 + "y" * 256 + "z" * 256), @p.stage[m].read
547
- assert_equal (prev_stage_size + 768), @p.stage_size
548
-
549
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
550
- assert_equal [@dm2,@dm3,m], @p.stage.keys
551
-
552
- @p.update_timekeys
553
-
554
- assert @p.timekeys.include?(timekey)
555
- end
556
-
557
- test '#write tries to enqueue and store data into a new chunk if existing chunk is full' do
558
- assert_equal 8 * 1024 * 1024, @p.chunk_limit_size
559
- assert_equal 0.95, @p.chunk_full_threshold
560
-
561
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
562
- assert_equal [@dm2,@dm3], @p.stage.keys
563
-
564
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
565
-
566
- row = "x" * 1024 * 1024
567
- small_row = "x" * 1024 * 512
568
- @p.write({m => [row] * 7 + [small_row]})
569
-
570
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
571
- assert_equal [@dm2,@dm3,m], @p.stage.keys
572
- assert_equal 1, @p.stage[m].append_count
573
-
574
- @p.write({m => [row]})
575
-
576
- assert_equal [@dm0,@dm1,@dm1,m], @p.queue.map(&:metadata)
577
- assert_equal [@dm2,@dm3,m], @p.stage.keys
578
- assert_equal 1, @p.stage[m].append_count
579
- assert_equal 1024*1024, @p.stage[m].bytesize
580
- assert_equal 3, @p.queue.last.append_count # 1 -> write (2) -> write_step_by_step (3)
581
- assert @p.queue.last.rollbacked
582
- end
583
-
584
- test '#write rollbacks if commit raises errors' do
585
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
586
- assert_equal [@dm2,@dm3], @p.stage.keys
587
-
588
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
589
-
590
- row = "x" * 1024
591
- @p.write({m => [row] * 8})
592
-
593
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
594
- assert_equal [@dm2,@dm3,m], @p.stage.keys
595
-
596
- target_chunk = @p.stage[m]
597
-
598
- assert_equal 1, target_chunk.append_count
599
- assert !target_chunk.rollbacked
600
-
601
- (class << target_chunk; self; end).module_eval do
602
- define_method(:commit){ raise "yay" }
603
- end
604
-
605
- assert_raise RuntimeError.new("yay") do
606
- @p.write({m => [row]})
607
- end
608
-
609
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
610
- assert_equal [@dm2,@dm3,m], @p.stage.keys
611
-
612
- assert_equal 2, target_chunk.append_count
613
- assert target_chunk.rollbacked
614
- assert_equal row * 8, target_chunk.read
615
- end
616
-
617
- test '#write w/ format raises BufferOverflowError if buffer is not storable' do
618
- @p.stage_size = 256 * 1024 * 1024
619
- @p.queue_size = 256 * 1024 * 1024
620
-
621
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
622
-
623
- es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:40:01 +0000'), {"message" => "xxxxxxxxxxxxxx"} ] ])
624
-
625
- assert_raise Fluent::Plugin::Buffer::BufferOverflowError do
626
- @p.write({m => es}, format: ->(e){e.to_msgpack_stream})
627
- end
628
- end
629
-
630
- test '#write w/ format stores data into an existing chunk with metadata specified' do
631
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
632
- assert_equal [@dm2,@dm3], @p.stage.keys
633
-
634
- dm3data = @p.stage[@dm3].read.dup
635
- prev_stage_size = @p.stage_size
636
-
637
- assert_equal 1, @p.stage[@dm3].append_count
638
-
639
- es = Fluent::ArrayEventStream.new(
640
- [
641
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 128}],
642
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "y" * 128}],
643
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "z" * 128}],
644
- ]
645
- )
646
-
647
- @p.write({@dm3 => es}, format: ->(e){e.to_msgpack_stream})
648
-
649
- assert_equal 2, @p.stage[@dm3].append_count
650
- assert_equal (dm3data + es.to_msgpack_stream), @p.stage[@dm3].read
651
- assert_equal (prev_stage_size + es.to_msgpack_stream.bytesize), @p.stage_size
652
-
653
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
654
- assert_equal [@dm2,@dm3], @p.stage.keys
655
- end
656
-
657
- test '#write w/ format creates new chunk and store data into it if there are not chunks for specified metadata' do
658
- assert_equal 8 * 1024 * 1024, @p.chunk_limit_size
659
-
660
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
661
- assert_equal [@dm2,@dm3], @p.stage.keys
662
-
663
- timekey = Time.parse('2016-04-11 16:40:00 +0000').to_i
664
- assert !@p.timekeys.include?(timekey)
665
-
666
- m = @p.metadata(timekey: timekey)
667
-
668
- es = Fluent::ArrayEventStream.new(
669
- [
670
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
671
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
672
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
673
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
674
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
675
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
676
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
677
- [event_time('2016-04-11 16:40:03 +0000'), {"message" => "z" * 1024 * 512}],
678
- ]
679
- )
680
- @p.write({m => es}, format: ->(e){e.to_msgpack_stream})
681
-
682
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
683
- assert_equal [@dm2,@dm3,m], @p.stage.keys
684
- assert_equal 1, @p.stage[m].append_count
685
-
686
- @p.update_timekeys
687
-
688
- assert @p.timekeys.include?(timekey)
689
- end
690
-
691
- test '#write w/ format tries to enqueue and store data into a new chunk if existing chunk does not have enough space' do
692
- assert_equal 8 * 1024 * 1024, @p.chunk_limit_size
693
-
694
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
695
- assert_equal [@dm2,@dm3], @p.stage.keys
696
-
697
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
698
-
699
- es = Fluent::ArrayEventStream.new(
700
- [
701
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
702
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
703
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
704
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
705
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
706
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
707
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
708
- [event_time('2016-04-11 16:40:03 +0000'), {"message" => "z" * 1024 * 512}],
709
- ]
710
- )
711
- @p.write({m => es}, format: ->(e){e.to_msgpack_stream})
712
-
713
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
714
- assert_equal [@dm2,@dm3,m], @p.stage.keys
715
- assert_equal 1, @p.stage[m].append_count
716
-
717
- es2 = Fluent::OneEventStream.new(event_time('2016-04-11 16:40:03 +0000'), {"message" => "z" * 1024 * 1024})
718
- @p.write({m => es2}, format: ->(e){e.to_msgpack_stream})
719
-
720
- assert_equal [@dm0,@dm1,@dm1,m], @p.queue.map(&:metadata)
721
- assert_equal [@dm2,@dm3,m], @p.stage.keys
722
- assert_equal 1, @p.stage[m].append_count
723
- assert_equal es2.to_msgpack_stream.bytesize, @p.stage[m].bytesize
724
- assert_equal 2, @p.queue.last.append_count # 1 -> write (2) -> rollback&enqueue
725
- assert @p.queue.last.rollbacked
726
- end
727
-
728
- test '#write w/ format enqueues chunk if it is already full after adding data' do
729
- assert_equal 8 * 1024 * 1024, @p.chunk_limit_size
730
-
731
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
732
- assert_equal [@dm2,@dm3], @p.stage.keys
733
-
734
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
735
- es = Fluent::ArrayEventStream.new(
736
- [
737
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}], # 1024 * 1024 bytes as msgpack stream
738
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}],
739
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}],
740
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}],
741
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}],
742
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}],
743
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}],
744
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * (1024 * 1024 - 25)}],
745
- ]
746
- )
747
- @p.write({m => es}, format: ->(e){e.to_msgpack_stream})
748
-
749
- assert_equal [@dm0,@dm1,@dm1,m], @p.queue.map(&:metadata)
750
- assert_equal [@dm2,@dm3], @p.stage.keys
751
- assert_equal 1, @p.queue.last.append_count
752
- end
753
-
754
- test '#write w/ format rollbacks if commit raises errors' do
755
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
756
- assert_equal [@dm2,@dm3], @p.stage.keys
757
-
758
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
759
-
760
- es = Fluent::ArrayEventStream.new(
761
- [
762
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
763
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
764
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
765
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
766
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
767
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
768
- [event_time('2016-04-11 16:40:01 +0000'), {"message" => "x" * 1024 * 1024}],
769
- [event_time('2016-04-11 16:40:03 +0000'), {"message" => "z" * 1024 * 512}],
770
- ]
771
- )
772
- @p.write({m => es}, format: ->(e){e.to_msgpack_stream})
773
-
774
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
775
- assert_equal [@dm2,@dm3,m], @p.stage.keys
776
-
777
- target_chunk = @p.stage[m]
778
-
779
- assert_equal 1, target_chunk.append_count
780
- assert !target_chunk.rollbacked
781
-
782
- (class << target_chunk; self; end).module_eval do
783
- define_method(:commit){ raise "yay" }
784
- end
785
-
786
- es2 = Fluent::ArrayEventStream.new(
787
- [
788
- [event_time('2016-04-11 16:40:04 +0000'), {"message" => "z" * 1024 * 128}],
789
- ]
790
- )
791
- assert_raise RuntimeError.new("yay") do
792
- @p.write({m => es2}, format: ->(e){e.to_msgpack_stream})
793
- end
794
-
795
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
796
- assert_equal [@dm2,@dm3,m], @p.stage.keys
797
-
798
- assert_equal 2, target_chunk.append_count
799
- assert target_chunk.rollbacked
800
- assert_equal es.to_msgpack_stream, target_chunk.read
801
- end
802
-
803
- test '#write writes many metadata and data pairs at once' do
804
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
805
- assert_equal [@dm2,@dm3], @p.stage.keys
806
-
807
- row = "x" * 1024
808
- @p.write({ @dm0 => [row, row, row], @dm1 => [row, row] })
809
-
810
- assert_equal [@dm2,@dm3,@dm0,@dm1], @p.stage.keys
811
- end
812
-
813
- test '#write does not commit on any chunks if any append operation on chunk fails' do
814
- assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
815
- assert_equal [@dm2,@dm3], @p.stage.keys
816
-
817
- row = "x" * 1024
818
- @p.write({ @dm0 => [row, row, row], @dm1 => [row, row] })
819
-
820
- assert_equal [@dm2,@dm3,@dm0,@dm1], @p.stage.keys
821
-
822
- dm2_size = @p.stage[@dm2].size
823
- assert !@p.stage[@dm2].rollbacked
824
- dm3_size = @p.stage[@dm3].size
825
- assert !@p.stage[@dm3].rollbacked
826
-
827
- assert{ @p.stage[@dm0].size == 3 }
828
- assert !@p.stage[@dm0].rollbacked
829
- assert{ @p.stage[@dm1].size == 2 }
830
- assert !@p.stage[@dm1].rollbacked
831
-
832
- meta_list = [@dm0, @dm1, @dm2, @dm3].sort
833
- @p.stage[meta_list.last].failing = true
834
-
835
- assert_raise(FluentPluginBufferTest::DummyMemoryChunkError) do
836
- @p.write({ @dm2 => [row], @dm3 => [row], @dm0 => [row, row, row], @dm1 => [row, row] })
837
- end
838
-
839
- assert{ @p.stage[@dm2].size == dm2_size }
840
- assert @p.stage[@dm2].rollbacked
841
- assert{ @p.stage[@dm3].size == dm3_size }
842
- assert @p.stage[@dm3].rollbacked
843
-
844
- assert{ @p.stage[@dm0].size == 3 }
845
- assert @p.stage[@dm0].rollbacked
846
- assert{ @p.stage[@dm1].size == 2 }
847
- assert @p.stage[@dm1].rollbacked
848
- end
849
-
850
- test '#compress returns :text' do
851
- assert_equal :text, @p.compress
852
- end
853
-
854
- # https://github.com/fluent/fluentd/issues/3089
855
- test "closed chunk should not be committed" do
856
- assert_equal 8 * 1024 * 1024, @p.chunk_limit_size
857
- assert_equal 0.95, @p.chunk_full_threshold
858
-
859
- purge_count = 0
860
-
861
- stub.proxy(@p).generate_chunk(anything) do |chunk|
862
- stub.proxy(chunk).purge do |result|
863
- purge_count += 1
864
- result
865
- end
866
- stub.proxy(chunk).commit do |result|
867
- assert_false(chunk.closed?)
868
- result
869
- end
870
- stub.proxy(chunk).rollback do |result|
871
- assert_false(chunk.closed?)
872
- result
873
- end
874
- chunk
875
- end
876
-
877
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
878
- small_row = "x" * 1024 * 400
879
- big_row = "x" * 1024 * 1024 * 8 # just `chunk_size_limit`, it does't cause BufferOverFlowError.
880
-
881
- # Write 42 events in 1 event stream, last one is for triggering `ShouldRetry`
882
- @p.write({m => [small_row] * 40 + [big_row] + ["x"]})
883
-
884
- # Above event strem will be splitted twice by `Buffer#write_step_by_step`
885
- #
886
- # 1. `write_once`: 42 [events] * 1 [stream]
887
- # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream]
888
- # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams]
889
- #
890
- # The problematic data is built in the 2nd stage.
891
- # In the 2nd stage, 5 streams are packed in a chunk.
892
- # ((1024 * 400) [bytes] * 4 [events] * 5 [streams] = 8192000 [bytes] < `chunk_limit_size` (8MB)).
893
- # So 3 chunks are used to store all data.
894
- # The 1st chunk is already staged by `write_once`.
895
- # The 2nd & 3rd chunks are newly created as unstaged.
896
- # The 3rd chunk is purged before `ShouldRetry`, it's no problem:
897
- # https://github.com/fluent/fluentd/blob/7e9eba736ff40ad985341be800ddc46558be75f2/lib/fluent/plugin/buffer.rb#L850
898
- # The 2nd chunk is purged in `rescue ShouldRetry`:
899
- # https://github.com/fluent/fluentd/blob/7e9eba736ff40ad985341be800ddc46558be75f2/lib/fluent/plugin/buffer.rb#L862
900
- # It causes the issue described in https://github.com/fluent/fluentd/issues/3089#issuecomment-1811839198
901
-
902
- assert_equal 2, purge_count
903
- end
904
-
905
- # https://github.com/fluent/fluentd/issues/4446
906
- test "#write_step_by_step keeps chunks kept in locked in entire #write process" do
907
- assert_equal 8 * 1024 * 1024, @p.chunk_limit_size
908
- assert_equal 0.95, @p.chunk_full_threshold
909
-
910
- mon_enter_counts_by_chunk = {}
911
- mon_exit_counts_by_chunk = {}
912
-
913
- stub.proxy(@p).generate_chunk(anything) do |chunk|
914
- stub(chunk).mon_enter do
915
- enter_count = 1 + mon_enter_counts_by_chunk.fetch(chunk, 0)
916
- exit_count = mon_exit_counts_by_chunk.fetch(chunk, 0)
917
- mon_enter_counts_by_chunk[chunk] = enter_count
918
-
919
- # Assert that chunk is passed to &block of write_step_by_step before exiting the lock.
920
- # (i.e. The lock count must be 2 greater than the exit count).
921
- # Since ShouldRetry occurs once, the staged chunk takes the lock 3 times when calling the block.
922
- if chunk.staged?
923
- lock_in_block = enter_count == 3
924
- assert_equal(enter_count - 2, exit_count) if lock_in_block
925
- else
926
- lock_in_block = enter_count == 2
927
- assert_equal(enter_count - 2, exit_count) if lock_in_block
928
- end
929
- end
930
- stub(chunk).mon_exit do
931
- exit_count = 1 + mon_exit_counts_by_chunk.fetch(chunk, 0)
932
- mon_exit_counts_by_chunk[chunk] = exit_count
933
- end
934
- chunk
935
- end
936
-
937
- m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
938
- small_row = "x" * 1024 * 400
939
- big_row = "x" * 1024 * 1024 * 8 # just `chunk_size_limit`, it does't cause BufferOverFlowError.
940
-
941
- # Write 42 events in 1 event stream, last one is for triggering `ShouldRetry`
942
- @p.write({m => [small_row] * 40 + [big_row] + ["x"]})
943
-
944
- # Above event strem will be splitted twice by `Buffer#write_step_by_step`
945
- #
946
- # 1. `write_once`: 42 [events] * 1 [stream]
947
- # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream]
948
- # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams]
949
- #
950
- # Example of staged chunk lock behavior:
951
- #
952
- # 1. mon_enter in write_step_by_step
953
- # 2. ShouldRetry occurs
954
- # 3. mon_exit in write_step_by_step
955
- # 4. mon_enter again in write_step_by_step (retry)
956
- # 5. passed to &block of write_step_by_step
957
- # 6. mon_enter in the block (write)
958
- # 7. mon_exit in write_step_by_step
959
- # 8. mon_exit in write
960
-
961
- assert_equal(mon_enter_counts_by_chunk.values, mon_exit_counts_by_chunk.values)
962
- end
963
- end
964
-
965
- sub_test_case 'standard format with configuration for test with lower chunk limit size' do
966
- setup do
967
- @p = create_buffer({"chunk_limit_size" => 1_280_000})
968
- @format = ->(e){e.to_msgpack_stream}
969
- @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
970
- # 1 record is 128bytes in msgpack stream
971
- @es0 = es0 = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:01 +0000'), {"message" => "x" * (128 - 22)}] ] * 5000)
972
- (class << @p; self; end).module_eval do
973
- define_method(:resume) {
974
- staged = {
975
- dm0 => create_chunk_es(dm0, es0).staged!,
976
- }
977
- queued = []
978
- return staged, queued
979
- }
980
- end
981
- @p.start
982
- end
983
-
984
- test '#write appends event stream into staged chunk' do
985
- assert_equal [@dm0], @p.stage.keys
986
- assert_equal [], @p.queue.map(&:metadata)
987
-
988
- assert_equal 1_280_000, @p.chunk_limit_size
989
-
990
- es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * (128 - 22)}] ] * 1000)
991
- @p.write({@dm0 => es}, format: @format)
992
-
993
- assert_equal [@dm0], @p.stage.keys
994
- assert_equal [], @p.queue.map(&:metadata)
995
-
996
- assert_equal (@es0.to_msgpack_stream + es.to_msgpack_stream), @p.stage[@dm0].read
997
- end
998
-
999
- test '#write writes event stream into a new chunk with enqueueing existing chunk if event stream is larger than available space of existing chunk' do
1000
- assert_equal [@dm0], @p.stage.keys
1001
- assert_equal [], @p.queue.map(&:metadata)
1002
-
1003
- assert_equal 1_280_000, @p.chunk_limit_size
1004
-
1005
- es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * (128 - 22)}] ] * 8000)
1006
- @p.write({@dm0 => es}, format: @format)
1007
-
1008
- assert_equal [@dm0], @p.stage.keys
1009
- assert_equal [@dm0], @p.queue.map(&:metadata)
1010
-
1011
- assert_equal (es.to_msgpack_stream), @p.stage[@dm0].read
1012
- end
1013
-
1014
- test '#write writes event stream into many chunks excluding staged chunk if event stream is larger than chunk limit size' do
1015
- assert_equal [@dm0], @p.stage.keys
1016
- assert_equal [], @p.queue.map(&:metadata)
1017
-
1018
- assert_equal 1_280_000, @p.chunk_limit_size
1019
-
1020
- es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * (128 - 22)}] ] * 45000)
1021
- @p.write({@dm0 => es}, format: @format)
1022
-
1023
- # metadata whose seq is 4 is created, but overwrite with original metadata(seq=0) for next use of this chunk https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L357
1024
- assert_equal [@dm0], @p.stage.keys
1025
- assert_equal 5400, @p.stage[@dm0].size
1026
- assert_equal [@dm0, @dm0, @dm0, @dm0, @dm0], @p.queue.map(&:metadata)
1027
- assert_equal [5000, 9900, 9900, 9900, 9900], @p.queue.map(&:size) # splits: 45000 / 100 => 450 * ...
1028
- # 9900 * 4 + 5400 == 45000
1029
- end
1030
-
1031
- test '#dequeue_chunk succeeds when chunk is splited' do
1032
- assert_equal [@dm0], @p.stage.keys
1033
- assert_equal [], @p.queue.map(&:metadata)
1034
-
1035
- assert_equal 1_280_000, @p.chunk_limit_size
1036
-
1037
- es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * (128 - 22)}] ] * 45000)
1038
- @p.write({@dm0 => es}, format: @format)
1039
- @p.enqueue_all(true)
1040
-
1041
- dequeued_chunks = 6.times.map { |e| @p.dequeue_chunk } # splits: 45000 / 100 => 450 * ...
1042
- assert_equal [5000, 9900, 9900, 9900, 9900, 5400], dequeued_chunks.map(&:size)
1043
- assert_equal [@dm0, @dm0, @dm0, @dm0, @dm0, @dm0], dequeued_chunks.map(&:metadata)
1044
- end
1045
-
1046
- test '#write raises BufferChunkOverflowError if a record is biggar than chunk limit size' do
1047
- assert_equal [@dm0], @p.stage.keys
1048
- assert_equal [], @p.queue.map(&:metadata)
1049
-
1050
- assert_equal 1_280_000, @p.chunk_limit_size
1051
-
1052
- es = Fluent::ArrayEventStream.new([ [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}] ])
1053
- assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError do
1054
- @p.write({@dm0 => es}, format: @format)
1055
- end
1056
- end
1057
-
1058
- data(
1059
- first_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}],
1060
- [event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}],
1061
- [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}]]),
1062
- intermediate_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}],
1063
- [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}],
1064
- [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}]]),
1065
- last_chunk: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}],
1066
- [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}],
1067
- [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}]]),
1068
- multiple_chunks: Fluent::ArrayEventStream.new([[event_time('2016-04-11 16:00:02 +0000'), {"message" => "a"}],
1069
- [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}],
1070
- [event_time('2016-04-11 16:00:02 +0000'), {"message" => "b"}],
1071
- [event_time('2016-04-11 16:00:02 +0000'), {"message" => "x" * 1_280_000}]])
1072
- )
1073
- test '#write exceeds chunk_limit_size, raise BufferChunkOverflowError, but not lost whole messages' do |(es)|
1074
- assert_equal [@dm0], @p.stage.keys
1075
- assert_equal [], @p.queue.map(&:metadata)
1076
-
1077
- assert_equal 1_280_000, @p.chunk_limit_size
1078
-
1079
- nth = []
1080
- es.entries.each_with_index do |entry, index|
1081
- if entry.last["message"].size == @p.chunk_limit_size
1082
- nth << index
1083
- end
1084
- end
1085
- messages = []
1086
- nth.each do |n|
1087
- messages << "a 1280025 bytes record (nth: #{n}) is larger than buffer chunk limit size (1280000)"
1088
- end
1089
-
1090
- assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError.new(messages.join(", ")) do
1091
- @p.write({@dm0 => es}, format: @format)
1092
- end
1093
- # message a and b are concatenated and staged
1094
- staged_messages = Fluent::MessagePackFactory.msgpack_unpacker.feed_each(@p.stage[@dm0].chunk).collect do |record|
1095
- record.last
1096
- end
1097
- assert_equal([2, [{"message" => "a"}, {"message" => "b"}]],
1098
- [@p.stage[@dm0].size, staged_messages])
1099
- # only es0 message is queued
1100
- assert_equal [@dm0], @p.queue.map(&:metadata)
1101
- assert_equal [5000], @p.queue.map(&:size)
1102
- end
1103
-
1104
- test "confirm that every message which is smaller than chunk threshold does not raise BufferChunkOverflowError" do
1105
- assert_equal [@dm0], @p.stage.keys
1106
- assert_equal [], @p.queue.map(&:metadata)
1107
- timestamp = event_time('2016-04-11 16:00:02 +0000')
1108
- es = Fluent::ArrayEventStream.new([[timestamp, {"message" => "a" * 1_000_000}],
1109
- [timestamp, {"message" => "b" * 1_000_000}],
1110
- [timestamp, {"message" => "c" * 1_000_000}]])
1111
-
1112
- # https://github.com/fluent/fluentd/issues/1849
1113
- # Even though 1_000_000 < 1_280_000 (chunk_limit_size), it raised BufferChunkOverflowError before.
1114
- # It should not be raised and message a,b,c should be stored into 3 chunks.
1115
- assert_nothing_raised do
1116
- @p.write({@dm0 => es}, format: @format)
1117
- end
1118
- messages = []
1119
- # pick up first letter to check whether chunk is queued in expected order
1120
- 3.times do |index|
1121
- chunk = @p.queue[index]
1122
- es = Fluent::MessagePackEventStream.new(chunk.chunk)
1123
- es.ensure_unpacked!
1124
- records = es.instance_eval{ @unpacked_records }
1125
- records.each do |record|
1126
- messages << record["message"][0]
1127
- end
1128
- end
1129
- es = Fluent::MessagePackEventStream.new(@p.stage[@dm0].chunk)
1130
- es.ensure_unpacked!
1131
- staged_message = es.instance_eval{ @unpacked_records }.first["message"]
1132
- # message a and b are queued, message c is staged
1133
- assert_equal([
1134
- [@dm0],
1135
- "c" * 1_000_000,
1136
- [@dm0, @dm0, @dm0],
1137
- [5000, 1, 1],
1138
- [["x"] * 5000, "a", "b"].flatten
1139
- ],
1140
- [
1141
- @p.stage.keys,
1142
- staged_message,
1143
- @p.queue.map(&:metadata),
1144
- @p.queue.map(&:size),
1145
- messages
1146
- ])
1147
- end
1148
- end
1149
-
1150
- sub_test_case 'custom format with configuration for test with lower chunk limit size' do
1151
- setup do
1152
- @p = create_buffer({"chunk_limit_size" => 1_280_000})
1153
- @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
1154
- @row = "x" * 128
1155
- @data0 = data0 = [@row] * 5000
1156
- (class << @p; self; end).module_eval do
1157
- define_method(:resume) {
1158
- staged = {
1159
- dm0 => create_chunk(dm0, data0).staged!,
1160
- }
1161
- queued = []
1162
- return staged, queued
1163
- }
1164
- end
1165
- @p.start
1166
- end
1167
-
1168
- test '#write appends event stream into staged chunk' do
1169
- assert_equal [@dm0], @p.stage.keys
1170
- assert_equal [], @p.queue.map(&:metadata)
1171
-
1172
- assert_equal 1_280_000, @p.chunk_limit_size
1173
-
1174
- data = [@row] * 1000
1175
- @p.write({@dm0 => data})
1176
-
1177
- assert_equal [@dm0], @p.stage.keys
1178
- assert_equal [], @p.queue.map(&:metadata)
1179
-
1180
- assert_equal (@row * 6000), @p.stage[@dm0].read
1181
- end
1182
-
1183
- test '#write writes event stream into a new chunk with enqueueing existing chunk if event stream is larger than available space of existing chunk' do
1184
- assert_equal [@dm0], @p.stage.keys
1185
- assert_equal [], @p.queue.map(&:metadata)
1186
-
1187
- staged_chunk_object_id = @p.stage[@dm0].object_id
1188
-
1189
- assert_equal 1_280_000, @p.chunk_limit_size
1190
-
1191
- data = [@row] * 8000
1192
- @p.write({@dm0 => data})
1193
-
1194
- assert_equal [@dm0], @p.queue.map(&:metadata)
1195
- assert_equal [staged_chunk_object_id], @p.queue.map(&:object_id)
1196
- assert_equal [@dm0], @p.stage.keys
1197
-
1198
- assert_equal [9800], @p.queue.map(&:size)
1199
- assert_equal 3200, @p.stage[@dm0].size
1200
- # 9800 + 3200 == 5000 + 8000
1201
- end
1202
-
1203
- test '#write writes event stream into many chunks including staging chunk if event stream is larger than chunk limit size' do
1204
- assert_equal [@dm0], @p.stage.keys
1205
- assert_equal [], @p.queue.map(&:metadata)
1206
-
1207
- staged_chunk_object_id = @p.stage[@dm0].object_id
1208
-
1209
- assert_equal 1_280_000, @p.chunk_limit_size
1210
-
1211
- assert_equal 5000, @p.stage[@dm0].size
1212
-
1213
- data = [@row] * 45000
1214
- @p.write({@dm0 => data})
1215
-
1216
- assert_equal staged_chunk_object_id, @p.queue.first.object_id
1217
-
1218
- assert_equal [@dm0], @p.stage.keys
1219
- assert_equal 900, @p.stage[@dm0].size
1220
- assert_equal [@dm0, @dm0, @dm0, @dm0, @dm0], @p.queue.map(&:metadata)
1221
- assert_equal [9500, 9900, 9900, 9900, 9900], @p.queue.map(&:size) # splits: 45000 / 100 => 450 * ...
1222
- ##### 900 + 9500 + 9900 * 4 == 5000 + 45000
1223
- end
1224
-
1225
- test '#write raises BufferChunkOverflowError if a record is bigger than chunk limit size' do
1226
- assert_equal [@dm0], @p.stage.keys
1227
- assert_equal [], @p.queue.map(&:metadata)
1228
-
1229
- assert_equal 1_280_000, @p.chunk_limit_size
1230
-
1231
- es = ["x" * 1_280_000 + "x" * 300]
1232
- assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError do
1233
- @p.write({@dm0 => es})
1234
- end
1235
- end
1236
-
1237
- test 'confirm that every array message which is smaller than chunk threshold does not raise BufferChunkOverflowError' do
1238
- assert_equal [@dm0], @p.stage.keys
1239
- assert_equal [], @p.queue.map(&:metadata)
1240
-
1241
- assert_equal 1_280_000, @p.chunk_limit_size
1242
-
1243
- es = ["a" * 1_000_000, "b" * 1_000_000, "c" * 1_000_000]
1244
- assert_nothing_raised do
1245
- @p.write({@dm0 => es})
1246
- end
1247
- queue_messages = @p.queue.collect do |chunk|
1248
- # collect first character of each message
1249
- chunk.chunk[0]
1250
- end
1251
- assert_equal([
1252
- [@dm0],
1253
- 1,
1254
- "c",
1255
- [@dm0, @dm0, @dm0],
1256
- [5000, 1, 1],
1257
- ["x", "a", "b"]
1258
- ],
1259
- [
1260
- @p.stage.keys,
1261
- @p.stage[@dm0].size,
1262
- @p.stage[@dm0].chunk[0],
1263
- @p.queue.map(&:metadata),
1264
- @p.queue.map(&:size),
1265
- queue_messages
1266
- ])
1267
- end
1268
- end
1269
-
1270
- sub_test_case 'with configuration for test with lower limits' do
1271
- setup do
1272
- @p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240})
1273
- @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
1274
- @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)
1275
- @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)
1276
- @dm3 = dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil)
1277
- (class << @p; self; end).module_eval do
1278
- define_method(:resume) {
1279
- staged = {
1280
- dm2 => create_chunk(dm2, ["b" * 128] * 7).staged!,
1281
- dm3 => create_chunk(dm3, ["c" * 128] * 5).staged!,
1282
- }
1283
- queued = [
1284
- create_chunk(dm0, ["0" * 128] * 8).enqueued!,
1285
- create_chunk(dm0, ["0" * 128] * 8).enqueued!,
1286
- create_chunk(dm0, ["0" * 128] * 8).enqueued!,
1287
- create_chunk(dm0, ["0" * 128] * 8).enqueued!,
1288
- create_chunk(dm0, ["0" * 128] * 8).enqueued!,
1289
- create_chunk(dm1, ["a" * 128] * 8).enqueued!,
1290
- create_chunk(dm1, ["a" * 128] * 8).enqueued!,
1291
- create_chunk(dm1, ["a" * 128] * 8).enqueued!, # 8th queued chunk
1292
- create_chunk(dm1, ["a" * 128] * 3).enqueued!,
1293
- ]
1294
- return staged, queued
1295
- }
1296
- end
1297
- @p.start
1298
- end
1299
-
1300
- test '#storable? returns false when too many data exist' do
1301
- assert_equal [@dm0,@dm0,@dm0,@dm0,@dm0,@dm1,@dm1,@dm1,@dm1], @p.queue.map(&:metadata)
1302
- assert_equal [@dm2,@dm3], @p.stage.keys
1303
-
1304
- assert_equal 128*8*8+128*3, @p.queue_size
1305
- assert_equal 128*7+128*5, @p.stage_size
1306
-
1307
- assert @p.storable?
1308
-
1309
- dm3 = @p.metadata(timekey: @dm3.timekey)
1310
- @p.write({dm3 => ["c" * 128]})
1311
-
1312
- assert_equal 10240, (@p.stage_size + @p.queue_size)
1313
- assert !@p.storable?
1314
- end
1315
-
1316
- test '#chunk_size_over? returns true if chunk size is bigger than limit' do
1317
- m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i)
1318
-
1319
- c1 = create_chunk(m, ["a" * 128] * 8)
1320
- assert !@p.chunk_size_over?(c1)
1321
-
1322
- c2 = create_chunk(m, ["a" * 128] * 9)
1323
- assert @p.chunk_size_over?(c2)
1324
-
1325
- c3 = create_chunk(m, ["a" * 128] * 8 + ["a"])
1326
- assert @p.chunk_size_over?(c3)
1327
- end
1328
-
1329
- test '#chunk_size_full? returns true if chunk size is enough big against limit' do
1330
- m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i)
1331
-
1332
- c1 = create_chunk(m, ["a" * 128] * 7)
1333
- assert !@p.chunk_size_full?(c1)
1334
-
1335
- c2 = create_chunk(m, ["a" * 128] * 8)
1336
- assert @p.chunk_size_full?(c2)
1337
-
1338
- assert_equal 0.95, @p.chunk_full_threshold
1339
- c3 = create_chunk(m, ["a" * 128] * 6 + ["a" * 64])
1340
- assert !@p.chunk_size_full?(c3)
1341
- end
1342
- end
1343
-
1344
- sub_test_case 'with configuration includes chunk_limit_records' do
1345
- setup do
1346
- @p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240, "chunk_limit_records" => 6})
1347
- @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
1348
- @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)
1349
- @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)
1350
- @dm3 = dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil)
1351
- (class << @p; self; end).module_eval do
1352
- define_method(:resume) {
1353
- staged = {
1354
- dm2 => create_chunk(dm2, ["b" * 128] * 1).staged!,
1355
- dm3 => create_chunk(dm3, ["c" * 128] * 2).staged!,
1356
- }
1357
- queued = [
1358
- create_chunk(dm0, ["0" * 128] * 6).enqueued!,
1359
- create_chunk(dm1, ["a" * 128] * 6).enqueued!,
1360
- create_chunk(dm1, ["a" * 128] * 6).enqueued!,
1361
- create_chunk(dm1, ["a" * 128] * 3).enqueued!,
1362
- ]
1363
- return staged, queued
1364
- }
1365
- end
1366
- @p.start
1367
- end
1368
-
1369
- test '#chunk_size_over? returns true if too many records exists in a chunk even if its bytes is less than limit' do
1370
- assert_equal 6, @p.chunk_limit_records
1371
-
1372
- m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i)
1373
-
1374
- c1 = create_chunk(m, ["a" * 128] * 6)
1375
- assert_equal 6, c1.size
1376
- assert !@p.chunk_size_over?(c1)
1377
-
1378
- c2 = create_chunk(m, ["a" * 128] * 7)
1379
- assert @p.chunk_size_over?(c2)
1380
-
1381
- c3 = create_chunk(m, ["a" * 128] * 6 + ["a"])
1382
- assert @p.chunk_size_over?(c3)
1383
- end
1384
-
1385
- test '#chunk_size_full? returns true if enough many records exists in a chunk even if its bytes is less than limit' do
1386
- assert_equal 6, @p.chunk_limit_records
1387
-
1388
- m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i)
1389
-
1390
- c1 = create_chunk(m, ["a" * 128] * 5)
1391
- assert_equal 5, c1.size
1392
- assert !@p.chunk_size_full?(c1)
1393
-
1394
- c2 = create_chunk(m, ["a" * 128] * 6)
1395
- assert @p.chunk_size_full?(c2)
1396
-
1397
- c3 = create_chunk(m, ["a" * 128] * 5 + ["a"])
1398
- assert @p.chunk_size_full?(c3)
1399
- end
1400
- end
1401
-
1402
- sub_test_case 'with configuration includes queue_limit_length' do
1403
- setup do
1404
- @p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240, "queue_limit_length" => 5})
1405
- @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
1406
- @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)
1407
- @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)
1408
- @dm3 = dm3 = create_metadata(Time.parse('2016-04-11 16:30:00 +0000').to_i, nil, nil)
1409
- (class << @p; self; end).module_eval do
1410
- define_method(:resume) {
1411
- staged = {
1412
- dm2 => create_chunk(dm2, ["b" * 128] * 1).staged!,
1413
- dm3 => create_chunk(dm3, ["c" * 128] * 2).staged!,
1414
- }
1415
- queued = [
1416
- create_chunk(dm0, ["0" * 128] * 6).enqueued!,
1417
- create_chunk(dm1, ["a" * 128] * 6).enqueued!,
1418
- create_chunk(dm1, ["a" * 128] * 6).enqueued!,
1419
- create_chunk(dm1, ["a" * 128] * 3).enqueued!,
1420
- ]
1421
- return staged, queued
1422
- }
1423
- end
1424
- @p.start
1425
- end
1426
-
1427
- test '#configure will overwrite standard configuration if queue_limit_length' do
1428
- assert_equal 1024, @p.chunk_limit_size
1429
- assert_equal 5, @p.queue_limit_length
1430
- assert_equal (1024*5), @p.total_limit_size
1431
- end
1432
- end
1433
-
1434
- sub_test_case 'when compress is gzip' do
1435
- setup do
1436
- @p = create_buffer({'compress' => 'gzip'})
1437
- @dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
1438
- end
1439
-
1440
- test '#compress returns :gzip' do
1441
- assert_equal :gzip, @p.compress
1442
- end
1443
-
1444
- test 'create decompressable chunk' do
1445
- chunk = @p.generate_chunk(create_metadata)
1446
- assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::Decompressable)
1447
- end
1448
-
1449
- test '#write compressed data which exceeds chunk_limit_size, it raises BufferChunkOverflowError' do
1450
- @p = create_buffer({'compress' => 'gzip', 'chunk_limit_size' => 70})
1451
- timestamp = event_time('2016-04-11 16:00:02 +0000')
1452
- es = Fluent::ArrayEventStream.new([[timestamp, {"message" => "012345"}], # overflow
1453
- [timestamp, {"message" => "aaa"}],
1454
- [timestamp, {"message" => "bbb"}]])
1455
- assert_equal [], @p.queue.map(&:metadata)
1456
- assert_equal 70, @p.chunk_limit_size
1457
-
1458
- # calculate the actual boundary value. it varies on machine
1459
- c = @p.generate_chunk(create_metadata)
1460
- c.append(Fluent::ArrayEventStream.new([[timestamp, {"message" => "012345"}]]), compress: :gzip)
1461
- overflow_bytes = c.bytesize
1462
-
1463
- messages = "concatenated/appended a #{overflow_bytes} bytes record (nth: 0) is larger than buffer chunk limit size (70)"
1464
- assert_raise Fluent::Plugin::Buffer::BufferChunkOverflowError.new(messages) do
1465
- # test format == nil && compress == :gzip
1466
- @p.write({@dm0 => es})
1467
- end
1468
- # message a and b occupies each chunks in full, so both of messages are queued (no staged chunk)
1469
- assert_equal([2, [@dm0, @dm0], [1, 1], nil],
1470
- [@p.queue.size, @p.queue.map(&:metadata), @p.queue.map(&:size), @p.stage[@dm0]])
1471
- end
1472
- end
1473
-
1474
- sub_test_case '#statistics' do
1475
- setup do
1476
- @p = create_buffer({ "total_limit_size" => 1024 })
1477
- dm = create_metadata(Time.parse('2020-03-13 16:00:00 +0000').to_i, nil, nil)
1478
-
1479
- (class << @p; self; end).module_eval do
1480
- define_method(:resume) {
1481
- queued = [create_chunk(dm, ["a" * (1024 - 102)]).enqueued!]
1482
- return {}, queued
1483
- }
1484
- end
1485
-
1486
- @p.start
1487
- end
1488
-
1489
- test 'returns available_buffer_space_ratios' do
1490
- assert_equal 10.0, @p.statistics['buffer']['available_buffer_space_ratios']
1491
- end
1492
- end
1493
- end