fluentd 1.17.0 → 1.17.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
@@ -1,611 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/plugin/buffer/file_single_chunk'
3
- require 'fluent/plugin/compressable'
4
- require 'fluent/unique_id'
5
-
6
- require 'fileutils'
7
- require 'msgpack'
8
- require 'time'
9
-
10
- class BufferFileSingleChunkTest < Test::Unit::TestCase
11
- include Fluent::Plugin::Compressable
12
-
13
- setup do
14
- @klass = Fluent::Plugin::Buffer::FileSingleChunk
15
- @chunkdir = File.expand_path('../../tmp/buffer_file_single_chunk', __FILE__)
16
- FileUtils.rm_r(@chunkdir) rescue nil
17
- FileUtils.mkdir_p(@chunkdir)
18
- end
19
-
20
- Metadata = Struct.new(:timekey, :tag, :variables)
21
- def gen_metadata(timekey: nil, tag: 'testing', variables: nil)
22
- Metadata.new(timekey, tag, variables)
23
- end
24
-
25
- def gen_path(path)
26
- File.join(@chunkdir, path)
27
- end
28
-
29
- def gen_test_chunk_id
30
- now = Time.parse('2016-04-07 14:31:33 +0900')
31
- u1 = ((now.to_i * 1000 * 1000 + now.usec) << 12 | 1725) # 1725 is one of `rand(0xfff)`
32
- u3 = 2979763054 # one of rand(0xffffffff)
33
- u4 = 438020492 # ditto
34
- [u1 >> 32, u1 & 0xffffffff, u3, u4].pack('NNNN')
35
- # unique_id.unpack('N*').map{|n| n.to_s(16)}.join => "52fde6425d7406bdb19b936e1a1ba98c"
36
- end
37
-
38
- def hex_id(id)
39
- id.unpack('N*').map { |n| n.to_s(16) }.join
40
- end
41
-
42
- sub_test_case 'classmethods' do
43
- data(
44
- correct_staged: ['/mydir/mypath/fsb.b00ff.buf', :staged],
45
- correct_queued: ['/mydir/mypath/fsb.q00ff.buf', :queued],
46
- incorrect_staged: ['/mydir/mypath/fsb.b00ff.buf/unknown', :unknown],
47
- incorrect_queued: ['/mydir/mypath/fsb.q00ff.buf/unknown', :unknown],
48
- output_file: ['/mydir/mypath/fsb.20160716.buf', :unknown],
49
- )
50
- test 'can .assume_chunk_state' do |data|
51
- path, expected = data
52
- assert_equal expected, @klass.assume_chunk_state(path)
53
- end
54
-
55
- test '.generate_stage_chunk_path generates path with staged mark & chunk unique_id' do
56
- assert_equal gen_path("fsb.foo.b52fde6425d7406bdb19b936e1a1ba98c.buf"), @klass.generate_stage_chunk_path(gen_path("fsb.*.buf"), 'foo', gen_test_chunk_id)
57
- assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do
58
- @klass.generate_stage_chunk_path(gen_path("fsb.buf"), 'foo', gen_test_chunk_id)
59
- end
60
- assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do
61
- @klass.generate_stage_chunk_path(gen_path("fsb.*"), 'foo', gen_test_chunk_id)
62
- end
63
- assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do
64
- @klass.generate_stage_chunk_path(gen_path("*.buf"), 'foo', gen_test_chunk_id)
65
- end
66
- end
67
-
68
- test '.generate_queued_chunk_path generates path with enqueued mark for staged chunk path' do
69
- assert_equal(
70
- gen_path("fsb.q52fde6425d7406bdb19b936e1a1ba98c.buf"),
71
- @klass.generate_queued_chunk_path(gen_path("fsb.b52fde6425d7406bdb19b936e1a1ba98c.buf"), gen_test_chunk_id)
72
- )
73
- end
74
-
75
- test '.generate_queued_chunk_path generates special path with chunk unique_id for non staged chunk path' do
76
- assert_equal(
77
- gen_path("fsb.buf.q52fde6425d7406bdb19b936e1a1ba98c.chunk"),
78
- @klass.generate_queued_chunk_path(gen_path("fsb.buf"), gen_test_chunk_id)
79
- )
80
- assert_equal(
81
- gen_path("fsb.q55555555555555555555555555555555.buf.q52fde6425d7406bdb19b936e1a1ba98c.chunk"),
82
- @klass.generate_queued_chunk_path(gen_path("fsb.q55555555555555555555555555555555.buf"), gen_test_chunk_id)
83
- )
84
- end
85
-
86
- data('1 word tag' => 'foo',
87
- '2 words tag' => 'test.log',
88
- 'empty' => '')
89
- test '.unique_id_and_key_from_path recreates unique_id and key from file path' do |key|
90
- path = @klass.unique_id_and_key_from_path(gen_path("fsb.#{key}.q52fde6425d7406bdb19b936e1a1ba98c.buf"))
91
- assert_equal [gen_test_chunk_id, key], path
92
- end
93
- end
94
-
95
- sub_test_case 'newly created chunk' do
96
- setup do
97
- @path_conf = File.join(@chunkdir, 'fsb.*.buf')
98
- @chunk_path = File.join(@chunkdir, "fsb.testing.b#{hex_id(gen_test_chunk_id)}.buf")
99
- @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @path_conf, :create, nil)
100
- end
101
-
102
- def gen_chunk_path(prefix, unique_id)
103
- File.join(@chunkdir, "fsb.testing.#{prefix}#{Fluent::UniqueId.hex(unique_id)}.buf")
104
- end
105
-
106
- teardown do
107
- if @c
108
- @c.purge rescue nil
109
- end
110
- if File.exist?(@chunk_path)
111
- File.unlink(@chunk_path)
112
- end
113
- end
114
-
115
- test 'creates new files for chunk and metadata with specified path & permission' do
116
- assert_equal 16, @c.unique_id.size
117
- assert_equal gen_chunk_path('b', @c.unique_id), @c.path
118
-
119
- assert File.exist?(gen_chunk_path('b', @c.unique_id))
120
- assert { File.stat(gen_chunk_path('b', @c.unique_id)).mode.to_s(8).end_with?(Fluent::DEFAULT_FILE_PERMISSION.to_s(8)) }
121
-
122
- assert_equal :unstaged, @c.state
123
- assert @c.empty?
124
- end
125
-
126
- test 'can #append, #commit and #read it' do
127
- assert @c.empty?
128
-
129
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
130
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
131
- data = [d1.to_json + "\n", d2.to_json + "\n"]
132
- @c.append(data)
133
- @c.commit
134
-
135
- ds = @c.read.split("\n").select { |d| !d.empty? }
136
- assert_equal 2, ds.size
137
- assert_equal d1, JSON.parse(ds[0])
138
- assert_equal d2, JSON.parse(ds[1])
139
-
140
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
141
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
142
- @c.append([d3.to_json + "\n", d4.to_json + "\n"])
143
- @c.commit
144
-
145
- ds = @c.read.split("\n").select{|d| !d.empty? }
146
- assert_equal 4, ds.size
147
- assert_equal d1, JSON.parse(ds[0])
148
- assert_equal d2, JSON.parse(ds[1])
149
- assert_equal d3, JSON.parse(ds[2])
150
- assert_equal d4, JSON.parse(ds[3])
151
- end
152
-
153
- test 'can #concat, #commit and #read it' do
154
- assert @c.empty?
155
-
156
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
157
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
158
- data = [d1.to_json + "\n", d2.to_json + "\n"].join
159
- @c.concat(data, 2)
160
- @c.commit
161
-
162
- ds = @c.read.split("\n").select{|d| !d.empty? }
163
- assert_equal 2, ds.size
164
- assert_equal d1, JSON.parse(ds[0])
165
- assert_equal d2, JSON.parse(ds[1])
166
-
167
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
168
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
169
- @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2)
170
- @c.commit
171
-
172
- ds = @c.read.split("\n").select { |d| !d.empty? }
173
- assert_equal 4, ds.size
174
- assert_equal d1, JSON.parse(ds[0])
175
- assert_equal d2, JSON.parse(ds[1])
176
- assert_equal d3, JSON.parse(ds[2])
177
- assert_equal d4, JSON.parse(ds[3])
178
- end
179
-
180
- test 'has its contents in binary (ascii-8bit)' do
181
- data1 = "aaa bbb ccc".force_encoding('utf-8')
182
- @c.append([data1])
183
- @c.commit
184
- assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.external_encoding }
185
- assert_equal Encoding::ASCII_8BIT, @c.read.encoding
186
- end
187
-
188
- test 'has #bytesize and #size' do
189
- assert @c.empty?
190
-
191
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
192
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
193
- data = [d1.to_json + "\n", d2.to_json + "\n"]
194
- @c.append(data)
195
-
196
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
197
- assert_equal 2, @c.size
198
-
199
- @c.commit
200
-
201
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
202
- assert_equal 2, @c.size
203
-
204
- first_bytesize = @c.bytesize
205
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
206
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
207
- @c.append([d3.to_json + "\n", d4.to_json + "\n"])
208
-
209
- assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
210
- assert_equal 4, @c.size
211
-
212
- @c.commit
213
-
214
- assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
215
- assert_equal 4, @c.size
216
- end
217
-
218
- test 'can #rollback to revert non-committed data' do
219
- assert @c.empty?
220
-
221
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
222
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
223
- data = [d1.to_json + "\n", d2.to_json + "\n"]
224
- @c.append(data)
225
-
226
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
227
- assert_equal 2, @c.size
228
-
229
- @c.rollback
230
-
231
- assert @c.empty?
232
- assert_equal '', File.read(@c.path)
233
-
234
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
235
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
236
- data = [d1.to_json + "\n", d2.to_json + "\n"]
237
- @c.append(data)
238
- @c.commit
239
-
240
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
241
- assert_equal 2, @c.size
242
-
243
- first_bytesize = @c.bytesize
244
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
245
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
246
- @c.append([d3.to_json + "\n", d4.to_json + "\n"])
247
-
248
- assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
249
- assert_equal 4, @c.size
250
-
251
- @c.rollback
252
-
253
- assert_equal first_bytesize, @c.bytesize
254
- assert_equal 2, @c.size
255
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.read(@c.path)
256
- end
257
-
258
- test 'can #rollback to revert non-committed data from #concat' do
259
- assert @c.empty?
260
-
261
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
262
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
263
- data = [d1.to_json + "\n", d2.to_json + "\n"].join
264
- @c.concat(data, 2)
265
-
266
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
267
- assert_equal 2, @c.size
268
-
269
- @c.rollback
270
-
271
- assert @c.empty?
272
- assert_equal '', File.read(@c.path)
273
-
274
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
275
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
276
- data = [d1.to_json + "\n", d2.to_json + "\n"]
277
- @c.append(data)
278
- @c.commit
279
-
280
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
281
- assert_equal 2, @c.size
282
-
283
- first_bytesize = @c.bytesize
284
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
285
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
286
- @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2)
287
-
288
- assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
289
- assert_equal 4, @c.size
290
-
291
- @c.rollback
292
-
293
- assert_equal first_bytesize, @c.bytesize
294
- assert_equal 2, @c.size
295
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.read(@c.path)
296
- end
297
-
298
- test 'can store its data by #close' do
299
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
300
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
301
- data = [d1.to_json + "\n", d2.to_json + "\n"]
302
- @c.append(data)
303
- @c.commit
304
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
305
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
306
- @c.append([d3.to_json + "\n", d4.to_json + "\n"])
307
- @c.commit
308
- content = @c.read
309
- @c.close
310
-
311
- assert_equal content, File.read(@c.path)
312
- end
313
-
314
- test 'deletes all data by #purge' do
315
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
316
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
317
- data = [d1.to_json + "\n", d2.to_json + "\n"]
318
- @c.append(data)
319
- @c.commit
320
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
321
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
322
- @c.append([d3.to_json + "\n", d4.to_json + "\n"])
323
- @c.commit
324
- @c.purge
325
-
326
- assert @c.empty?
327
- assert_equal 0, @c.bytesize
328
- assert_equal 0, @c.size
329
- assert !File.exist?(@c.path)
330
- end
331
-
332
- test 'can #open its contents as io' do
333
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
334
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
335
- data = [d1.to_json + "\n", d2.to_json + "\n"]
336
- @c.append(data)
337
- @c.commit
338
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
339
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
340
- @c.append([d3.to_json + "\n", d4.to_json + "\n"])
341
- @c.commit
342
-
343
- lines = []
344
- @c.open do |io|
345
- assert io
346
- io.readlines.each do |l|
347
- lines << l
348
- end
349
- end
350
-
351
- assert_equal d1.to_json + "\n", lines[0]
352
- assert_equal d2.to_json + "\n", lines[1]
353
- assert_equal d3.to_json + "\n", lines[2]
354
- assert_equal d4.to_json + "\n", lines[3]
355
- end
356
- end
357
-
358
- sub_test_case 'chunk with file for staged chunk' do
359
- setup do
360
- @chunk_id = gen_test_chunk_id
361
- @staged_path = File.join(@chunkdir, "fsb.testing.b#{hex_id(@chunk_id)}.buf")
362
- @enqueued_path = File.join(@chunkdir, "fsb.testing.q#{hex_id(@chunk_id)}.buf")
363
-
364
- @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
365
- @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
366
- @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
367
- @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
368
- @d = [@d1, @d2, @d3, @d4].map{ |d| d.to_json + "\n" }.join
369
- File.write(@staged_path, @d, :mode => 'wb')
370
-
371
- @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @staged_path, :staged, nil)
372
- end
373
-
374
- teardown do
375
- if @c
376
- @c.purge rescue nil
377
- end
378
- [@staged_path, @enqueued_path].each do |path|
379
- File.unlink(path) if File.exist?(path)
380
- end
381
- end
382
-
383
- test 'can load as staged chunk from file with metadata' do
384
- assert_equal @staged_path, @c.path
385
- assert_equal :staged, @c.state
386
-
387
- assert_nil @c.metadata.timekey
388
- assert_equal 'testing', @c.metadata.tag
389
- assert_nil @c.metadata.variables
390
- assert_equal 0, @c.size
391
- assert_equal @d, @c.read
392
-
393
- @c.restore_size(:text)
394
- assert_equal 4, @c.size
395
- end
396
-
397
- test 'can be enqueued' do
398
- stage_path = @c.path
399
- queue_path = @enqueued_path
400
- assert File.exist?(stage_path)
401
- assert !File.exist?(queue_path)
402
-
403
- @c.enqueued!
404
-
405
- assert_equal queue_path, @c.path
406
- assert !File.exist?(stage_path)
407
- assert File.exist?(queue_path)
408
-
409
- assert_nil @c.metadata.timekey
410
- assert_equal 'testing', @c.metadata.tag
411
- assert_nil @c.metadata.variables
412
-
413
- assert_equal 0, @c.size
414
- assert_equal @d, File.read(@c.path)
415
-
416
- @c.restore_size(:text)
417
- assert_equal 4, @c.size
418
- end
419
-
420
- test '#file_rename can rename chunk files even in windows, and call callback with file size' do
421
- data = "aaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbccccccccccccccccccccccccccccc"
422
-
423
- testing_file1 = gen_path('rename1.test')
424
- testing_file2 = gen_path('rename2.test')
425
- f = File.open(testing_file1, 'wb', @c.permission)
426
- f.set_encoding(Encoding::ASCII_8BIT)
427
- f.sync = true
428
- f.binmode
429
- f.write data
430
- pos = f.pos
431
-
432
- assert f.binmode?
433
- assert f.sync
434
- assert_equal data.bytesize, f.size
435
-
436
- io = nil
437
- @c.file_rename(f, testing_file1, testing_file2, ->(new_io){ io = new_io })
438
- assert io
439
- if Fluent.windows?
440
- assert { f != io }
441
- else
442
- assert_equal f, io
443
- end
444
- assert_equal Encoding::ASCII_8BIT, io.external_encoding
445
- assert io.sync
446
- assert io.binmode?
447
- assert_equal data.bytesize, io.size
448
- assert_equal pos, io.pos
449
- assert_equal '', io.read
450
-
451
- io.rewind
452
- assert_equal data, io.read
453
- end
454
- end
455
-
456
- sub_test_case 'chunk with file for enqueued chunk' do
457
- setup do
458
- @chunk_id = gen_test_chunk_id
459
- @enqueued_path = File.join(@chunkdir, "fsb.testing.q#{hex_id(@chunk_id)}.buf")
460
-
461
- @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
462
- @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
463
- @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
464
- @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
465
- @d = [@d1, @d2, @d3, @d4].map { |d| d.to_json + "\n" }.join
466
- File.write(@enqueued_path, @d, :mode => 'wb')
467
-
468
- @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @enqueued_path, :queued, nil)
469
- end
470
-
471
- teardown do
472
- if @c
473
- @c.purge rescue nil
474
- end
475
- File.unlink(@enqueued_path) if File.exist?(@enqueued_path)
476
- end
477
-
478
- test 'can load as queued chunk (read only) with metadata' do
479
- assert @c
480
- assert_equal @chunk_id, @c.unique_id
481
- assert_equal :queued, @c.state
482
- stat = File.stat(@enqueued_path)
483
- assert_equal stat.ctime.to_i, @c.created_at.to_i
484
- assert_equal stat.mtime.to_i, @c.modified_at.to_i
485
- assert_equal 0, @c.size
486
- assert_equal @d.bytesize, @c.bytesize
487
- assert_equal @d, @c.read
488
-
489
- @c.restore_size(:text)
490
- assert_equal 4, @c.size
491
-
492
- assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do
493
- @c.append(["queued chunk is read only"])
494
- end
495
- assert_raise IOError do
496
- @c.instance_eval{ @chunk }.write "chunk io is opened as read only"
497
- end
498
- end
499
- end
500
-
501
- sub_test_case 'chunk with queued chunk file' do
502
- setup do
503
- @chunk_id = gen_test_chunk_id
504
- @chunk_path = File.join(@chunkdir, "fsb.testing.q#{hex_id(@chunk_id)}.buf")
505
-
506
- @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
507
- @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
508
- @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
509
- @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
510
- @d = [@d1, @d2, @d3, @d4].map { |d| d.to_json + "\n" }.join
511
- File.write(@chunk_path, @d, :mode => 'wb')
512
-
513
- @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @chunk_path, :queued, nil)
514
- end
515
-
516
- teardown do
517
- if @c
518
- @c.purge rescue nil
519
- end
520
- File.unlink(@chunk_path) if File.exist?(@chunk_path)
521
- end
522
-
523
- test 'can load as queued chunk' do
524
- assert @c
525
- assert_equal :queued, @c.state
526
- assert_equal @chunk_id, @c.unique_id
527
- assert_equal gen_metadata, @c.metadata
528
- assert_equal @d.bytesize, @c.bytesize
529
- assert_equal 0, @c.size
530
- assert_equal @d, @c.read
531
-
532
- assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do
533
- @c.append(["queued chunk is read only"])
534
- end
535
- assert_raise IOError do
536
- @c.instance_eval{ @chunk }.write "chunk io is opened as read only"
537
- end
538
- end
539
- end
540
-
541
- sub_test_case 'compressed buffer' do
542
- setup do
543
- @src = 'text data for compressing' * 5
544
- @gzipped_src = compress(@src)
545
- end
546
-
547
- test '#append with compress option writes compressed data to chunk when compress is gzip' do
548
- c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)
549
- c.append([@src, @src], compress: :gzip)
550
- c.commit
551
-
552
- # check chunk is compressed
553
- assert c.read(compressed: :gzip).size < [@src, @src].join("").size
554
-
555
- assert_equal @src + @src, c.read
556
- end
557
-
558
- test '#open passes io object having decompressed data to a block when compress is gzip' do
559
- c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)
560
- c.concat(@gzipped_src, @src.size)
561
- c.commit
562
-
563
- decomressed_data = c.open do |io|
564
- v = io.read
565
- assert_equal @src, v
566
- v
567
- end
568
- assert_equal @src, decomressed_data
569
- end
570
-
571
- test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do
572
- c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)
573
- c.concat(@gzipped_src, @src.size)
574
- c.commit
575
-
576
- comressed_data = c.open(compressed: :gzip) do |io|
577
- v = io.read
578
- assert_equal @gzipped_src, v
579
- v
580
- end
581
- assert_equal @gzipped_src, comressed_data
582
- end
583
-
584
- test '#write_to writes decompressed data when compress is gzip' do
585
- c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)
586
- c.concat(@gzipped_src, @src.size)
587
- c.commit
588
-
589
- assert_equal @src, c.read
590
- assert_equal @gzipped_src, c.read(compressed: :gzip)
591
-
592
- io = StringIO.new
593
- c.write_to(io)
594
- assert_equal @src, io.string
595
- end
596
-
597
- test '#write_to with compressed option writes compressed data when compress is gzip' do
598
- c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)
599
- c.concat(@gzipped_src, @src.size)
600
- c.commit
601
-
602
- assert_equal @src, c.read
603
- assert_equal @gzipped_src, c.read(compressed: :gzip)
604
-
605
- io = StringIO.new
606
- io.set_encoding(Encoding::ASCII_8BIT)
607
- c.write_to(io, compressed: :gzip)
608
- assert_equal @gzipped_src, io.string
609
- end
610
- end
611
- end