fluentd 1.17.0-x86-mingw32 → 1.17.1-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
@@ -1,611 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/plugin/buffer/file_single_chunk'
3
- require 'fluent/plugin/compressable'
4
- require 'fluent/unique_id'
5
-
6
- require 'fileutils'
7
- require 'msgpack'
8
- require 'time'
9
-
10
- class BufferFileSingleChunkTest < Test::Unit::TestCase
11
- include Fluent::Plugin::Compressable
12
-
13
- setup do
14
- @klass = Fluent::Plugin::Buffer::FileSingleChunk
15
- @chunkdir = File.expand_path('../../tmp/buffer_file_single_chunk', __FILE__)
16
- FileUtils.rm_r(@chunkdir) rescue nil
17
- FileUtils.mkdir_p(@chunkdir)
18
- end
19
-
20
- Metadata = Struct.new(:timekey, :tag, :variables)
21
- def gen_metadata(timekey: nil, tag: 'testing', variables: nil)
22
- Metadata.new(timekey, tag, variables)
23
- end
24
-
25
- def gen_path(path)
26
- File.join(@chunkdir, path)
27
- end
28
-
29
- def gen_test_chunk_id
30
- now = Time.parse('2016-04-07 14:31:33 +0900')
31
- u1 = ((now.to_i * 1000 * 1000 + now.usec) << 12 | 1725) # 1725 is one of `rand(0xfff)`
32
- u3 = 2979763054 # one of rand(0xffffffff)
33
- u4 = 438020492 # ditto
34
- [u1 >> 32, u1 & 0xffffffff, u3, u4].pack('NNNN')
35
- # unique_id.unpack('N*').map{|n| n.to_s(16)}.join => "52fde6425d7406bdb19b936e1a1ba98c"
36
- end
37
-
38
- def hex_id(id)
39
- id.unpack('N*').map { |n| n.to_s(16) }.join
40
- end
41
-
42
- sub_test_case 'classmethods' do
43
- data(
44
- correct_staged: ['/mydir/mypath/fsb.b00ff.buf', :staged],
45
- correct_queued: ['/mydir/mypath/fsb.q00ff.buf', :queued],
46
- incorrect_staged: ['/mydir/mypath/fsb.b00ff.buf/unknown', :unknown],
47
- incorrect_queued: ['/mydir/mypath/fsb.q00ff.buf/unknown', :unknown],
48
- output_file: ['/mydir/mypath/fsb.20160716.buf', :unknown],
49
- )
50
- test 'can .assume_chunk_state' do |data|
51
- path, expected = data
52
- assert_equal expected, @klass.assume_chunk_state(path)
53
- end
54
-
55
- test '.generate_stage_chunk_path generates path with staged mark & chunk unique_id' do
56
- assert_equal gen_path("fsb.foo.b52fde6425d7406bdb19b936e1a1ba98c.buf"), @klass.generate_stage_chunk_path(gen_path("fsb.*.buf"), 'foo', gen_test_chunk_id)
57
- assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do
58
- @klass.generate_stage_chunk_path(gen_path("fsb.buf"), 'foo', gen_test_chunk_id)
59
- end
60
- assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do
61
- @klass.generate_stage_chunk_path(gen_path("fsb.*"), 'foo', gen_test_chunk_id)
62
- end
63
- assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do
64
- @klass.generate_stage_chunk_path(gen_path("*.buf"), 'foo', gen_test_chunk_id)
65
- end
66
- end
67
-
68
- test '.generate_queued_chunk_path generates path with enqueued mark for staged chunk path' do
69
- assert_equal(
70
- gen_path("fsb.q52fde6425d7406bdb19b936e1a1ba98c.buf"),
71
- @klass.generate_queued_chunk_path(gen_path("fsb.b52fde6425d7406bdb19b936e1a1ba98c.buf"), gen_test_chunk_id)
72
- )
73
- end
74
-
75
- test '.generate_queued_chunk_path generates special path with chunk unique_id for non staged chunk path' do
76
- assert_equal(
77
- gen_path("fsb.buf.q52fde6425d7406bdb19b936e1a1ba98c.chunk"),
78
- @klass.generate_queued_chunk_path(gen_path("fsb.buf"), gen_test_chunk_id)
79
- )
80
- assert_equal(
81
- gen_path("fsb.q55555555555555555555555555555555.buf.q52fde6425d7406bdb19b936e1a1ba98c.chunk"),
82
- @klass.generate_queued_chunk_path(gen_path("fsb.q55555555555555555555555555555555.buf"), gen_test_chunk_id)
83
- )
84
- end
85
-
86
- data('1 word tag' => 'foo',
87
- '2 words tag' => 'test.log',
88
- 'empty' => '')
89
- test '.unique_id_and_key_from_path recreates unique_id and key from file path' do |key|
90
- path = @klass.unique_id_and_key_from_path(gen_path("fsb.#{key}.q52fde6425d7406bdb19b936e1a1ba98c.buf"))
91
- assert_equal [gen_test_chunk_id, key], path
92
- end
93
- end
94
-
95
- sub_test_case 'newly created chunk' do
96
- setup do
97
- @path_conf = File.join(@chunkdir, 'fsb.*.buf')
98
- @chunk_path = File.join(@chunkdir, "fsb.testing.b#{hex_id(gen_test_chunk_id)}.buf")
99
- @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @path_conf, :create, nil)
100
- end
101
-
102
- def gen_chunk_path(prefix, unique_id)
103
- File.join(@chunkdir, "fsb.testing.#{prefix}#{Fluent::UniqueId.hex(unique_id)}.buf")
104
- end
105
-
106
- teardown do
107
- if @c
108
- @c.purge rescue nil
109
- end
110
- if File.exist?(@chunk_path)
111
- File.unlink(@chunk_path)
112
- end
113
- end
114
-
115
- test 'creates new files for chunk and metadata with specified path & permission' do
116
- assert_equal 16, @c.unique_id.size
117
- assert_equal gen_chunk_path('b', @c.unique_id), @c.path
118
-
119
- assert File.exist?(gen_chunk_path('b', @c.unique_id))
120
- assert { File.stat(gen_chunk_path('b', @c.unique_id)).mode.to_s(8).end_with?(Fluent::DEFAULT_FILE_PERMISSION.to_s(8)) }
121
-
122
- assert_equal :unstaged, @c.state
123
- assert @c.empty?
124
- end
125
-
126
- test 'can #append, #commit and #read it' do
127
- assert @c.empty?
128
-
129
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
130
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
131
- data = [d1.to_json + "\n", d2.to_json + "\n"]
132
- @c.append(data)
133
- @c.commit
134
-
135
- ds = @c.read.split("\n").select { |d| !d.empty? }
136
- assert_equal 2, ds.size
137
- assert_equal d1, JSON.parse(ds[0])
138
- assert_equal d2, JSON.parse(ds[1])
139
-
140
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
141
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
142
- @c.append([d3.to_json + "\n", d4.to_json + "\n"])
143
- @c.commit
144
-
145
- ds = @c.read.split("\n").select{|d| !d.empty? }
146
- assert_equal 4, ds.size
147
- assert_equal d1, JSON.parse(ds[0])
148
- assert_equal d2, JSON.parse(ds[1])
149
- assert_equal d3, JSON.parse(ds[2])
150
- assert_equal d4, JSON.parse(ds[3])
151
- end
152
-
153
- test 'can #concat, #commit and #read it' do
154
- assert @c.empty?
155
-
156
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
157
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
158
- data = [d1.to_json + "\n", d2.to_json + "\n"].join
159
- @c.concat(data, 2)
160
- @c.commit
161
-
162
- ds = @c.read.split("\n").select{|d| !d.empty? }
163
- assert_equal 2, ds.size
164
- assert_equal d1, JSON.parse(ds[0])
165
- assert_equal d2, JSON.parse(ds[1])
166
-
167
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
168
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
169
- @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2)
170
- @c.commit
171
-
172
- ds = @c.read.split("\n").select { |d| !d.empty? }
173
- assert_equal 4, ds.size
174
- assert_equal d1, JSON.parse(ds[0])
175
- assert_equal d2, JSON.parse(ds[1])
176
- assert_equal d3, JSON.parse(ds[2])
177
- assert_equal d4, JSON.parse(ds[3])
178
- end
179
-
180
- test 'has its contents in binary (ascii-8bit)' do
181
- data1 = "aaa bbb ccc".force_encoding('utf-8')
182
- @c.append([data1])
183
- @c.commit
184
- assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.external_encoding }
185
- assert_equal Encoding::ASCII_8BIT, @c.read.encoding
186
- end
187
-
188
- test 'has #bytesize and #size' do
189
- assert @c.empty?
190
-
191
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
192
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
193
- data = [d1.to_json + "\n", d2.to_json + "\n"]
194
- @c.append(data)
195
-
196
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
197
- assert_equal 2, @c.size
198
-
199
- @c.commit
200
-
201
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
202
- assert_equal 2, @c.size
203
-
204
- first_bytesize = @c.bytesize
205
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
206
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
207
- @c.append([d3.to_json + "\n", d4.to_json + "\n"])
208
-
209
- assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
210
- assert_equal 4, @c.size
211
-
212
- @c.commit
213
-
214
- assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
215
- assert_equal 4, @c.size
216
- end
217
-
218
- test 'can #rollback to revert non-committed data' do
219
- assert @c.empty?
220
-
221
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
222
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
223
- data = [d1.to_json + "\n", d2.to_json + "\n"]
224
- @c.append(data)
225
-
226
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
227
- assert_equal 2, @c.size
228
-
229
- @c.rollback
230
-
231
- assert @c.empty?
232
- assert_equal '', File.read(@c.path)
233
-
234
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
235
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
236
- data = [d1.to_json + "\n", d2.to_json + "\n"]
237
- @c.append(data)
238
- @c.commit
239
-
240
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
241
- assert_equal 2, @c.size
242
-
243
- first_bytesize = @c.bytesize
244
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
245
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
246
- @c.append([d3.to_json + "\n", d4.to_json + "\n"])
247
-
248
- assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
249
- assert_equal 4, @c.size
250
-
251
- @c.rollback
252
-
253
- assert_equal first_bytesize, @c.bytesize
254
- assert_equal 2, @c.size
255
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.read(@c.path)
256
- end
257
-
258
- test 'can #rollback to revert non-committed data from #concat' do
259
- assert @c.empty?
260
-
261
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
262
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
263
- data = [d1.to_json + "\n", d2.to_json + "\n"].join
264
- @c.concat(data, 2)
265
-
266
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
267
- assert_equal 2, @c.size
268
-
269
- @c.rollback
270
-
271
- assert @c.empty?
272
- assert_equal '', File.read(@c.path)
273
-
274
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
275
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
276
- data = [d1.to_json + "\n", d2.to_json + "\n"]
277
- @c.append(data)
278
- @c.commit
279
-
280
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
281
- assert_equal 2, @c.size
282
-
283
- first_bytesize = @c.bytesize
284
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
285
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
286
- @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2)
287
-
288
- assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
289
- assert_equal 4, @c.size
290
-
291
- @c.rollback
292
-
293
- assert_equal first_bytesize, @c.bytesize
294
- assert_equal 2, @c.size
295
- assert_equal (d1.to_json + "\n" + d2.to_json + "\n"), File.read(@c.path)
296
- end
297
-
298
- test 'can store its data by #close' do
299
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
300
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
301
- data = [d1.to_json + "\n", d2.to_json + "\n"]
302
- @c.append(data)
303
- @c.commit
304
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
305
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
306
- @c.append([d3.to_json + "\n", d4.to_json + "\n"])
307
- @c.commit
308
- content = @c.read
309
- @c.close
310
-
311
- assert_equal content, File.read(@c.path)
312
- end
313
-
314
- test 'deletes all data by #purge' do
315
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
316
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
317
- data = [d1.to_json + "\n", d2.to_json + "\n"]
318
- @c.append(data)
319
- @c.commit
320
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
321
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
322
- @c.append([d3.to_json + "\n", d4.to_json + "\n"])
323
- @c.commit
324
- @c.purge
325
-
326
- assert @c.empty?
327
- assert_equal 0, @c.bytesize
328
- assert_equal 0, @c.size
329
- assert !File.exist?(@c.path)
330
- end
331
-
332
- test 'can #open its contents as io' do
333
- d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
334
- d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
335
- data = [d1.to_json + "\n", d2.to_json + "\n"]
336
- @c.append(data)
337
- @c.commit
338
- d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
339
- d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
340
- @c.append([d3.to_json + "\n", d4.to_json + "\n"])
341
- @c.commit
342
-
343
- lines = []
344
- @c.open do |io|
345
- assert io
346
- io.readlines.each do |l|
347
- lines << l
348
- end
349
- end
350
-
351
- assert_equal d1.to_json + "\n", lines[0]
352
- assert_equal d2.to_json + "\n", lines[1]
353
- assert_equal d3.to_json + "\n", lines[2]
354
- assert_equal d4.to_json + "\n", lines[3]
355
- end
356
- end
357
-
358
- sub_test_case 'chunk with file for staged chunk' do
359
- setup do
360
- @chunk_id = gen_test_chunk_id
361
- @staged_path = File.join(@chunkdir, "fsb.testing.b#{hex_id(@chunk_id)}.buf")
362
- @enqueued_path = File.join(@chunkdir, "fsb.testing.q#{hex_id(@chunk_id)}.buf")
363
-
364
- @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
365
- @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
366
- @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
367
- @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
368
- @d = [@d1, @d2, @d3, @d4].map{ |d| d.to_json + "\n" }.join
369
- File.write(@staged_path, @d, :mode => 'wb')
370
-
371
- @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @staged_path, :staged, nil)
372
- end
373
-
374
- teardown do
375
- if @c
376
- @c.purge rescue nil
377
- end
378
- [@staged_path, @enqueued_path].each do |path|
379
- File.unlink(path) if File.exist?(path)
380
- end
381
- end
382
-
383
- test 'can load as staged chunk from file with metadata' do
384
- assert_equal @staged_path, @c.path
385
- assert_equal :staged, @c.state
386
-
387
- assert_nil @c.metadata.timekey
388
- assert_equal 'testing', @c.metadata.tag
389
- assert_nil @c.metadata.variables
390
- assert_equal 0, @c.size
391
- assert_equal @d, @c.read
392
-
393
- @c.restore_size(:text)
394
- assert_equal 4, @c.size
395
- end
396
-
397
- test 'can be enqueued' do
398
- stage_path = @c.path
399
- queue_path = @enqueued_path
400
- assert File.exist?(stage_path)
401
- assert !File.exist?(queue_path)
402
-
403
- @c.enqueued!
404
-
405
- assert_equal queue_path, @c.path
406
- assert !File.exist?(stage_path)
407
- assert File.exist?(queue_path)
408
-
409
- assert_nil @c.metadata.timekey
410
- assert_equal 'testing', @c.metadata.tag
411
- assert_nil @c.metadata.variables
412
-
413
- assert_equal 0, @c.size
414
- assert_equal @d, File.read(@c.path)
415
-
416
- @c.restore_size(:text)
417
- assert_equal 4, @c.size
418
- end
419
-
420
- test '#file_rename can rename chunk files even in windows, and call callback with file size' do
421
- data = "aaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbccccccccccccccccccccccccccccc"
422
-
423
- testing_file1 = gen_path('rename1.test')
424
- testing_file2 = gen_path('rename2.test')
425
- f = File.open(testing_file1, 'wb', @c.permission)
426
- f.set_encoding(Encoding::ASCII_8BIT)
427
- f.sync = true
428
- f.binmode
429
- f.write data
430
- pos = f.pos
431
-
432
- assert f.binmode?
433
- assert f.sync
434
- assert_equal data.bytesize, f.size
435
-
436
- io = nil
437
- @c.file_rename(f, testing_file1, testing_file2, ->(new_io){ io = new_io })
438
- assert io
439
- if Fluent.windows?
440
- assert { f != io }
441
- else
442
- assert_equal f, io
443
- end
444
- assert_equal Encoding::ASCII_8BIT, io.external_encoding
445
- assert io.sync
446
- assert io.binmode?
447
- assert_equal data.bytesize, io.size
448
- assert_equal pos, io.pos
449
- assert_equal '', io.read
450
-
451
- io.rewind
452
- assert_equal data, io.read
453
- end
454
- end
455
-
456
- sub_test_case 'chunk with file for enqueued chunk' do
457
- setup do
458
- @chunk_id = gen_test_chunk_id
459
- @enqueued_path = File.join(@chunkdir, "fsb.testing.q#{hex_id(@chunk_id)}.buf")
460
-
461
- @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
462
- @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
463
- @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
464
- @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
465
- @d = [@d1, @d2, @d3, @d4].map { |d| d.to_json + "\n" }.join
466
- File.write(@enqueued_path, @d, :mode => 'wb')
467
-
468
- @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @enqueued_path, :queued, nil)
469
- end
470
-
471
- teardown do
472
- if @c
473
- @c.purge rescue nil
474
- end
475
- File.unlink(@enqueued_path) if File.exist?(@enqueued_path)
476
- end
477
-
478
- test 'can load as queued chunk (read only) with metadata' do
479
- assert @c
480
- assert_equal @chunk_id, @c.unique_id
481
- assert_equal :queued, @c.state
482
- stat = File.stat(@enqueued_path)
483
- assert_equal stat.ctime.to_i, @c.created_at.to_i
484
- assert_equal stat.mtime.to_i, @c.modified_at.to_i
485
- assert_equal 0, @c.size
486
- assert_equal @d.bytesize, @c.bytesize
487
- assert_equal @d, @c.read
488
-
489
- @c.restore_size(:text)
490
- assert_equal 4, @c.size
491
-
492
- assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do
493
- @c.append(["queued chunk is read only"])
494
- end
495
- assert_raise IOError do
496
- @c.instance_eval{ @chunk }.write "chunk io is opened as read only"
497
- end
498
- end
499
- end
500
-
501
- sub_test_case 'chunk with queued chunk file' do
502
- setup do
503
- @chunk_id = gen_test_chunk_id
504
- @chunk_path = File.join(@chunkdir, "fsb.testing.q#{hex_id(@chunk_id)}.buf")
505
-
506
- @d1 = {"k" => "x", "f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
507
- @d2 = {"k" => "x", "f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
508
- @d3 = {"k" => "x", "f1" => 'x', "f2" => 'y', "f3" => 'z'}
509
- @d4 = {"k" => "x", "f1" => 'a', "f2" => 'b', "f3" => 'c'}
510
- @d = [@d1, @d2, @d3, @d4].map { |d| d.to_json + "\n" }.join
511
- File.write(@chunk_path, @d, :mode => 'wb')
512
-
513
- @c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, @chunk_path, :queued, nil)
514
- end
515
-
516
- teardown do
517
- if @c
518
- @c.purge rescue nil
519
- end
520
- File.unlink(@chunk_path) if File.exist?(@chunk_path)
521
- end
522
-
523
- test 'can load as queued chunk' do
524
- assert @c
525
- assert_equal :queued, @c.state
526
- assert_equal @chunk_id, @c.unique_id
527
- assert_equal gen_metadata, @c.metadata
528
- assert_equal @d.bytesize, @c.bytesize
529
- assert_equal 0, @c.size
530
- assert_equal @d, @c.read
531
-
532
- assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do
533
- @c.append(["queued chunk is read only"])
534
- end
535
- assert_raise IOError do
536
- @c.instance_eval{ @chunk }.write "chunk io is opened as read only"
537
- end
538
- end
539
- end
540
-
541
- sub_test_case 'compressed buffer' do
542
- setup do
543
- @src = 'text data for compressing' * 5
544
- @gzipped_src = compress(@src)
545
- end
546
-
547
- test '#append with compress option writes compressed data to chunk when compress is gzip' do
548
- c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)
549
- c.append([@src, @src], compress: :gzip)
550
- c.commit
551
-
552
- # check chunk is compressed
553
- assert c.read(compressed: :gzip).size < [@src, @src].join("").size
554
-
555
- assert_equal @src + @src, c.read
556
- end
557
-
558
- test '#open passes io object having decompressed data to a block when compress is gzip' do
559
- c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)
560
- c.concat(@gzipped_src, @src.size)
561
- c.commit
562
-
563
- decomressed_data = c.open do |io|
564
- v = io.read
565
- assert_equal @src, v
566
- v
567
- end
568
- assert_equal @src, decomressed_data
569
- end
570
-
571
- test '#open with compressed option passes io object having decompressed data to a block when compress is gzip' do
572
- c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)
573
- c.concat(@gzipped_src, @src.size)
574
- c.commit
575
-
576
- comressed_data = c.open(compressed: :gzip) do |io|
577
- v = io.read
578
- assert_equal @gzipped_src, v
579
- v
580
- end
581
- assert_equal @gzipped_src, comressed_data
582
- end
583
-
584
- test '#write_to writes decompressed data when compress is gzip' do
585
- c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)
586
- c.concat(@gzipped_src, @src.size)
587
- c.commit
588
-
589
- assert_equal @src, c.read
590
- assert_equal @gzipped_src, c.read(compressed: :gzip)
591
-
592
- io = StringIO.new
593
- c.write_to(io)
594
- assert_equal @src, io.string
595
- end
596
-
597
- test '#write_to with compressed option writes compressed data when compress is gzip' do
598
- c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :gzip)
599
- c.concat(@gzipped_src, @src.size)
600
- c.commit
601
-
602
- assert_equal @src, c.read
603
- assert_equal @gzipped_src, c.read(compressed: :gzip)
604
-
605
- io = StringIO.new
606
- io.set_encoding(Encoding::ASCII_8BIT)
607
- c.write_to(io, compressed: :gzip)
608
- assert_equal @gzipped_src, io.string
609
- end
610
- end
611
- end