fluentd 1.17.0-x86-mingw32 → 1.17.1-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
@@ -1,1038 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/output'
3
- require 'fluent/plugin/out_file'
4
- require 'fileutils'
5
- require 'time'
6
- require 'timecop'
7
- require 'zlib'
8
- require 'fluent/file_wrapper'
9
-
10
- class FileOutputTest < Test::Unit::TestCase
11
- def setup
12
- Fluent::Test.setup
13
- FileUtils.rm_rf(TMP_DIR)
14
- FileUtils.mkdir_p(TMP_DIR)
15
- @default_newline = if Fluent.windows?
16
- "\r\n"
17
- else
18
- "\n"
19
- end
20
- end
21
-
22
- TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/out_file#{ENV['TEST_ENV_NUMBER']}")
23
-
24
- CONFIG = %[
25
- path #{TMP_DIR}/out_file_test
26
- compress gz
27
- utc
28
- <buffer>
29
- timekey_use_utc true
30
- </buffer>
31
- ]
32
-
33
- def create_driver(conf = CONFIG, opts = {})
34
- Fluent::Test::Driver::Output.new(Fluent::Plugin::FileOutput, opts: opts).configure(conf)
35
- end
36
-
37
- sub_test_case 'configuration' do
38
- test 'basic configuration' do
39
- d = create_driver %[
40
- path test_path
41
- compress gz
42
- ]
43
- assert_equal 'test_path', d.instance.path
44
- assert_equal :gz, d.instance.compress
45
- assert_equal :gzip, d.instance.instance_eval{ @compress_method }
46
- end
47
-
48
- test 'using root_dir for buffer path' do
49
- system_conf_opts = {'root_dir' => File.join(TMP_DIR, 'testrootdir')}
50
- buf_conf = config_element('buffer', '', {'flush_interval' => '1s'})
51
- conf = config_element('match', '**', {'@id' => 'myout', 'path' => 'test_path', 'append' => 'true'}, [buf_conf])
52
- d = create_driver(conf, system_conf_opts)
53
-
54
- assert_equal 'test_path', d.instance.path
55
- assert d.instance.append
56
-
57
- assert d.instance.buffer.respond_to?(:path) # file buffer
58
- assert_equal 1, d.instance.buffer_config.flush_interval
59
-
60
- assert_equal File.join(TMP_DIR, 'testrootdir', 'worker0', 'myout'), d.instance.plugin_root_dir
61
-
62
- buffer_path_under_root_dir = File.join(TMP_DIR, 'testrootdir', 'worker0', 'myout', 'buffer', 'buffer.*.log')
63
- assert_equal buffer_path_under_root_dir, d.instance.buffer.path
64
- end
65
-
66
- test 'path should be writable' do
67
- assert_raise(Fluent::ConfigError.new("'path' parameter is required")) do
68
- create_driver ""
69
- end
70
-
71
- assert_nothing_raised do
72
- create_driver %[path #{TMP_DIR}/test_path]
73
- end
74
-
75
- assert_nothing_raised do
76
- FileUtils.mkdir_p("#{TMP_DIR}/test_dir")
77
- File.chmod(0777, "#{TMP_DIR}/test_dir")
78
- create_driver %[path #{TMP_DIR}/test_dir/foo/bar/baz]
79
- end
80
-
81
- if Process.uid.nonzero?
82
- assert_raise(Fluent::ConfigError) do
83
- FileUtils.mkdir_p("#{TMP_DIR}/test_dir")
84
- File.chmod(0555, "#{TMP_DIR}/test_dir")
85
- create_driver %[path #{TMP_DIR}/test_dir/foo/bar/baz]
86
- end
87
- end
88
- end
89
-
90
- test 'default timezone is localtime' do
91
- d = create_driver(%[path #{TMP_DIR}/out_file_test])
92
- time = event_time("2011-01-02 13:14:15 UTC")
93
-
94
- with_timezone(Fluent.windows? ? 'NST-8' : 'Asia/Taipei') do
95
- d.run(default_tag: 'test') do
96
- d.feed(time, {"a"=>1})
97
- end
98
- end
99
- assert_equal 1, d.formatted.size
100
- assert_equal %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
101
- end
102
-
103
- test 'no configuration error raised for basic configuration using "*" (v0.12 style)' do
104
- conf = config_element('match', '**', {
105
- 'path' => "#{TMP_DIR}/test_out.*.log",
106
- 'time_slice_format' => '%Y%m%d',
107
- })
108
- assert_nothing_raised do
109
- create_driver(conf)
110
- end
111
- end
112
-
113
- if Process.uid.nonzero?
114
- test 'configuration error raised if specified directory via template is not writable' do
115
- Timecop.freeze(Time.parse("2016-10-04 21:33:27 UTC")) do
116
- conf = config_element('match', '**', {
117
- 'path' => "#{TMP_DIR}/prohibited/${tag}/file.%Y%m%d.log",
118
- }, [ config_element('buffer', 'time,tag', {'timekey' => 86400, 'timekey_zone' => '+0000'}) ])
119
- FileUtils.mkdir_p("#{TMP_DIR}/prohibited")
120
- File.chmod(0555, "#{TMP_DIR}/prohibited")
121
- assert_raise Fluent::ConfigError.new("out_file: `#{TMP_DIR}/prohibited/a/file.20161004.log_**.log` is not writable") do
122
- create_driver(conf)
123
- end
124
- end
125
- end
126
- end
127
-
128
- test 'configuration using inject/format/buffer sections fully' do
129
- conf = config_element('match', '**', {
130
- 'path' => "#{TMP_DIR}/${tag}/${type}/conf_test.%Y%m%d.%H%M.log",
131
- 'add_path_suffix' => 'false',
132
- 'append' => "true",
133
- 'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
134
- 'compress' => 'gzip',
135
- 'recompress' => 'true',
136
- }, [
137
- config_element('inject', '', {
138
- 'hostname_key' => 'hostname',
139
- 'hostname' => 'testing.local',
140
- 'tag_key' => 'tag',
141
- 'time_key' => 'time',
142
- 'time_type' => 'string',
143
- 'time_format' => '%Y/%m/%d %H:%M:%S %z',
144
- 'timezone' => '+0900',
145
- }),
146
- config_element('format', '', {
147
- '@type' => 'out_file',
148
- 'include_tag' => 'true',
149
- 'include_time' => 'true',
150
- 'delimiter' => 'COMMA',
151
- 'time_type' => 'string',
152
- 'time_format' => '%Y-%m-%d %H:%M:%S %z',
153
- 'utc' => 'true',
154
- }),
155
- config_element('buffer', 'time,tag,type', {
156
- '@type' => 'file',
157
- 'timekey' => '15m',
158
- 'timekey_wait' => '5s',
159
- 'timekey_zone' => '+0000',
160
- 'path' => "#{TMP_DIR}/buf_conf_test",
161
- 'chunk_limit_size' => '50m',
162
- 'total_limit_size' => '1g',
163
- 'compress' => 'gzip',
164
- }),
165
- ])
166
- assert_nothing_raised do
167
- create_driver(conf)
168
- end
169
- end
170
-
171
- test 'configured as secondary with primary using chunk_key_tag and not using chunk_key_time' do
172
- require 'fluent/plugin/out_null'
173
- conf = config_element('match', '**', {
174
- }, [
175
- config_element('buffer', 'tag', {
176
- }),
177
- config_element('secondary', '', {
178
- '@type' => 'file',
179
- 'path' => "#{TMP_DIR}/testing_to_dump_by_out_file",
180
- }),
181
- ])
182
- assert_nothing_raised do
183
- Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)
184
- end
185
- end
186
- end
187
-
188
- sub_test_case 'fully configured output' do
189
- setup do
190
- Timecop.freeze(Time.parse("2016-10-03 23:58:00 UTC"))
191
- conf = config_element('match', '**', {
192
- 'path' => "#{TMP_DIR}/${tag}/${type}/full.%Y%m%d.%H%M.log",
193
- 'add_path_suffix' => 'false',
194
- 'append' => "true",
195
- 'symlink_path' => "#{TMP_DIR}/full.current.log",
196
- 'compress' => 'gzip',
197
- 'recompress' => 'true',
198
- }, [
199
- config_element('inject', '', {
200
- 'hostname_key' => 'hostname',
201
- 'hostname' => 'testing.local',
202
- 'tag_key' => 'tag',
203
- 'time_key' => 'time',
204
- 'time_type' => 'string',
205
- 'time_format' => '%Y/%m/%d %H:%M:%S %z',
206
- 'timezone' => '+0900',
207
- }),
208
- config_element('format', '', {
209
- '@type' => 'out_file',
210
- 'include_tag' => 'true',
211
- 'include_time' => 'true',
212
- 'delimiter' => 'COMMA',
213
- 'time_type' => 'string',
214
- 'time_format' => '%Y-%m-%d %H:%M:%S %z',
215
- 'utc' => 'true',
216
- }),
217
- config_element('buffer', 'time,tag,type', {
218
- '@type' => 'file',
219
- 'timekey' => '15m',
220
- 'timekey_wait' => '5s',
221
- 'timekey_zone' => '+0000',
222
- 'path' => "#{TMP_DIR}/buf_full",
223
- 'chunk_limit_size' => '50m',
224
- 'total_limit_size' => '1g',
225
- 'compress' => 'gzip',
226
- }),
227
- ])
228
- @d = create_driver(conf)
229
- end
230
-
231
- teardown do
232
- FileUtils.rm_rf("#{TMP_DIR}/buf_full")
233
- FileUtils.rm_rf("#{TMP_DIR}/my.data")
234
- FileUtils.rm_rf("#{TMP_DIR}/your.data")
235
- FileUtils.rm_rf("#{TMP_DIR}/full.current.log")
236
- Timecop.return
237
- end
238
-
239
- test 'can format/write data correctly' do
240
- d = @d
241
-
242
- assert_equal 50*1024*1024, d.instance.buffer.chunk_limit_size
243
- assert_equal 1*1024*1024*1024, d.instance.buffer.total_limit_size
244
-
245
- assert !(File.symlink?("#{TMP_DIR}/full.current.log"))
246
-
247
- t1 = event_time("2016-10-03 23:58:09 UTC")
248
- t2 = event_time("2016-10-03 23:59:33 UTC")
249
- t3 = event_time("2016-10-03 23:59:57 UTC")
250
- t4 = event_time("2016-10-04 00:00:17 UTC")
251
- t5 = event_time("2016-10-04 00:01:59 UTC")
252
-
253
- Timecop.freeze(Time.parse("2016-10-03 23:58:30 UTC"))
254
-
255
- d.run(start: true, flush: false, shutdown: false) do
256
- d.feed('my.data', t1, {"type" => "a", "message" => "data raw content"})
257
- d.feed('my.data', t2, {"type" => "a", "message" => "data raw content"})
258
- d.feed('your.data', t3, {"type" => "a", "message" => "data raw content"})
259
- end
260
-
261
- assert_equal 3, d.formatted.size
262
-
263
- assert Dir.exist?("#{TMP_DIR}/buf_full")
264
- assert !(Dir.exist?("#{TMP_DIR}/my.data/a"))
265
- assert !(Dir.exist?("#{TMP_DIR}/your.data/a"))
266
- buffer_files = Dir.entries("#{TMP_DIR}/buf_full").reject{|e| e =~ /^\.+$/ }
267
- assert_equal 2, buffer_files.count{|n| n.end_with?('.meta') }
268
- assert_equal 2, buffer_files.count{|n| !n.end_with?('.meta') }
269
-
270
- m1 = d.instance.metadata('my.data', t1, {"type" => "a"})
271
- m2 = d.instance.metadata('your.data', t3, {"type" => "a"})
272
-
273
- assert_equal 2, d.instance.buffer.stage.size
274
- b1_path = d.instance.buffer.stage[m1].path
275
- b1_size = File.lstat(b1_path).size
276
-
277
- unless Fluent.windows?
278
- assert File.symlink?("#{TMP_DIR}/full.current.log")
279
- assert_equal d.instance.buffer.stage[m2].path, File.readlink("#{TMP_DIR}/full.current.log")
280
- end
281
-
282
- Timecop.freeze(Time.parse("2016-10-04 00:00:06 UTC"))
283
-
284
- d.run(start: false, flush: true, shutdown: true) do
285
- d.feed('my.data', t4, {"type" => "a", "message" => "data raw content"})
286
- d.feed('your.data', t5, {"type" => "a", "message" => "data raw content"})
287
- end
288
-
289
- assert Dir.exist?("#{TMP_DIR}/buf_full")
290
- assert Dir.exist?("#{TMP_DIR}/my.data/a")
291
- assert Dir.exist?("#{TMP_DIR}/your.data/a")
292
-
293
- buffer_files = Dir.entries("#{TMP_DIR}/buf_full").reject{|e| e =~ /^\.+$/ }
294
- assert_equal 0, buffer_files.size
295
-
296
- assert File.exist?("#{TMP_DIR}/my.data/a/full.20161003.2345.log.gz")
297
- assert File.exist?("#{TMP_DIR}/my.data/a/full.20161004.0000.log.gz")
298
- assert File.exist?("#{TMP_DIR}/your.data/a/full.20161003.2345.log.gz")
299
- assert File.exist?("#{TMP_DIR}/your.data/a/full.20161004.0000.log.gz")
300
-
301
- assert{ File.lstat("#{TMP_DIR}/my.data/a/full.20161003.2345.log.gz").size < b1_size } # recompress
302
-
303
- assert_equal 5, d.formatted.size
304
-
305
- r1 = %!2016-10-03 23:58:09 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 08:58:09 +0900"}#{@default_newline}!
306
- r2 = %!2016-10-03 23:59:33 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 08:59:33 +0900"}#{@default_newline}!
307
- r3 = %!2016-10-03 23:59:57 +0000,your.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"your.data","time":"2016/10/04 08:59:57 +0900"}#{@default_newline}!
308
- r4 = %!2016-10-04 00:00:17 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 09:00:17 +0900"}#{@default_newline}!
309
- r5 = %!2016-10-04 00:01:59 +0000,your.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"your.data","time":"2016/10/04 09:01:59 +0900"}#{@default_newline}!
310
- assert_equal r1, d.formatted[0]
311
- assert_equal r2, d.formatted[1]
312
- assert_equal r3, d.formatted[2]
313
- assert_equal r4, d.formatted[3]
314
- assert_equal r5, d.formatted[4]
315
-
316
- read_gunzip = ->(path){
317
- File.open(path){ |fio|
318
- Zlib::GzipReader.new(StringIO.new(fio.read)).read
319
- }
320
- }
321
- assert_equal r1 + r2, read_gunzip.call("#{TMP_DIR}/my.data/a/full.20161003.2345.log.gz")
322
- assert_equal r3, read_gunzip.call("#{TMP_DIR}/your.data/a/full.20161003.2345.log.gz")
323
- assert_equal r4, read_gunzip.call("#{TMP_DIR}/my.data/a/full.20161004.0000.log.gz")
324
- assert_equal r5, read_gunzip.call("#{TMP_DIR}/your.data/a/full.20161004.0000.log.gz")
325
- end
326
- end
327
-
328
- sub_test_case 'format' do
329
- test 'timezone UTC specified' do
330
- d = create_driver
331
-
332
- time = event_time("2011-01-02 13:14:15 UTC")
333
- d.run(default_tag: 'test') do
334
- d.feed(time, {"a"=>1})
335
- d.feed(time, {"a"=>2})
336
- end
337
- assert_equal 2, d.formatted.size
338
- assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
339
- assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}], d.formatted[1]
340
- end
341
-
342
- test 'time formatted with specified timezone, using area name' do
343
- d = create_driver %[
344
- path #{TMP_DIR}/out_file_test
345
- timezone Asia/Taipei
346
- ]
347
-
348
- time = event_time("2011-01-02 13:14:15 UTC")
349
- d.run(default_tag: 'test') do
350
- d.feed(time, {"a"=>1})
351
- end
352
- assert_equal 1, d.formatted.size
353
- assert_equal %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
354
- end
355
-
356
- test 'time formatted with specified timezone, using offset' do
357
- d = create_driver %[
358
- path #{TMP_DIR}/out_file_test
359
- timezone -03:30
360
- ]
361
-
362
- time = event_time("2011-01-02 13:14:15 UTC")
363
- d.run(default_tag: 'test') do
364
- d.feed(time, {"a"=>1})
365
- end
366
- assert_equal 1, d.formatted.size
367
- assert_equal %[2011-01-02T09:44:15-03:30\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
368
- end
369
-
370
- test 'configuration error raised for invalid timezone' do
371
- assert_raise(Fluent::ConfigError) do
372
- create_driver %[
373
- path #{TMP_DIR}/out_file_test
374
- timezone Invalid/Invalid
375
- ]
376
- end
377
- end
378
- end
379
-
380
- def check_gzipped_result(path, expect)
381
- # Zlib::GzipReader has a bug of concatenated file: https://bugs.ruby-lang.org/issues/9790
382
- # Following code from https://www.ruby-forum.com/topic/971591#979520
383
- result = ''
384
- File.open(path, "rb") { |io|
385
- loop do
386
- gzr = Zlib::GzipReader.new(StringIO.new(io.read))
387
- result << gzr.read
388
- unused = gzr.unused
389
- gzr.finish
390
- break if unused.nil?
391
- io.pos -= unused.length
392
- end
393
- }
394
-
395
- assert_equal expect, result
396
- end
397
-
398
- def check_result(path, expect)
399
- result = File.read(path, mode: "rb")
400
- assert_equal expect, result
401
- end
402
-
403
- sub_test_case 'write' do
404
- test 'basic case' do
405
- d = create_driver
406
-
407
- assert_false File.exist?("#{TMP_DIR}/out_file_test.20110102_0.log.gz")
408
-
409
- time = event_time("2011-01-02 13:14:15 UTC")
410
- d.run(default_tag: 'test') do
411
- d.feed(time, {"a"=>1})
412
- d.feed(time, {"a"=>2})
413
- end
414
-
415
- assert File.exist?("#{TMP_DIR}/out_file_test.20110102_0.log.gz")
416
- check_gzipped_result("#{TMP_DIR}/out_file_test.20110102_0.log.gz", %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}])
417
- end
418
- end
419
-
420
- sub_test_case 'file/directory permissions' do
421
- TMP_DIR_WITH_SYSTEM = File.expand_path(File.dirname(__FILE__) + "/../tmp/out_file_system#{ENV['TEST_ENV_NUMBER']}")
422
- # 0750 interprets as "488". "488".to_i(8) # => 4. So, it makes wrong permission. Umm....
423
- OVERRIDE_DIR_PERMISSION = 750
424
- OVERRIDE_FILE_PERMISSION = 0620
425
- CONFIG_WITH_SYSTEM = %[
426
- path #{TMP_DIR_WITH_SYSTEM}/out_file_test
427
- compress gz
428
- utc
429
- <buffer>
430
- timekey_use_utc true
431
- </buffer>
432
- <system>
433
- file_permission #{OVERRIDE_FILE_PERMISSION}
434
- dir_permission #{OVERRIDE_DIR_PERMISSION}
435
- </system>
436
- ]
437
-
438
- setup do
439
- omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
440
- FileUtils.rm_rf(TMP_DIR_WITH_SYSTEM)
441
- end
442
-
443
- def parse_system(text)
444
- basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
445
- Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
446
- end
447
-
448
- test 'write to file with permission specifications' do
449
- system_conf = parse_system(CONFIG_WITH_SYSTEM)
450
- sc = Fluent::SystemConfig.new(system_conf)
451
- Fluent::Engine.init(sc)
452
- d = create_driver CONFIG_WITH_SYSTEM
453
-
454
- assert_false File.exist?("#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz")
455
-
456
- time = event_time("2011-01-02 13:14:15 UTC")
457
- d.run(default_tag: 'test') do
458
- d.feed(time, {"a"=>1})
459
- d.feed(time, {"a"=>2})
460
- end
461
-
462
- assert File.exist?("#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz")
463
-
464
- check_gzipped_result("#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz", %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
465
- dir_mode = "%o" % File::stat(TMP_DIR_WITH_SYSTEM).mode
466
- assert_equal(OVERRIDE_DIR_PERMISSION, dir_mode[-3, 3].to_i)
467
- file_mode = "%o" % File::stat("#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz").mode
468
- assert_equal(OVERRIDE_FILE_PERMISSION, file_mode[-3, 3].to_i)
469
- end
470
- end
471
-
472
- sub_test_case 'format specified' do
473
- test 'json' do
474
- d = create_driver [CONFIG, 'format json', 'include_time_key true', 'time_as_epoch'].join("\n")
475
-
476
- time = event_time("2011-01-02 13:14:15 UTC")
477
- d.run(default_tag: 'test') do
478
- d.feed(time, {"a"=>1})
479
- d.feed(time, {"a"=>2})
480
- end
481
-
482
- path = d.instance.last_written_path
483
- check_gzipped_result(path, %[#{Yajl.dump({"a" => 1, 'time' => time.to_i})}#{@default_newline}] + %[#{Yajl.dump({"a" => 2, 'time' => time.to_i})}#{@default_newline}])
484
- end
485
-
486
- test 'ltsv' do
487
- d = create_driver [CONFIG, 'format ltsv', 'include_time_key true'].join("\n")
488
-
489
- time = event_time("2011-01-02 13:14:15 UTC")
490
- d.run(default_tag: 'test') do
491
- d.feed(time, {"a"=>1})
492
- d.feed(time, {"a"=>2})
493
- end
494
-
495
- path = d.instance.last_written_path
496
- check_gzipped_result(path, %[a:1\ttime:2011-01-02T13:14:15Z#{@default_newline}] + %[a:2\ttime:2011-01-02T13:14:15Z#{@default_newline}])
497
- end
498
-
499
- test 'single_value' do
500
- d = create_driver [CONFIG, 'format single_value', 'message_key a'].join("\n")
501
-
502
- time = event_time("2011-01-02 13:14:15 UTC")
503
- d.run(default_tag: 'test') do
504
- d.feed(time, {"a"=>1})
505
- d.feed(time, {"a"=>2})
506
- end
507
-
508
- path = d.instance.last_written_path
509
- check_gzipped_result(path, %[1#{@default_newline}] + %[2#{@default_newline}])
510
- end
511
- end
512
-
513
- test 'path with index number' do
514
- time = event_time("2011-01-02 13:14:15 UTC")
515
- formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}]
516
-
517
- write_once = ->(){
518
- d = create_driver
519
- d.run(default_tag: 'test'){
520
- d.feed(time, {"a"=>1})
521
- d.feed(time, {"a"=>2})
522
- }
523
- d.instance.last_written_path
524
- }
525
-
526
- assert !File.exist?("#{TMP_DIR}/out_file_test.20110102_0.log.gz")
527
-
528
- path = write_once.call
529
- assert_equal "#{TMP_DIR}/out_file_test.20110102_0.log.gz", path
530
- check_gzipped_result(path, formatted_lines)
531
- assert_equal 1, Dir.glob("#{TMP_DIR}/out_file_test.*").size
532
-
533
- path = write_once.call
534
- assert_equal "#{TMP_DIR}/out_file_test.20110102_1.log.gz", path
535
- check_gzipped_result(path, formatted_lines)
536
- assert_equal 2, Dir.glob("#{TMP_DIR}/out_file_test.*").size
537
-
538
- path = write_once.call
539
- assert_equal "#{TMP_DIR}/out_file_test.20110102_2.log.gz", path
540
- check_gzipped_result(path, formatted_lines)
541
- assert_equal 3, Dir.glob("#{TMP_DIR}/out_file_test.*").size
542
- end
543
-
544
- data(
545
- "with compression" => true,
546
- "without compression" => false,
547
- )
548
- test 'append' do |compression|
549
- time = event_time("2011-01-02 13:14:15 UTC")
550
- formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}]
551
-
552
- write_once = ->(){
553
- config = %[
554
- path #{TMP_DIR}/out_file_test
555
- utc
556
- append true
557
- <buffer>
558
- timekey_use_utc true
559
- </buffer>
560
- ]
561
- if compression
562
- config << " compress gz"
563
- end
564
- d = create_driver(config)
565
- d.run(default_tag: 'test'){
566
- d.feed(time, {"a"=>1})
567
- d.feed(time, {"a"=>2})
568
- }
569
- d.instance.last_written_path
570
- }
571
-
572
- log_file_name = "out_file_test.20110102.log"
573
- if compression
574
- log_file_name << ".gz"
575
- end
576
-
577
- 1.upto(3) do |i|
578
- path = write_once.call
579
- assert_equal "#{TMP_DIR}/#{log_file_name}", path
580
- expect = formatted_lines * i
581
- if compression
582
- check_gzipped_result(path, expect)
583
- else
584
- check_result(path, expect)
585
- end
586
- end
587
- end
588
-
589
- test 'append when JST' do
590
- with_timezone(Fluent.windows? ? "JST-9" : "Asia/Tokyo") do
591
- time = event_time("2011-01-02 03:14:15+09:00")
592
- formatted_lines = %[2011-01-02T03:14:15+09:00\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T03:14:15+09:00\ttest\t{"a":2}#{@default_newline}]
593
-
594
- write_once = ->(){
595
- d = create_driver %[
596
- path #{TMP_DIR}/out_file_test
597
- compress gz
598
- append true
599
- <buffer>
600
- timekey_use_utc false
601
- timekey_zone Asia/Tokyo
602
- </buffer>
603
- ]
604
- d.run(default_tag: 'test'){
605
- d.feed(time, {"a"=>1})
606
- d.feed(time, {"a"=>2})
607
- }
608
- d.instance.last_written_path
609
- }
610
-
611
- path = write_once.call
612
- assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
613
- check_gzipped_result(path, formatted_lines)
614
-
615
- path = write_once.call
616
- assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
617
- check_gzipped_result(path, formatted_lines * 2)
618
-
619
- path = write_once.call
620
- assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
621
- check_gzipped_result(path, formatted_lines * 3)
622
- end
623
- end
624
-
625
- test 'append when UTC-02 but timekey_zone is +0900' do
626
- with_timezone("UTC-02") do # +0200
627
- time = event_time("2011-01-02 17:14:15+02:00")
628
- formatted_lines = %[2011-01-02T17:14:15+02:00\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T17:14:15+02:00\ttest\t{"a":2}#{@default_newline}]
629
-
630
- write_once = ->(){
631
- d = create_driver %[
632
- path #{TMP_DIR}/out_file_test
633
- compress gz
634
- append true
635
- <buffer>
636
- timekey_use_utc false
637
- timekey_zone +0900
638
- </buffer>
639
- ]
640
- d.run(default_tag: 'test'){
641
- d.feed(time, {"a"=>1})
642
- d.feed(time, {"a"=>2})
643
- }
644
- d.instance.last_written_path
645
- }
646
-
647
- path = write_once.call
648
- # Rotated at 2011-01-02 17:00:00+02:00
649
- assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
650
- check_gzipped_result(path, formatted_lines)
651
-
652
- path = write_once.call
653
- assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
654
- check_gzipped_result(path, formatted_lines * 2)
655
-
656
- path = write_once.call
657
- assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
658
- check_gzipped_result(path, formatted_lines * 3)
659
- end
660
- end
661
-
662
- test '${chunk_id}' do
663
- time = event_time("2011-01-02 13:14:15 UTC")
664
-
665
- write_once = ->(){
666
- d = create_driver %[
667
- path #{TMP_DIR}/out_file_chunk_id_${chunk_id}
668
- utc
669
- append true
670
- <buffer>
671
- timekey_use_utc true
672
- </buffer>
673
- ]
674
- d.run(default_tag: 'test'){
675
- d.feed(time, {"a"=>1})
676
- d.feed(time, {"a"=>2})
677
- }
678
- d.instance.last_written_path
679
- }
680
-
681
- path = write_once.call
682
- if File.basename(path) =~ /out_file_chunk_id_([-_.@a-zA-Z0-9].*).20110102.log/
683
- unique_id = Fluent::UniqueId.hex(Fluent::UniqueId.generate)
684
- assert_equal unique_id.size, $1.size, "chunk_id size is mismatched"
685
- else
686
- flunk "chunk_id is not included in the path"
687
- end
688
- end
689
-
690
- SYMLINK_PATH = File.expand_path("#{TMP_DIR}/current")
691
-
692
- sub_test_case 'symlink' do
693
- test 'static symlink' do
694
- omit "Windows doesn't support symlink" if Fluent.windows?
695
- conf = CONFIG + %[
696
- symlink_path #{SYMLINK_PATH}
697
- ]
698
- symlink_path = "#{SYMLINK_PATH}"
699
-
700
- d = create_driver(conf)
701
- begin
702
- run_and_check(d, symlink_path)
703
- ensure
704
- FileUtils.rm_rf(symlink_path)
705
- end
706
- end
707
-
708
- test 'symlink with placeholders' do
709
- omit "Windows doesn't support symlink" if Fluent.windows?
710
- conf = %[
711
- path #{TMP_DIR}/${tag}/out_file_test
712
- symlink_path #{SYMLINK_PATH}/foo/${tag}
713
- <buffer tag,time>
714
- </buffer>
715
- ]
716
- symlink_path = "#{SYMLINK_PATH}/foo/tag"
717
-
718
- d = create_driver(conf)
719
- begin
720
- run_and_check(d, symlink_path)
721
- ensure
722
- FileUtils.rm_rf(symlink_path)
723
- end
724
- end
725
-
726
- def run_and_check(d, symlink_path)
727
- d.run(default_tag: 'tag') do
728
- es = Fluent::OneEventStream.new(event_time("2011-01-02 13:14:15 UTC"), {"a"=>1})
729
- d.feed(es)
730
-
731
- assert File.symlink?(symlink_path)
732
- assert File.exist?(symlink_path) # This checks dest of symlink exists or not.
733
-
734
- es = Fluent::OneEventStream.new(event_time("2011-01-03 14:15:16 UTC"), {"a"=>2})
735
- d.feed(es)
736
-
737
- assert File.symlink?(symlink_path)
738
- assert File.exist?(symlink_path)
739
-
740
- meta = d.instance.metadata('tag', event_time("2011-01-03 14:15:16 UTC"), {})
741
- assert_equal d.instance.buffer.instance_eval{ @stage[meta].path }, File.readlink(symlink_path)
742
- end
743
- end
744
- end
745
-
746
- sub_test_case 'path' do
747
- test 'normal' do
748
- d = create_driver(%[
749
- path #{TMP_DIR}/out_file_test
750
- time_slice_format %Y-%m-%d-%H
751
- utc true
752
- ])
753
- time = event_time("2011-01-02 13:14:15 UTC")
754
- d.run(default_tag: 'test') do
755
- d.feed(time, {"a"=>1})
756
- end
757
- path = d.instance.last_written_path
758
- assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13_0.log", path
759
- end
760
-
761
- test 'normal with append' do
762
- d = create_driver(%[
763
- path #{TMP_DIR}/out_file_test
764
- time_slice_format %Y-%m-%d-%H
765
- utc true
766
- append true
767
- ])
768
- time = event_time("2011-01-02 13:14:15 UTC")
769
- d.run(default_tag: 'test') do
770
- d.feed(time, {"a"=>1})
771
- end
772
- path = d.instance.last_written_path
773
- assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13.log", path
774
- end
775
-
776
- test '*' do
777
- d = create_driver(%[
778
- path #{TMP_DIR}/out_file_test.*.txt
779
- time_slice_format %Y-%m-%d-%H
780
- utc true
781
- ])
782
- time = event_time("2011-01-02 13:14:15 UTC")
783
- d.run(default_tag: 'test') do
784
- d.feed(time, {"a"=>1})
785
- end
786
- path = d.instance.last_written_path
787
- assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13_0.txt", path
788
- end
789
-
790
- test '* with append' do
791
- d = create_driver(%[
792
- path #{TMP_DIR}/out_file_test.*.txt
793
- time_slice_format %Y-%m-%d-%H
794
- utc true
795
- append true
796
- ])
797
- time = event_time("2011-01-02 13:14:15 UTC")
798
- d.run(default_tag: 'test') do
799
- d.feed(time, {"a"=>1})
800
- end
801
- path = d.instance.last_written_path
802
- assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13.txt", path
803
- end
804
- end
805
-
806
- sub_test_case '#timekey_to_timeformat' do
807
- setup do
808
- @d = create_driver
809
- @i = @d.instance
810
- end
811
-
812
- test 'returns empty string for nil' do
813
- assert_equal '', @i.timekey_to_timeformat(nil)
814
- end
815
-
816
- test 'returns timestamp string with seconds for timekey smaller than 60' do
817
- assert_equal '%Y%m%d%H%M%S', @i.timekey_to_timeformat(1)
818
- assert_equal '%Y%m%d%H%M%S', @i.timekey_to_timeformat(30)
819
- assert_equal '%Y%m%d%H%M%S', @i.timekey_to_timeformat(59)
820
- end
821
-
822
- test 'returns timestamp string with minutes for timekey smaller than 3600' do
823
- assert_equal '%Y%m%d%H%M', @i.timekey_to_timeformat(60)
824
- assert_equal '%Y%m%d%H%M', @i.timekey_to_timeformat(180)
825
- assert_equal '%Y%m%d%H%M', @i.timekey_to_timeformat(1800)
826
- assert_equal '%Y%m%d%H%M', @i.timekey_to_timeformat(3599)
827
- end
828
-
829
- test 'returns timestamp string with hours for timekey smaller than 86400 (1 day)' do
830
- assert_equal '%Y%m%d%H', @i.timekey_to_timeformat(3600)
831
- assert_equal '%Y%m%d%H', @i.timekey_to_timeformat(7200)
832
- assert_equal '%Y%m%d%H', @i.timekey_to_timeformat(86399)
833
- end
834
-
835
- test 'returns timestamp string with days for timekey equal or greater than 86400' do
836
- assert_equal '%Y%m%d', @i.timekey_to_timeformat(86400)
837
- assert_equal '%Y%m%d', @i.timekey_to_timeformat(1000000)
838
- assert_equal '%Y%m%d', @i.timekey_to_timeformat(1000000000)
839
- end
840
- end
841
-
842
- sub_test_case '#compression_suffix' do
843
- setup do
844
- @i = create_driver.instance
845
- end
846
-
847
- test 'returns empty string for nil (no compression method specified)' do
848
- assert_equal '', @i.compression_suffix(nil)
849
- end
850
-
851
- test 'returns .gz for gzip' do
852
- assert_equal '.gz', @i.compression_suffix(:gzip)
853
- end
854
- end
855
-
856
- sub_test_case '#generate_path_template' do
857
- setup do
858
- @i = create_driver.instance
859
- end
860
-
861
- data(
862
- 'day' => [86400, '%Y%m%d', '%Y-%m-%d'],
863
- 'hour' => [3600, '%Y%m%d%H', '%Y-%m-%d_%H'],
864
- 'minute' => [60, '%Y%m%d%H%M', '%Y-%m-%d_%H%M'],
865
- )
866
- test 'generates path with timestamp placeholder for original path with tailing star with timekey' do |data|
867
- timekey, placeholder, time_slice_format = data
868
- # with index placeholder, without compression suffix when append disabled and compression disabled
869
- assert_equal "/path/to/file.#{placeholder}_**", @i.generate_path_template('/path/to/file.*', timekey, false, nil)
870
- # with index placeholder, with .gz suffix when append disabled and gzip compression enabled
871
- assert_equal "/path/to/file.#{placeholder}_**.gz", @i.generate_path_template('/path/to/file.*', timekey, false, :gzip)
872
- # without index placeholder, without compression suffix when append enabled and compression disabled
873
- assert_equal "/path/to/file.#{placeholder}", @i.generate_path_template('/path/to/file.*', timekey, true, nil)
874
- # without index placeholder, with .gz suffix when append disabled and gzip compression enabled
875
- assert_equal "/path/to/file.#{placeholder}.gz", @i.generate_path_template('/path/to/file.*', timekey, true, :gzip)
876
-
877
- # time_slice_format will used instead of computed placeholder if specified
878
- assert_equal "/path/to/file.#{time_slice_format}_**", @i.generate_path_template('/path/to/file.*', timekey, false, nil, time_slice_format: time_slice_format)
879
- assert_equal "/path/to/file.#{time_slice_format}_**.gz", @i.generate_path_template('/path/to/file.*', timekey, false, :gzip, time_slice_format: time_slice_format)
880
- assert_equal "/path/to/file.#{time_slice_format}", @i.generate_path_template('/path/to/file.*', timekey, true, nil, time_slice_format: time_slice_format)
881
- assert_equal "/path/to/file.#{time_slice_format}.gz", @i.generate_path_template('/path/to/file.*', timekey, true, :gzip, time_slice_format: time_slice_format)
882
- end
883
-
884
- data(
885
- 'day' => [86400 * 2, '%Y%m%d', '%Y-%m-%d'],
886
- 'hour' => [7200, '%Y%m%d%H', '%Y-%m-%d_%H'],
887
- 'minute' => [180, '%Y%m%d%H%M', '%Y-%m-%d_%H%M'],
888
- )
889
- test 'generates path with timestamp placeholder for original path with star and suffix with timekey' do |data|
890
- timekey, placeholder, time_slice_format = data
891
- # with index placeholder, without compression suffix when append disabled and compression disabled
892
- assert_equal "/path/to/file.#{placeholder}_**.data", @i.generate_path_template('/path/to/file.*.data', timekey, false, nil)
893
- # with index placeholder, with .gz suffix when append disabled and gzip compression enabled
894
- assert_equal "/path/to/file.#{placeholder}_**.data.gz", @i.generate_path_template('/path/to/file.*.data', timekey, false, :gzip)
895
- # without index placeholder, without compression suffix when append enabled and compression disabled
896
- assert_equal "/path/to/file.#{placeholder}.data", @i.generate_path_template('/path/to/file.*.data', timekey, true, nil)
897
- # without index placeholder, with .gz suffix when append disabled and gzip compression enabled
898
- assert_equal "/path/to/file.#{placeholder}.data.gz", @i.generate_path_template('/path/to/file.*.data', timekey, true, :gzip)
899
-
900
- # time_slice_format will used instead of computed placeholder if specified
901
- assert_equal "/path/to/file.#{time_slice_format}_**.data", @i.generate_path_template('/path/to/file.*.data', timekey, false, nil, time_slice_format: time_slice_format)
902
- assert_equal "/path/to/file.#{time_slice_format}_**.data.gz", @i.generate_path_template('/path/to/file.*.data', timekey, false, :gzip, time_slice_format: time_slice_format)
903
- assert_equal "/path/to/file.#{time_slice_format}.data", @i.generate_path_template('/path/to/file.*.data', timekey, true, nil, time_slice_format: time_slice_format)
904
- assert_equal "/path/to/file.#{time_slice_format}.data.gz", @i.generate_path_template('/path/to/file.*.data', timekey, true, :gzip, time_slice_format: time_slice_format)
905
- end
906
-
907
- test 'raise error to show it is a bug when path including * specified without timekey' do
908
- assert_raise RuntimeError.new("BUG: configuration error must be raised for path including '*' without timekey") do
909
- @i.generate_path_template('/path/to/file.*.log', nil, false, nil)
910
- end
911
- end
912
-
913
- data(
914
- 'day' => [86400 * 7, '%Y%m%d', '%Y-%m-%d'],
915
- 'hour' => [3600 * 6, '%Y%m%d%H', '%Y-%m-%d_%H'],
916
- 'minute' => [60 * 15, '%Y%m%d%H%M', '%Y-%m-%d_%H%M'],
917
- )
918
- test 'generates path with timestamp placeholder for original path without time placeholders & star with timekey, and path_suffix configured' do |data|
919
- timekey, placeholder, time_slice_format = data
920
- # with index placeholder, without compression suffix when append disabled and compression disabled
921
- assert_equal "/path/to/file.#{placeholder}_**.log", @i.generate_path_template('/path/to/file', timekey, false, nil, path_suffix: '.log')
922
- # with index placeholder, with .gz suffix when append disabled and gzip compression enabled
923
- assert_equal "/path/to/file.#{placeholder}_**.log.gz", @i.generate_path_template('/path/to/file', timekey, false, :gzip, path_suffix: '.log')
924
- # without index placeholder, without compression suffix when append enabled and compression disabled
925
- assert_equal "/path/to/file.#{placeholder}.log", @i.generate_path_template('/path/to/file', timekey, true, nil, path_suffix: '.log')
926
- # without index placeholder, with compression suffix when append enabled and gzip compression enabled
927
- assert_equal "/path/to/file.#{placeholder}.log.gz", @i.generate_path_template('/path/to/file', timekey, true, :gzip, path_suffix: '.log')
928
-
929
- # time_slice_format will be appended always if it's specified
930
- assert_equal "/path/to/file.#{time_slice_format}_**.log", @i.generate_path_template('/path/to/file', timekey, false, nil, path_suffix: '.log', time_slice_format: time_slice_format)
931
- assert_equal "/path/to/file.#{time_slice_format}_**.log.gz", @i.generate_path_template('/path/to/file', timekey, false, :gzip, path_suffix: '.log', time_slice_format: time_slice_format)
932
- assert_equal "/path/to/file.#{time_slice_format}.log", @i.generate_path_template('/path/to/file', timekey, true, nil, path_suffix: '.log', time_slice_format: time_slice_format)
933
- assert_equal "/path/to/file.#{time_slice_format}.log.gz", @i.generate_path_template('/path/to/file', timekey, true, :gzip, path_suffix: '.log', time_slice_format: time_slice_format)
934
- end
935
-
936
- data(
937
- 'day' => [86400, '%Y%m%d'],
938
- 'hour' => [3600, '%Y%m%d%H'],
939
- 'minute' => [60, '%Y%m%d%H%M'],
940
- )
941
- test 'generates path with timestamp placeholder for original path without star with timekey, and path_suffix not configured' do |data|
942
- timekey, placeholder = data
943
- # with index placeholder, without compression suffix when append disabled and compression disabled
944
- assert_equal "/path/to/file.#{placeholder}_**", @i.generate_path_template('/path/to/file', timekey, false, nil)
945
- # with index placeholder, with .gz suffix when append disabled and gzip compression enabled
946
- assert_equal "/path/to/file.#{placeholder}_**.gz", @i.generate_path_template('/path/to/file', timekey, false, :gzip)
947
- # without index placeholder, without compression suffix when append enabled and compression disabled
948
- assert_equal "/path/to/file.#{placeholder}", @i.generate_path_template('/path/to/file', timekey, true, nil)
949
- # without index placeholder, with compression suffix when append enabled and gzip compression enabled
950
- assert_equal "/path/to/file.#{placeholder}.gz", @i.generate_path_template('/path/to/file', timekey, true, :gzip)
951
- end
952
-
953
- test 'generates path without adding timestamp placeholder part if original path has enough placeholders for specified timekey' do
954
- assert_equal "/path/to/file.%Y%m%d", @i.generate_path_template('/path/to/file.%Y%m%d', 86400, true, nil)
955
- assert_equal "/path/to/%Y%m%d/file", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, true, nil)
956
-
957
- assert_equal "/path/to/%Y%m%d/file_**", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, false, nil)
958
-
959
- assert_raise Fluent::ConfigError.new("insufficient timestamp placeholders in path") do
960
- @i.generate_path_template('/path/to/%Y%m/file', 86400, true, nil)
961
- end
962
- assert_raise Fluent::ConfigError.new("insufficient timestamp placeholders in path") do
963
- @i.generate_path_template('/path/to/file.%Y%m%d.log', 3600, true, nil)
964
- end
965
-
966
- assert_equal "/path/to/file.%Y%m%d_%H_**.log.gz", @i.generate_path_template('/path/to/file.%Y%m%d_%H', 7200, false, :gzip, path_suffix: '.log')
967
- assert_equal "/path/to/${tag}/file.%Y%m%d_%H_**.log.gz", @i.generate_path_template('/path/to/${tag}/file.%Y%m%d_%H', 7200, false, :gzip, path_suffix: '.log')
968
- end
969
-
970
- test 'generates path with specified time_slice_format appended even if path has sufficient timestamp placeholders' do
971
- assert_equal "/path/to/%Y%m%d/file.%Y-%m-%d_%H_**", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, false, nil, time_slice_format: '%Y-%m-%d_%H')
972
- assert_equal "/path/to/%Y%m%d/file.%Y-%m-%d_%H", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, true, nil, time_slice_format: '%Y-%m-%d_%H')
973
- assert_equal "/path/to/%Y%m%d/file.%Y-%m-%d_%H_**.log", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, false, nil, time_slice_format: '%Y-%m-%d_%H', path_suffix: '.log')
974
- assert_equal "/path/to/%Y%m%d/file.%Y-%m-%d_%H.log", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, true, nil, time_slice_format: '%Y-%m-%d_%H', path_suffix: '.log')
975
- assert_equal "/path/to/%Y%m%d/file.%Y-%m-%d_%H.log.gz", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, true, :gzip, time_slice_format: '%Y-%m-%d_%H', path_suffix: '.log')
976
- end
977
-
978
- test 'generates path without timestamp placeholder when path does not include * and timekey not specified' do
979
- assert_equal '/path/to/file.log', @i.generate_path_template('/path/to/file.log', nil, true, nil)
980
- assert_equal '/path/to/file.log_**', @i.generate_path_template('/path/to/file.log', nil, false, nil)
981
- assert_equal '/path/to/file.${tag}.log_**', @i.generate_path_template('/path/to/file.${tag}.log', nil, false, nil)
982
- assert_equal '/path/to/file.${tag}_**.log', @i.generate_path_template('/path/to/file.${tag}', nil, false, nil, path_suffix: '.log')
983
- end
984
- end
985
-
986
- sub_test_case '#find_filepath_available' do
987
- setup do
988
- @tmp = File.join(TMP_DIR, 'find_filepath_test')
989
- FileUtils.mkdir_p @tmp
990
- @i = create_driver.instance
991
- end
992
-
993
- teardown do
994
- FileUtils.rm_rf @tmp
995
- end
996
-
997
- test 'raise error if argument path does not include index placeholder' do
998
- assert_raise RuntimeError.new("BUG: index placeholder not found in path: #{@tmp}/myfile") do
999
- @i.find_filepath_available("#{@tmp}/myfile") do |path|
1000
- # ...
1001
- end
1002
- end
1003
- end
1004
-
1005
- data(
1006
- 'without suffix' => ['myfile_0', 'myfile_**'],
1007
- 'with timestamp' => ['myfile_20161003_0', 'myfile_20161003_**'],
1008
- 'with base suffix' => ['myfile_0.log', 'myfile_**.log'],
1009
- 'with compression suffix' => ['myfile_0.log.gz', 'myfile_**.log.gz'],
1010
- )
1011
- test 'returns filepath with _0 at first' do |data|
1012
- expected, argument = data
1013
- @i.find_filepath_available(File.join(@tmp, argument)) do |path|
1014
- assert_equal File.join(@tmp, expected), path
1015
- end
1016
- end
1017
-
1018
- test 'returns filepath with index which does not exist yet' do
1019
- 5.times do |i|
1020
- Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
1021
- end
1022
- @i.find_filepath_available(File.join(@tmp, "exist_**.log")) do |path|
1023
- assert_equal File.join(@tmp, "exist_5.log"), path
1024
- end
1025
- end
1026
-
1027
- test 'creates lock directory when with_lock is true to exclude operations of other worker process' do
1028
- 5.times do |i|
1029
- Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a')
1030
- end
1031
- Dir.mkdir(File.join(@tmp, "exist_5.log.lock"))
1032
- @i.find_filepath_available(File.join(@tmp, "exist_**.log"), with_lock: true) do |path|
1033
- assert Dir.exist?(File.join(@tmp, "exist_6.log.lock"))
1034
- assert_equal File.join(@tmp, "exist_6.log"), path
1035
- end
1036
- end
1037
- end
1038
- end