fluentd 1.17.0-x86-mingw32 → 1.18.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (278) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +100 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/command/fluentd.rb +7 -1
  8. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  9. data/lib/fluent/compat/propagate_default.rb +4 -4
  10. data/lib/fluent/config/literal_parser.rb +9 -2
  11. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  12. data/lib/fluent/engine.rb +49 -33
  13. data/lib/fluent/env.rb +3 -0
  14. data/lib/fluent/event_router.rb +2 -2
  15. data/lib/fluent/log/console_adapter.rb +4 -2
  16. data/lib/fluent/plugin/filter_parser.rb +27 -51
  17. data/lib/fluent/plugin/in_exec.rb +14 -2
  18. data/lib/fluent/plugin/in_http.rb +6 -1
  19. data/lib/fluent/plugin/in_sample.rb +13 -7
  20. data/lib/fluent/plugin/in_syslog.rb +4 -0
  21. data/lib/fluent/plugin/in_tail.rb +65 -23
  22. data/lib/fluent/plugin/in_tcp.rb +4 -0
  23. data/lib/fluent/plugin/in_udp.rb +10 -1
  24. data/lib/fluent/plugin/input.rb +4 -0
  25. data/lib/fluent/plugin/out_buffer.rb +40 -0
  26. data/lib/fluent/plugin/out_copy.rb +1 -1
  27. data/lib/fluent/plugin/out_file.rb +8 -0
  28. data/lib/fluent/plugin/out_http.rb +12 -0
  29. data/lib/fluent/plugin/output.rb +2 -0
  30. data/lib/fluent/plugin/parser_json.rb +4 -12
  31. data/lib/fluent/plugin_helper/cert_option.rb +8 -0
  32. data/lib/fluent/plugin_helper/event_emitter.rb +12 -0
  33. data/lib/fluent/plugin_helper/http_server/server.rb +24 -8
  34. data/lib/fluent/plugin_helper/server.rb +9 -0
  35. data/lib/fluent/root_agent.rb +114 -19
  36. data/lib/fluent/source_only_buffer_agent.rb +102 -0
  37. data/lib/fluent/supervisor.rb +207 -34
  38. data/lib/fluent/system_config.rb +15 -3
  39. data/lib/fluent/version.rb +1 -1
  40. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  41. metadata +24 -483
  42. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  43. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  44. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  45. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  46. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  47. data/.github/ISSUE_TEMPLATE.md +0 -17
  48. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  49. data/.github/workflows/stale-actions.yml +0 -24
  50. data/.github/workflows/test-ruby-head.yml +0 -31
  51. data/.github/workflows/test.yml +0 -32
  52. data/.gitignore +0 -30
  53. data/Gemfile +0 -9
  54. data/fluentd.gemspec +0 -62
  55. data/test/command/test_binlog_reader.rb +0 -362
  56. data/test/command/test_ca_generate.rb +0 -70
  57. data/test/command/test_cap_ctl.rb +0 -100
  58. data/test/command/test_cat.rb +0 -128
  59. data/test/command/test_ctl.rb +0 -56
  60. data/test/command/test_fluentd.rb +0 -1291
  61. data/test/command/test_plugin_config_formatter.rb +0 -397
  62. data/test/command/test_plugin_generator.rb +0 -109
  63. data/test/compat/test_calls_super.rb +0 -166
  64. data/test/compat/test_parser.rb +0 -92
  65. data/test/config/assertions.rb +0 -42
  66. data/test/config/test_config_parser.rb +0 -551
  67. data/test/config/test_configurable.rb +0 -1784
  68. data/test/config/test_configure_proxy.rb +0 -604
  69. data/test/config/test_dsl.rb +0 -415
  70. data/test/config/test_element.rb +0 -518
  71. data/test/config/test_literal_parser.rb +0 -309
  72. data/test/config/test_plugin_configuration.rb +0 -56
  73. data/test/config/test_section.rb +0 -191
  74. data/test/config/test_system_config.rb +0 -195
  75. data/test/config/test_types.rb +0 -408
  76. data/test/counter/test_client.rb +0 -563
  77. data/test/counter/test_error.rb +0 -44
  78. data/test/counter/test_mutex_hash.rb +0 -179
  79. data/test/counter/test_server.rb +0 -589
  80. data/test/counter/test_store.rb +0 -258
  81. data/test/counter/test_validator.rb +0 -137
  82. data/test/helper.rb +0 -155
  83. data/test/helpers/fuzzy_assert.rb +0 -89
  84. data/test/helpers/process_extenstion.rb +0 -33
  85. data/test/log/test_console_adapter.rb +0 -117
  86. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  87. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  88. data/test/plugin/data/2010/01/20100102.log +0 -0
  89. data/test/plugin/data/log/bar +0 -0
  90. data/test/plugin/data/log/foo/bar.log +0 -0
  91. data/test/plugin/data/log/foo/bar2 +0 -0
  92. data/test/plugin/data/log/test.log +0 -0
  93. data/test/plugin/data/log_numeric/01.log +0 -0
  94. data/test/plugin/data/log_numeric/02.log +0 -0
  95. data/test/plugin/data/log_numeric/12.log +0 -0
  96. data/test/plugin/data/log_numeric/14.log +0 -0
  97. data/test/plugin/data/sd_file/config +0 -11
  98. data/test/plugin/data/sd_file/config.json +0 -17
  99. data/test/plugin/data/sd_file/config.yaml +0 -11
  100. data/test/plugin/data/sd_file/config.yml +0 -11
  101. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  102. data/test/plugin/in_tail/test_fifo.rb +0 -121
  103. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  104. data/test/plugin/in_tail/test_position_file.rb +0 -346
  105. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  106. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  107. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  108. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  109. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  110. data/test/plugin/test_bare_output.rb +0 -131
  111. data/test/plugin/test_base.rb +0 -247
  112. data/test/plugin/test_buf_file.rb +0 -1314
  113. data/test/plugin/test_buf_file_single.rb +0 -898
  114. data/test/plugin/test_buf_memory.rb +0 -42
  115. data/test/plugin/test_buffer.rb +0 -1493
  116. data/test/plugin/test_buffer_chunk.rb +0 -209
  117. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  118. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  119. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  120. data/test/plugin/test_compressable.rb +0 -87
  121. data/test/plugin/test_file_util.rb +0 -96
  122. data/test/plugin/test_filter.rb +0 -368
  123. data/test/plugin/test_filter_grep.rb +0 -697
  124. data/test/plugin/test_filter_parser.rb +0 -731
  125. data/test/plugin/test_filter_record_transformer.rb +0 -577
  126. data/test/plugin/test_filter_stdout.rb +0 -207
  127. data/test/plugin/test_formatter_csv.rb +0 -136
  128. data/test/plugin/test_formatter_hash.rb +0 -38
  129. data/test/plugin/test_formatter_json.rb +0 -61
  130. data/test/plugin/test_formatter_ltsv.rb +0 -70
  131. data/test/plugin/test_formatter_msgpack.rb +0 -28
  132. data/test/plugin/test_formatter_out_file.rb +0 -116
  133. data/test/plugin/test_formatter_single_value.rb +0 -44
  134. data/test/plugin/test_formatter_tsv.rb +0 -76
  135. data/test/plugin/test_in_debug_agent.rb +0 -49
  136. data/test/plugin/test_in_exec.rb +0 -261
  137. data/test/plugin/test_in_forward.rb +0 -1178
  138. data/test/plugin/test_in_gc_stat.rb +0 -62
  139. data/test/plugin/test_in_http.rb +0 -1124
  140. data/test/plugin/test_in_monitor_agent.rb +0 -922
  141. data/test/plugin/test_in_object_space.rb +0 -66
  142. data/test/plugin/test_in_sample.rb +0 -190
  143. data/test/plugin/test_in_syslog.rb +0 -505
  144. data/test/plugin/test_in_tail.rb +0 -3429
  145. data/test/plugin/test_in_tcp.rb +0 -328
  146. data/test/plugin/test_in_udp.rb +0 -296
  147. data/test/plugin/test_in_unix.rb +0 -181
  148. data/test/plugin/test_input.rb +0 -137
  149. data/test/plugin/test_metadata.rb +0 -89
  150. data/test/plugin/test_metrics.rb +0 -294
  151. data/test/plugin/test_metrics_local.rb +0 -96
  152. data/test/plugin/test_multi_output.rb +0 -204
  153. data/test/plugin/test_out_copy.rb +0 -308
  154. data/test/plugin/test_out_exec.rb +0 -312
  155. data/test/plugin/test_out_exec_filter.rb +0 -606
  156. data/test/plugin/test_out_file.rb +0 -1038
  157. data/test/plugin/test_out_forward.rb +0 -1349
  158. data/test/plugin/test_out_http.rb +0 -557
  159. data/test/plugin/test_out_null.rb +0 -105
  160. data/test/plugin/test_out_relabel.rb +0 -28
  161. data/test/plugin/test_out_roundrobin.rb +0 -146
  162. data/test/plugin/test_out_secondary_file.rb +0 -458
  163. data/test/plugin/test_out_stdout.rb +0 -205
  164. data/test/plugin/test_out_stream.rb +0 -103
  165. data/test/plugin/test_output.rb +0 -1334
  166. data/test/plugin/test_output_as_buffered.rb +0 -2024
  167. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  168. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  169. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  170. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  171. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  172. data/test/plugin/test_output_as_standard.rb +0 -374
  173. data/test/plugin/test_owned_by.rb +0 -34
  174. data/test/plugin/test_parser.rb +0 -399
  175. data/test/plugin/test_parser_apache.rb +0 -42
  176. data/test/plugin/test_parser_apache2.rb +0 -47
  177. data/test/plugin/test_parser_apache_error.rb +0 -45
  178. data/test/plugin/test_parser_csv.rb +0 -200
  179. data/test/plugin/test_parser_json.rb +0 -244
  180. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  181. data/test/plugin/test_parser_msgpack.rb +0 -127
  182. data/test/plugin/test_parser_multiline.rb +0 -111
  183. data/test/plugin/test_parser_nginx.rb +0 -88
  184. data/test/plugin/test_parser_none.rb +0 -52
  185. data/test/plugin/test_parser_regexp.rb +0 -284
  186. data/test/plugin/test_parser_syslog.rb +0 -650
  187. data/test/plugin/test_parser_tsv.rb +0 -122
  188. data/test/plugin/test_sd_file.rb +0 -228
  189. data/test/plugin/test_sd_srv.rb +0 -230
  190. data/test/plugin/test_storage.rb +0 -166
  191. data/test/plugin/test_storage_local.rb +0 -335
  192. data/test/plugin/test_string_util.rb +0 -26
  193. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  195. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  196. data/test/plugin_helper/data/cert/cert.pem +0 -19
  197. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  198. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  199. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  200. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  201. data/test/plugin_helper/data/cert/empty.pem +0 -0
  202. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  203. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  204. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  205. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  206. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  207. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  208. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  209. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  210. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  211. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  212. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  213. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  214. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  215. data/test/plugin_helper/http_server/test_app.rb +0 -65
  216. data/test/plugin_helper/http_server/test_route.rb +0 -32
  217. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  218. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  219. data/test/plugin_helper/test_cert_option.rb +0 -25
  220. data/test/plugin_helper/test_child_process.rb +0 -862
  221. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  222. data/test/plugin_helper/test_event_emitter.rb +0 -80
  223. data/test/plugin_helper/test_event_loop.rb +0 -52
  224. data/test/plugin_helper/test_extract.rb +0 -194
  225. data/test/plugin_helper/test_formatter.rb +0 -255
  226. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  227. data/test/plugin_helper/test_inject.rb +0 -561
  228. data/test/plugin_helper/test_metrics.rb +0 -137
  229. data/test/plugin_helper/test_parser.rb +0 -264
  230. data/test/plugin_helper/test_record_accessor.rb +0 -238
  231. data/test/plugin_helper/test_retry_state.rb +0 -1006
  232. data/test/plugin_helper/test_server.rb +0 -1895
  233. data/test/plugin_helper/test_service_discovery.rb +0 -165
  234. data/test/plugin_helper/test_socket.rb +0 -146
  235. data/test/plugin_helper/test_storage.rb +0 -542
  236. data/test/plugin_helper/test_thread.rb +0 -164
  237. data/test/plugin_helper/test_timer.rb +0 -130
  238. data/test/scripts/exec_script.rb +0 -32
  239. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  240. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  241. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  242. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  243. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  244. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  245. data/test/test_capability.rb +0 -74
  246. data/test/test_clock.rb +0 -164
  247. data/test/test_config.rb +0 -369
  248. data/test/test_configdsl.rb +0 -148
  249. data/test/test_daemonizer.rb +0 -91
  250. data/test/test_engine.rb +0 -203
  251. data/test/test_event.rb +0 -531
  252. data/test/test_event_router.rb +0 -348
  253. data/test/test_event_time.rb +0 -199
  254. data/test/test_file_wrapper.rb +0 -53
  255. data/test/test_filter.rb +0 -121
  256. data/test/test_fluent_log_event_router.rb +0 -99
  257. data/test/test_formatter.rb +0 -369
  258. data/test/test_input.rb +0 -31
  259. data/test/test_log.rb +0 -1076
  260. data/test/test_match.rb +0 -148
  261. data/test/test_mixin.rb +0 -351
  262. data/test/test_msgpack_factory.rb +0 -50
  263. data/test/test_oj_options.rb +0 -55
  264. data/test/test_output.rb +0 -278
  265. data/test/test_plugin.rb +0 -251
  266. data/test/test_plugin_classes.rb +0 -370
  267. data/test/test_plugin_helper.rb +0 -81
  268. data/test/test_plugin_id.rb +0 -119
  269. data/test/test_process.rb +0 -14
  270. data/test/test_root_agent.rb +0 -951
  271. data/test/test_static_config_analysis.rb +0 -177
  272. data/test/test_supervisor.rb +0 -821
  273. data/test/test_test_drivers.rb +0 -136
  274. data/test/test_time_formatter.rb +0 -301
  275. data/test/test_time_parser.rb +0 -362
  276. data/test/test_tls.rb +0 -65
  277. data/test/test_unique_id.rb +0 -47
  278. data/test/test_variable_store.rb +0 -65
@@ -1,1038 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/output'
3
- require 'fluent/plugin/out_file'
4
- require 'fileutils'
5
- require 'time'
6
- require 'timecop'
7
- require 'zlib'
8
- require 'fluent/file_wrapper'
9
-
10
- class FileOutputTest < Test::Unit::TestCase
11
- def setup
12
- Fluent::Test.setup
13
- FileUtils.rm_rf(TMP_DIR)
14
- FileUtils.mkdir_p(TMP_DIR)
15
- @default_newline = if Fluent.windows?
16
- "\r\n"
17
- else
18
- "\n"
19
- end
20
- end
21
-
22
- TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/out_file#{ENV['TEST_ENV_NUMBER']}")
23
-
24
- CONFIG = %[
25
- path #{TMP_DIR}/out_file_test
26
- compress gz
27
- utc
28
- <buffer>
29
- timekey_use_utc true
30
- </buffer>
31
- ]
32
-
33
- def create_driver(conf = CONFIG, opts = {})
34
- Fluent::Test::Driver::Output.new(Fluent::Plugin::FileOutput, opts: opts).configure(conf)
35
- end
36
-
37
- sub_test_case 'configuration' do
38
- test 'basic configuration' do
39
- d = create_driver %[
40
- path test_path
41
- compress gz
42
- ]
43
- assert_equal 'test_path', d.instance.path
44
- assert_equal :gz, d.instance.compress
45
- assert_equal :gzip, d.instance.instance_eval{ @compress_method }
46
- end
47
-
48
- test 'using root_dir for buffer path' do
49
- system_conf_opts = {'root_dir' => File.join(TMP_DIR, 'testrootdir')}
50
- buf_conf = config_element('buffer', '', {'flush_interval' => '1s'})
51
- conf = config_element('match', '**', {'@id' => 'myout', 'path' => 'test_path', 'append' => 'true'}, [buf_conf])
52
- d = create_driver(conf, system_conf_opts)
53
-
54
- assert_equal 'test_path', d.instance.path
55
- assert d.instance.append
56
-
57
- assert d.instance.buffer.respond_to?(:path) # file buffer
58
- assert_equal 1, d.instance.buffer_config.flush_interval
59
-
60
- assert_equal File.join(TMP_DIR, 'testrootdir', 'worker0', 'myout'), d.instance.plugin_root_dir
61
-
62
- buffer_path_under_root_dir = File.join(TMP_DIR, 'testrootdir', 'worker0', 'myout', 'buffer', 'buffer.*.log')
63
- assert_equal buffer_path_under_root_dir, d.instance.buffer.path
64
- end
65
-
66
- test 'path should be writable' do
67
- assert_raise(Fluent::ConfigError.new("'path' parameter is required")) do
68
- create_driver ""
69
- end
70
-
71
- assert_nothing_raised do
72
- create_driver %[path #{TMP_DIR}/test_path]
73
- end
74
-
75
- assert_nothing_raised do
76
- FileUtils.mkdir_p("#{TMP_DIR}/test_dir")
77
- File.chmod(0777, "#{TMP_DIR}/test_dir")
78
- create_driver %[path #{TMP_DIR}/test_dir/foo/bar/baz]
79
- end
80
-
81
- if Process.uid.nonzero?
82
- assert_raise(Fluent::ConfigError) do
83
- FileUtils.mkdir_p("#{TMP_DIR}/test_dir")
84
- File.chmod(0555, "#{TMP_DIR}/test_dir")
85
- create_driver %[path #{TMP_DIR}/test_dir/foo/bar/baz]
86
- end
87
- end
88
- end
89
-
90
- test 'default timezone is localtime' do
91
- d = create_driver(%[path #{TMP_DIR}/out_file_test])
92
- time = event_time("2011-01-02 13:14:15 UTC")
93
-
94
- with_timezone(Fluent.windows? ? 'NST-8' : 'Asia/Taipei') do
95
- d.run(default_tag: 'test') do
96
- d.feed(time, {"a"=>1})
97
- end
98
- end
99
- assert_equal 1, d.formatted.size
100
- assert_equal %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
101
- end
102
-
103
- test 'no configuration error raised for basic configuration using "*" (v0.12 style)' do
104
- conf = config_element('match', '**', {
105
- 'path' => "#{TMP_DIR}/test_out.*.log",
106
- 'time_slice_format' => '%Y%m%d',
107
- })
108
- assert_nothing_raised do
109
- create_driver(conf)
110
- end
111
- end
112
-
113
- if Process.uid.nonzero?
114
- test 'configuration error raised if specified directory via template is not writable' do
115
- Timecop.freeze(Time.parse("2016-10-04 21:33:27 UTC")) do
116
- conf = config_element('match', '**', {
117
- 'path' => "#{TMP_DIR}/prohibited/${tag}/file.%Y%m%d.log",
118
- }, [ config_element('buffer', 'time,tag', {'timekey' => 86400, 'timekey_zone' => '+0000'}) ])
119
- FileUtils.mkdir_p("#{TMP_DIR}/prohibited")
120
- File.chmod(0555, "#{TMP_DIR}/prohibited")
121
- assert_raise Fluent::ConfigError.new("out_file: `#{TMP_DIR}/prohibited/a/file.20161004.log_**.log` is not writable") do
122
- create_driver(conf)
123
- end
124
- end
125
- end
126
- end
127
-
128
- test 'configuration using inject/format/buffer sections fully' do
129
- conf = config_element('match', '**', {
130
- 'path' => "#{TMP_DIR}/${tag}/${type}/conf_test.%Y%m%d.%H%M.log",
131
- 'add_path_suffix' => 'false',
132
- 'append' => "true",
133
- 'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
134
- 'compress' => 'gzip',
135
- 'recompress' => 'true',
136
- }, [
137
- config_element('inject', '', {
138
- 'hostname_key' => 'hostname',
139
- 'hostname' => 'testing.local',
140
- 'tag_key' => 'tag',
141
- 'time_key' => 'time',
142
- 'time_type' => 'string',
143
- 'time_format' => '%Y/%m/%d %H:%M:%S %z',
144
- 'timezone' => '+0900',
145
- }),
146
- config_element('format', '', {
147
- '@type' => 'out_file',
148
- 'include_tag' => 'true',
149
- 'include_time' => 'true',
150
- 'delimiter' => 'COMMA',
151
- 'time_type' => 'string',
152
- 'time_format' => '%Y-%m-%d %H:%M:%S %z',
153
- 'utc' => 'true',
154
- }),
155
- config_element('buffer', 'time,tag,type', {
156
- '@type' => 'file',
157
- 'timekey' => '15m',
158
- 'timekey_wait' => '5s',
159
- 'timekey_zone' => '+0000',
160
- 'path' => "#{TMP_DIR}/buf_conf_test",
161
- 'chunk_limit_size' => '50m',
162
- 'total_limit_size' => '1g',
163
- 'compress' => 'gzip',
164
- }),
165
- ])
166
- assert_nothing_raised do
167
- create_driver(conf)
168
- end
169
- end
170
-
171
- test 'configured as secondary with primary using chunk_key_tag and not using chunk_key_time' do
172
- require 'fluent/plugin/out_null'
173
- conf = config_element('match', '**', {
174
- }, [
175
- config_element('buffer', 'tag', {
176
- }),
177
- config_element('secondary', '', {
178
- '@type' => 'file',
179
- 'path' => "#{TMP_DIR}/testing_to_dump_by_out_file",
180
- }),
181
- ])
182
- assert_nothing_raised do
183
- Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)
184
- end
185
- end
186
- end
187
-
188
- sub_test_case 'fully configured output' do
189
- setup do
190
- Timecop.freeze(Time.parse("2016-10-03 23:58:00 UTC"))
191
- conf = config_element('match', '**', {
192
- 'path' => "#{TMP_DIR}/${tag}/${type}/full.%Y%m%d.%H%M.log",
193
- 'add_path_suffix' => 'false',
194
- 'append' => "true",
195
- 'symlink_path' => "#{TMP_DIR}/full.current.log",
196
- 'compress' => 'gzip',
197
- 'recompress' => 'true',
198
- }, [
199
- config_element('inject', '', {
200
- 'hostname_key' => 'hostname',
201
- 'hostname' => 'testing.local',
202
- 'tag_key' => 'tag',
203
- 'time_key' => 'time',
204
- 'time_type' => 'string',
205
- 'time_format' => '%Y/%m/%d %H:%M:%S %z',
206
- 'timezone' => '+0900',
207
- }),
208
- config_element('format', '', {
209
- '@type' => 'out_file',
210
- 'include_tag' => 'true',
211
- 'include_time' => 'true',
212
- 'delimiter' => 'COMMA',
213
- 'time_type' => 'string',
214
- 'time_format' => '%Y-%m-%d %H:%M:%S %z',
215
- 'utc' => 'true',
216
- }),
217
- config_element('buffer', 'time,tag,type', {
218
- '@type' => 'file',
219
- 'timekey' => '15m',
220
- 'timekey_wait' => '5s',
221
- 'timekey_zone' => '+0000',
222
- 'path' => "#{TMP_DIR}/buf_full",
223
- 'chunk_limit_size' => '50m',
224
- 'total_limit_size' => '1g',
225
- 'compress' => 'gzip',
226
- }),
227
- ])
228
- @d = create_driver(conf)
229
- end
230
-
231
- teardown do
232
- FileUtils.rm_rf("#{TMP_DIR}/buf_full")
233
- FileUtils.rm_rf("#{TMP_DIR}/my.data")
234
- FileUtils.rm_rf("#{TMP_DIR}/your.data")
235
- FileUtils.rm_rf("#{TMP_DIR}/full.current.log")
236
- Timecop.return
237
- end
238
-
239
- test 'can format/write data correctly' do
240
- d = @d
241
-
242
- assert_equal 50*1024*1024, d.instance.buffer.chunk_limit_size
243
- assert_equal 1*1024*1024*1024, d.instance.buffer.total_limit_size
244
-
245
- assert !(File.symlink?("#{TMP_DIR}/full.current.log"))
246
-
247
- t1 = event_time("2016-10-03 23:58:09 UTC")
248
- t2 = event_time("2016-10-03 23:59:33 UTC")
249
- t3 = event_time("2016-10-03 23:59:57 UTC")
250
- t4 = event_time("2016-10-04 00:00:17 UTC")
251
- t5 = event_time("2016-10-04 00:01:59 UTC")
252
-
253
- Timecop.freeze(Time.parse("2016-10-03 23:58:30 UTC"))
254
-
255
- d.run(start: true, flush: false, shutdown: false) do
256
- d.feed('my.data', t1, {"type" => "a", "message" => "data raw content"})
257
- d.feed('my.data', t2, {"type" => "a", "message" => "data raw content"})
258
- d.feed('your.data', t3, {"type" => "a", "message" => "data raw content"})
259
- end
260
-
261
- assert_equal 3, d.formatted.size
262
-
263
- assert Dir.exist?("#{TMP_DIR}/buf_full")
264
- assert !(Dir.exist?("#{TMP_DIR}/my.data/a"))
265
- assert !(Dir.exist?("#{TMP_DIR}/your.data/a"))
266
- buffer_files = Dir.entries("#{TMP_DIR}/buf_full").reject{|e| e =~ /^\.+$/ }
267
- assert_equal 2, buffer_files.count{|n| n.end_with?('.meta') }
268
- assert_equal 2, buffer_files.count{|n| !n.end_with?('.meta') }
269
-
270
- m1 = d.instance.metadata('my.data', t1, {"type" => "a"})
271
- m2 = d.instance.metadata('your.data', t3, {"type" => "a"})
272
-
273
- assert_equal 2, d.instance.buffer.stage.size
274
- b1_path = d.instance.buffer.stage[m1].path
275
- b1_size = File.lstat(b1_path).size
276
-
277
- unless Fluent.windows?
278
- assert File.symlink?("#{TMP_DIR}/full.current.log")
279
- assert_equal d.instance.buffer.stage[m2].path, File.readlink("#{TMP_DIR}/full.current.log")
280
- end
281
-
282
- Timecop.freeze(Time.parse("2016-10-04 00:00:06 UTC"))
283
-
284
- d.run(start: false, flush: true, shutdown: true) do
285
- d.feed('my.data', t4, {"type" => "a", "message" => "data raw content"})
286
- d.feed('your.data', t5, {"type" => "a", "message" => "data raw content"})
287
- end
288
-
289
- assert Dir.exist?("#{TMP_DIR}/buf_full")
290
- assert Dir.exist?("#{TMP_DIR}/my.data/a")
291
- assert Dir.exist?("#{TMP_DIR}/your.data/a")
292
-
293
- buffer_files = Dir.entries("#{TMP_DIR}/buf_full").reject{|e| e =~ /^\.+$/ }
294
- assert_equal 0, buffer_files.size
295
-
296
- assert File.exist?("#{TMP_DIR}/my.data/a/full.20161003.2345.log.gz")
297
- assert File.exist?("#{TMP_DIR}/my.data/a/full.20161004.0000.log.gz")
298
- assert File.exist?("#{TMP_DIR}/your.data/a/full.20161003.2345.log.gz")
299
- assert File.exist?("#{TMP_DIR}/your.data/a/full.20161004.0000.log.gz")
300
-
301
- assert{ File.lstat("#{TMP_DIR}/my.data/a/full.20161003.2345.log.gz").size < b1_size } # recompress
302
-
303
- assert_equal 5, d.formatted.size
304
-
305
- r1 = %!2016-10-03 23:58:09 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 08:58:09 +0900"}#{@default_newline}!
306
- r2 = %!2016-10-03 23:59:33 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 08:59:33 +0900"}#{@default_newline}!
307
- r3 = %!2016-10-03 23:59:57 +0000,your.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"your.data","time":"2016/10/04 08:59:57 +0900"}#{@default_newline}!
308
- r4 = %!2016-10-04 00:00:17 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 09:00:17 +0900"}#{@default_newline}!
309
- r5 = %!2016-10-04 00:01:59 +0000,your.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"your.data","time":"2016/10/04 09:01:59 +0900"}#{@default_newline}!
310
- assert_equal r1, d.formatted[0]
311
- assert_equal r2, d.formatted[1]
312
- assert_equal r3, d.formatted[2]
313
- assert_equal r4, d.formatted[3]
314
- assert_equal r5, d.formatted[4]
315
-
316
- read_gunzip = ->(path){
317
- File.open(path){ |fio|
318
- Zlib::GzipReader.new(StringIO.new(fio.read)).read
319
- }
320
- }
321
- assert_equal r1 + r2, read_gunzip.call("#{TMP_DIR}/my.data/a/full.20161003.2345.log.gz")
322
- assert_equal r3, read_gunzip.call("#{TMP_DIR}/your.data/a/full.20161003.2345.log.gz")
323
- assert_equal r4, read_gunzip.call("#{TMP_DIR}/my.data/a/full.20161004.0000.log.gz")
324
- assert_equal r5, read_gunzip.call("#{TMP_DIR}/your.data/a/full.20161004.0000.log.gz")
325
- end
326
- end
327
-
328
- sub_test_case 'format' do
329
- test 'timezone UTC specified' do
330
- d = create_driver
331
-
332
- time = event_time("2011-01-02 13:14:15 UTC")
333
- d.run(default_tag: 'test') do
334
- d.feed(time, {"a"=>1})
335
- d.feed(time, {"a"=>2})
336
- end
337
- assert_equal 2, d.formatted.size
338
- assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
339
- assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}], d.formatted[1]
340
- end
341
-
342
- test 'time formatted with specified timezone, using area name' do
343
- d = create_driver %[
344
- path #{TMP_DIR}/out_file_test
345
- timezone Asia/Taipei
346
- ]
347
-
348
- time = event_time("2011-01-02 13:14:15 UTC")
349
- d.run(default_tag: 'test') do
350
- d.feed(time, {"a"=>1})
351
- end
352
- assert_equal 1, d.formatted.size
353
- assert_equal %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
354
- end
355
-
356
- test 'time formatted with specified timezone, using offset' do
357
- d = create_driver %[
358
- path #{TMP_DIR}/out_file_test
359
- timezone -03:30
360
- ]
361
-
362
- time = event_time("2011-01-02 13:14:15 UTC")
363
- d.run(default_tag: 'test') do
364
- d.feed(time, {"a"=>1})
365
- end
366
- assert_equal 1, d.formatted.size
367
- assert_equal %[2011-01-02T09:44:15-03:30\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
368
- end
369
-
370
- test 'configuration error raised for invalid timezone' do
371
- assert_raise(Fluent::ConfigError) do
372
- create_driver %[
373
- path #{TMP_DIR}/out_file_test
374
- timezone Invalid/Invalid
375
- ]
376
- end
377
- end
378
- end
379
-
380
- def check_gzipped_result(path, expect)
381
- # Zlib::GzipReader has a bug of concatenated file: https://bugs.ruby-lang.org/issues/9790
382
- # Following code from https://www.ruby-forum.com/topic/971591#979520
383
- result = ''
384
- File.open(path, "rb") { |io|
385
- loop do
386
- gzr = Zlib::GzipReader.new(StringIO.new(io.read))
387
- result << gzr.read
388
- unused = gzr.unused
389
- gzr.finish
390
- break if unused.nil?
391
- io.pos -= unused.length
392
- end
393
- }
394
-
395
- assert_equal expect, result
396
- end
397
-
398
- def check_result(path, expect)
399
- result = File.read(path, mode: "rb")
400
- assert_equal expect, result
401
- end
402
-
403
- sub_test_case 'write' do
404
- test 'basic case' do
405
- d = create_driver
406
-
407
- assert_false File.exist?("#{TMP_DIR}/out_file_test.20110102_0.log.gz")
408
-
409
- time = event_time("2011-01-02 13:14:15 UTC")
410
- d.run(default_tag: 'test') do
411
- d.feed(time, {"a"=>1})
412
- d.feed(time, {"a"=>2})
413
- end
414
-
415
- assert File.exist?("#{TMP_DIR}/out_file_test.20110102_0.log.gz")
416
- check_gzipped_result("#{TMP_DIR}/out_file_test.20110102_0.log.gz", %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}])
417
- end
418
- end
419
-
420
- sub_test_case 'file/directory permissions' do
421
- TMP_DIR_WITH_SYSTEM = File.expand_path(File.dirname(__FILE__) + "/../tmp/out_file_system#{ENV['TEST_ENV_NUMBER']}")
422
- # 0750 interprets as "488". "488".to_i(8) # => 4. So, it makes wrong permission. Umm....
423
- OVERRIDE_DIR_PERMISSION = 750
424
- OVERRIDE_FILE_PERMISSION = 0620
425
- CONFIG_WITH_SYSTEM = %[
426
- path #{TMP_DIR_WITH_SYSTEM}/out_file_test
427
- compress gz
428
- utc
429
- <buffer>
430
- timekey_use_utc true
431
- </buffer>
432
- <system>
433
- file_permission #{OVERRIDE_FILE_PERMISSION}
434
- dir_permission #{OVERRIDE_DIR_PERMISSION}
435
- </system>
436
- ]
437
-
438
- setup do
439
- omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
440
- FileUtils.rm_rf(TMP_DIR_WITH_SYSTEM)
441
- end
442
-
443
- def parse_system(text)
444
- basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
445
- Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
446
- end
447
-
448
- test 'write to file with permission specifications' do
449
- system_conf = parse_system(CONFIG_WITH_SYSTEM)
450
- sc = Fluent::SystemConfig.new(system_conf)
451
- Fluent::Engine.init(sc)
452
- d = create_driver CONFIG_WITH_SYSTEM
453
-
454
- assert_false File.exist?("#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz")
455
-
456
- time = event_time("2011-01-02 13:14:15 UTC")
457
- d.run(default_tag: 'test') do
458
- d.feed(time, {"a"=>1})
459
- d.feed(time, {"a"=>2})
460
- end
461
-
462
- assert File.exist?("#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz")
463
-
464
- check_gzipped_result("#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz", %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
465
- dir_mode = "%o" % File::stat(TMP_DIR_WITH_SYSTEM).mode
466
- assert_equal(OVERRIDE_DIR_PERMISSION, dir_mode[-3, 3].to_i)
467
- file_mode = "%o" % File::stat("#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz").mode
468
- assert_equal(OVERRIDE_FILE_PERMISSION, file_mode[-3, 3].to_i)
469
- end
470
- end
471
-
472
- sub_test_case 'format specified' do
473
- test 'json' do
474
- d = create_driver [CONFIG, 'format json', 'include_time_key true', 'time_as_epoch'].join("\n")
475
-
476
- time = event_time("2011-01-02 13:14:15 UTC")
477
- d.run(default_tag: 'test') do
478
- d.feed(time, {"a"=>1})
479
- d.feed(time, {"a"=>2})
480
- end
481
-
482
- path = d.instance.last_written_path
483
- check_gzipped_result(path, %[#{Yajl.dump({"a" => 1, 'time' => time.to_i})}#{@default_newline}] + %[#{Yajl.dump({"a" => 2, 'time' => time.to_i})}#{@default_newline}])
484
- end
485
-
486
- test 'ltsv' do
487
- d = create_driver [CONFIG, 'format ltsv', 'include_time_key true'].join("\n")
488
-
489
- time = event_time("2011-01-02 13:14:15 UTC")
490
- d.run(default_tag: 'test') do
491
- d.feed(time, {"a"=>1})
492
- d.feed(time, {"a"=>2})
493
- end
494
-
495
- path = d.instance.last_written_path
496
- check_gzipped_result(path, %[a:1\ttime:2011-01-02T13:14:15Z#{@default_newline}] + %[a:2\ttime:2011-01-02T13:14:15Z#{@default_newline}])
497
- end
498
-
499
- test 'single_value' do
500
- d = create_driver [CONFIG, 'format single_value', 'message_key a'].join("\n")
501
-
502
- time = event_time("2011-01-02 13:14:15 UTC")
503
- d.run(default_tag: 'test') do
504
- d.feed(time, {"a"=>1})
505
- d.feed(time, {"a"=>2})
506
- end
507
-
508
- path = d.instance.last_written_path
509
- check_gzipped_result(path, %[1#{@default_newline}] + %[2#{@default_newline}])
510
- end
511
- end
512
-
513
- test 'path with index number' do
514
- time = event_time("2011-01-02 13:14:15 UTC")
515
- formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}]
516
-
517
- write_once = ->(){
518
- d = create_driver
519
- d.run(default_tag: 'test'){
520
- d.feed(time, {"a"=>1})
521
- d.feed(time, {"a"=>2})
522
- }
523
- d.instance.last_written_path
524
- }
525
-
526
- assert !File.exist?("#{TMP_DIR}/out_file_test.20110102_0.log.gz")
527
-
528
- path = write_once.call
529
- assert_equal "#{TMP_DIR}/out_file_test.20110102_0.log.gz", path
530
- check_gzipped_result(path, formatted_lines)
531
- assert_equal 1, Dir.glob("#{TMP_DIR}/out_file_test.*").size
532
-
533
- path = write_once.call
534
- assert_equal "#{TMP_DIR}/out_file_test.20110102_1.log.gz", path
535
- check_gzipped_result(path, formatted_lines)
536
- assert_equal 2, Dir.glob("#{TMP_DIR}/out_file_test.*").size
537
-
538
- path = write_once.call
539
- assert_equal "#{TMP_DIR}/out_file_test.20110102_2.log.gz", path
540
- check_gzipped_result(path, formatted_lines)
541
- assert_equal 3, Dir.glob("#{TMP_DIR}/out_file_test.*").size
542
- end
543
-
544
- data(
545
- "with compression" => true,
546
- "without compression" => false,
547
- )
548
- test 'append' do |compression|
549
- time = event_time("2011-01-02 13:14:15 UTC")
550
- formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}]
551
-
552
- write_once = ->(){
553
- config = %[
554
- path #{TMP_DIR}/out_file_test
555
- utc
556
- append true
557
- <buffer>
558
- timekey_use_utc true
559
- </buffer>
560
- ]
561
- if compression
562
- config << " compress gz"
563
- end
564
- d = create_driver(config)
565
- d.run(default_tag: 'test'){
566
- d.feed(time, {"a"=>1})
567
- d.feed(time, {"a"=>2})
568
- }
569
- d.instance.last_written_path
570
- }
571
-
572
- log_file_name = "out_file_test.20110102.log"
573
- if compression
574
- log_file_name << ".gz"
575
- end
576
-
577
- 1.upto(3) do |i|
578
- path = write_once.call
579
- assert_equal "#{TMP_DIR}/#{log_file_name}", path
580
- expect = formatted_lines * i
581
- if compression
582
- check_gzipped_result(path, expect)
583
- else
584
- check_result(path, expect)
585
- end
586
- end
587
- end
588
-
589
- test 'append when JST' do
590
- with_timezone(Fluent.windows? ? "JST-9" : "Asia/Tokyo") do
591
- time = event_time("2011-01-02 03:14:15+09:00")
592
- formatted_lines = %[2011-01-02T03:14:15+09:00\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T03:14:15+09:00\ttest\t{"a":2}#{@default_newline}]
593
-
594
- write_once = ->(){
595
- d = create_driver %[
596
- path #{TMP_DIR}/out_file_test
597
- compress gz
598
- append true
599
- <buffer>
600
- timekey_use_utc false
601
- timekey_zone Asia/Tokyo
602
- </buffer>
603
- ]
604
- d.run(default_tag: 'test'){
605
- d.feed(time, {"a"=>1})
606
- d.feed(time, {"a"=>2})
607
- }
608
- d.instance.last_written_path
609
- }
610
-
611
- path = write_once.call
612
- assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
613
- check_gzipped_result(path, formatted_lines)
614
-
615
- path = write_once.call
616
- assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
617
- check_gzipped_result(path, formatted_lines * 2)
618
-
619
- path = write_once.call
620
- assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
621
- check_gzipped_result(path, formatted_lines * 3)
622
- end
623
- end
624
-
625
- test 'append when UTC-02 but timekey_zone is +0900' do
626
- with_timezone("UTC-02") do # +0200
627
- time = event_time("2011-01-02 17:14:15+02:00")
628
- formatted_lines = %[2011-01-02T17:14:15+02:00\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T17:14:15+02:00\ttest\t{"a":2}#{@default_newline}]
629
-
630
- write_once = ->(){
631
- d = create_driver %[
632
- path #{TMP_DIR}/out_file_test
633
- compress gz
634
- append true
635
- <buffer>
636
- timekey_use_utc false
637
- timekey_zone +0900
638
- </buffer>
639
- ]
640
- d.run(default_tag: 'test'){
641
- d.feed(time, {"a"=>1})
642
- d.feed(time, {"a"=>2})
643
- }
644
- d.instance.last_written_path
645
- }
646
-
647
- path = write_once.call
648
- # Rotated at 2011-01-02 17:00:00+02:00
649
- assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
650
- check_gzipped_result(path, formatted_lines)
651
-
652
- path = write_once.call
653
- assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
654
- check_gzipped_result(path, formatted_lines * 2)
655
-
656
- path = write_once.call
657
- assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
658
- check_gzipped_result(path, formatted_lines * 3)
659
- end
660
- end
661
-
662
- test '${chunk_id}' do
663
- time = event_time("2011-01-02 13:14:15 UTC")
664
-
665
- write_once = ->(){
666
- d = create_driver %[
667
- path #{TMP_DIR}/out_file_chunk_id_${chunk_id}
668
- utc
669
- append true
670
- <buffer>
671
- timekey_use_utc true
672
- </buffer>
673
- ]
674
- d.run(default_tag: 'test'){
675
- d.feed(time, {"a"=>1})
676
- d.feed(time, {"a"=>2})
677
- }
678
- d.instance.last_written_path
679
- }
680
-
681
- path = write_once.call
682
- if File.basename(path) =~ /out_file_chunk_id_([-_.@a-zA-Z0-9].*).20110102.log/
683
- unique_id = Fluent::UniqueId.hex(Fluent::UniqueId.generate)
684
- assert_equal unique_id.size, $1.size, "chunk_id size is mismatched"
685
- else
686
- flunk "chunk_id is not included in the path"
687
- end
688
- end
689
-
690
- SYMLINK_PATH = File.expand_path("#{TMP_DIR}/current")
691
-
692
- sub_test_case 'symlink' do
693
- test 'static symlink' do
694
- omit "Windows doesn't support symlink" if Fluent.windows?
695
- conf = CONFIG + %[
696
- symlink_path #{SYMLINK_PATH}
697
- ]
698
- symlink_path = "#{SYMLINK_PATH}"
699
-
700
- d = create_driver(conf)
701
- begin
702
- run_and_check(d, symlink_path)
703
- ensure
704
- FileUtils.rm_rf(symlink_path)
705
- end
706
- end
707
-
708
- test 'symlink with placeholders' do
709
- omit "Windows doesn't support symlink" if Fluent.windows?
710
- conf = %[
711
- path #{TMP_DIR}/${tag}/out_file_test
712
- symlink_path #{SYMLINK_PATH}/foo/${tag}
713
- <buffer tag,time>
714
- </buffer>
715
- ]
716
- symlink_path = "#{SYMLINK_PATH}/foo/tag"
717
-
718
- d = create_driver(conf)
719
- begin
720
- run_and_check(d, symlink_path)
721
- ensure
722
- FileUtils.rm_rf(symlink_path)
723
- end
724
- end
725
-
726
- def run_and_check(d, symlink_path)
727
- d.run(default_tag: 'tag') do
728
- es = Fluent::OneEventStream.new(event_time("2011-01-02 13:14:15 UTC"), {"a"=>1})
729
- d.feed(es)
730
-
731
- assert File.symlink?(symlink_path)
732
- assert File.exist?(symlink_path) # This checks dest of symlink exists or not.
733
-
734
- es = Fluent::OneEventStream.new(event_time("2011-01-03 14:15:16 UTC"), {"a"=>2})
735
- d.feed(es)
736
-
737
- assert File.symlink?(symlink_path)
738
- assert File.exist?(symlink_path)
739
-
740
- meta = d.instance.metadata('tag', event_time("2011-01-03 14:15:16 UTC"), {})
741
- assert_equal d.instance.buffer.instance_eval{ @stage[meta].path }, File.readlink(symlink_path)
742
- end
743
- end
744
- end
745
-
746
- sub_test_case 'path' do
747
- test 'normal' do
748
- d = create_driver(%[
749
- path #{TMP_DIR}/out_file_test
750
- time_slice_format %Y-%m-%d-%H
751
- utc true
752
- ])
753
- time = event_time("2011-01-02 13:14:15 UTC")
754
- d.run(default_tag: 'test') do
755
- d.feed(time, {"a"=>1})
756
- end
757
- path = d.instance.last_written_path
758
- assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13_0.log", path
759
- end
760
-
761
- test 'normal with append' do
762
- d = create_driver(%[
763
- path #{TMP_DIR}/out_file_test
764
- time_slice_format %Y-%m-%d-%H
765
- utc true
766
- append true
767
- ])
768
- time = event_time("2011-01-02 13:14:15 UTC")
769
- d.run(default_tag: 'test') do
770
- d.feed(time, {"a"=>1})
771
- end
772
- path = d.instance.last_written_path
773
- assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13.log", path
774
- end
775
-
776
- test '*' do
777
- d = create_driver(%[
778
- path #{TMP_DIR}/out_file_test.*.txt
779
- time_slice_format %Y-%m-%d-%H
780
- utc true
781
- ])
782
- time = event_time("2011-01-02 13:14:15 UTC")
783
- d.run(default_tag: 'test') do
784
- d.feed(time, {"a"=>1})
785
- end
786
- path = d.instance.last_written_path
787
- assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13_0.txt", path
788
- end
789
-
790
- test '* with append' do
791
- d = create_driver(%[
792
- path #{TMP_DIR}/out_file_test.*.txt
793
- time_slice_format %Y-%m-%d-%H
794
- utc true
795
- append true
796
- ])
797
- time = event_time("2011-01-02 13:14:15 UTC")
798
- d.run(default_tag: 'test') do
799
- d.feed(time, {"a"=>1})
800
- end
801
- path = d.instance.last_written_path
802
- assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13.txt", path
803
- end
804
- end
805
-
806
- sub_test_case '#timekey_to_timeformat' do
807
- setup do
808
- @d = create_driver
809
- @i = @d.instance
810
- end
811
-
812
- test 'returns empty string for nil' do
813
- assert_equal '', @i.timekey_to_timeformat(nil)
814
- end
815
-
816
- test 'returns timestamp string with seconds for timekey smaller than 60' do
817
- assert_equal '%Y%m%d%H%M%S', @i.timekey_to_timeformat(1)
818
- assert_equal '%Y%m%d%H%M%S', @i.timekey_to_timeformat(30)
819
- assert_equal '%Y%m%d%H%M%S', @i.timekey_to_timeformat(59)
820
- end
821
-
822
- test 'returns timestamp string with minutes for timekey smaller than 3600' do
823
- assert_equal '%Y%m%d%H%M', @i.timekey_to_timeformat(60)
824
- assert_equal '%Y%m%d%H%M', @i.timekey_to_timeformat(180)
825
- assert_equal '%Y%m%d%H%M', @i.timekey_to_timeformat(1800)
826
- assert_equal '%Y%m%d%H%M', @i.timekey_to_timeformat(3599)
827
- end
828
-
829
- test 'returns timestamp string with hours for timekey smaller than 86400 (1 day)' do
830
- assert_equal '%Y%m%d%H', @i.timekey_to_timeformat(3600)
831
- assert_equal '%Y%m%d%H', @i.timekey_to_timeformat(7200)
832
- assert_equal '%Y%m%d%H', @i.timekey_to_timeformat(86399)
833
- end
834
-
835
- test 'returns timestamp string with days for timekey equal or greater than 86400' do
836
- assert_equal '%Y%m%d', @i.timekey_to_timeformat(86400)
837
- assert_equal '%Y%m%d', @i.timekey_to_timeformat(1000000)
838
- assert_equal '%Y%m%d', @i.timekey_to_timeformat(1000000000)
839
- end
840
- end
841
-
842
- sub_test_case '#compression_suffix' do
843
- setup do
844
- @i = create_driver.instance
845
- end
846
-
847
- test 'returns empty string for nil (no compression method specified)' do
848
- assert_equal '', @i.compression_suffix(nil)
849
- end
850
-
851
- test 'returns .gz for gzip' do
852
- assert_equal '.gz', @i.compression_suffix(:gzip)
853
- end
854
- end
855
-
856
- sub_test_case '#generate_path_template' do
857
- setup do
858
- @i = create_driver.instance
859
- end
860
-
861
- data(
862
- 'day' => [86400, '%Y%m%d', '%Y-%m-%d'],
863
- 'hour' => [3600, '%Y%m%d%H', '%Y-%m-%d_%H'],
864
- 'minute' => [60, '%Y%m%d%H%M', '%Y-%m-%d_%H%M'],
865
- )
866
- test 'generates path with timestamp placeholder for original path with tailing star with timekey' do |data|
867
- timekey, placeholder, time_slice_format = data
868
- # with index placeholder, without compression suffix when append disabled and compression disabled
869
- assert_equal "/path/to/file.#{placeholder}_**", @i.generate_path_template('/path/to/file.*', timekey, false, nil)
870
- # with index placeholder, with .gz suffix when append disabled and gzip compression enabled
871
- assert_equal "/path/to/file.#{placeholder}_**.gz", @i.generate_path_template('/path/to/file.*', timekey, false, :gzip)
872
- # without index placeholder, without compression suffix when append enabled and compression disabled
873
- assert_equal "/path/to/file.#{placeholder}", @i.generate_path_template('/path/to/file.*', timekey, true, nil)
874
- # without index placeholder, with .gz suffix when append disabled and gzip compression enabled
875
- assert_equal "/path/to/file.#{placeholder}.gz", @i.generate_path_template('/path/to/file.*', timekey, true, :gzip)
876
-
877
- # time_slice_format will used instead of computed placeholder if specified
878
- assert_equal "/path/to/file.#{time_slice_format}_**", @i.generate_path_template('/path/to/file.*', timekey, false, nil, time_slice_format: time_slice_format)
879
- assert_equal "/path/to/file.#{time_slice_format}_**.gz", @i.generate_path_template('/path/to/file.*', timekey, false, :gzip, time_slice_format: time_slice_format)
880
- assert_equal "/path/to/file.#{time_slice_format}", @i.generate_path_template('/path/to/file.*', timekey, true, nil, time_slice_format: time_slice_format)
881
- assert_equal "/path/to/file.#{time_slice_format}.gz", @i.generate_path_template('/path/to/file.*', timekey, true, :gzip, time_slice_format: time_slice_format)
882
- end
883
-
884
- data(
885
- 'day' => [86400 * 2, '%Y%m%d', '%Y-%m-%d'],
886
- 'hour' => [7200, '%Y%m%d%H', '%Y-%m-%d_%H'],
887
- 'minute' => [180, '%Y%m%d%H%M', '%Y-%m-%d_%H%M'],
888
- )
889
- test 'generates path with timestamp placeholder for original path with star and suffix with timekey' do |data|
890
- timekey, placeholder, time_slice_format = data
891
- # with index placeholder, without compression suffix when append disabled and compression disabled
892
- assert_equal "/path/to/file.#{placeholder}_**.data", @i.generate_path_template('/path/to/file.*.data', timekey, false, nil)
893
- # with index placeholder, with .gz suffix when append disabled and gzip compression enabled
894
- assert_equal "/path/to/file.#{placeholder}_**.data.gz", @i.generate_path_template('/path/to/file.*.data', timekey, false, :gzip)
895
- # without index placeholder, without compression suffix when append enabled and compression disabled
896
- assert_equal "/path/to/file.#{placeholder}.data", @i.generate_path_template('/path/to/file.*.data', timekey, true, nil)
897
- # without index placeholder, with .gz suffix when append disabled and gzip compression enabled
898
- assert_equal "/path/to/file.#{placeholder}.data.gz", @i.generate_path_template('/path/to/file.*.data', timekey, true, :gzip)
899
-
900
- # time_slice_format will used instead of computed placeholder if specified
901
- assert_equal "/path/to/file.#{time_slice_format}_**.data", @i.generate_path_template('/path/to/file.*.data', timekey, false, nil, time_slice_format: time_slice_format)
902
- assert_equal "/path/to/file.#{time_slice_format}_**.data.gz", @i.generate_path_template('/path/to/file.*.data', timekey, false, :gzip, time_slice_format: time_slice_format)
903
- assert_equal "/path/to/file.#{time_slice_format}.data", @i.generate_path_template('/path/to/file.*.data', timekey, true, nil, time_slice_format: time_slice_format)
904
- assert_equal "/path/to/file.#{time_slice_format}.data.gz", @i.generate_path_template('/path/to/file.*.data', timekey, true, :gzip, time_slice_format: time_slice_format)
905
- end
906
-
907
- test 'raise error to show it is a bug when path including * specified without timekey' do
908
- assert_raise RuntimeError.new("BUG: configuration error must be raised for path including '*' without timekey") do
909
- @i.generate_path_template('/path/to/file.*.log', nil, false, nil)
910
- end
911
- end
912
-
913
- data(
914
- 'day' => [86400 * 7, '%Y%m%d', '%Y-%m-%d'],
915
- 'hour' => [3600 * 6, '%Y%m%d%H', '%Y-%m-%d_%H'],
916
- 'minute' => [60 * 15, '%Y%m%d%H%M', '%Y-%m-%d_%H%M'],
917
- )
918
- test 'generates path with timestamp placeholder for original path without time placeholders & star with timekey, and path_suffix configured' do |data|
919
- timekey, placeholder, time_slice_format = data
920
- # with index placeholder, without compression suffix when append disabled and compression disabled
921
- assert_equal "/path/to/file.#{placeholder}_**.log", @i.generate_path_template('/path/to/file', timekey, false, nil, path_suffix: '.log')
922
- # with index placeholder, with .gz suffix when append disabled and gzip compression enabled
923
- assert_equal "/path/to/file.#{placeholder}_**.log.gz", @i.generate_path_template('/path/to/file', timekey, false, :gzip, path_suffix: '.log')
924
- # without index placeholder, without compression suffix when append enabled and compression disabled
925
- assert_equal "/path/to/file.#{placeholder}.log", @i.generate_path_template('/path/to/file', timekey, true, nil, path_suffix: '.log')
926
- # without index placeholder, with compression suffix when append enabled and gzip compression enabled
927
- assert_equal "/path/to/file.#{placeholder}.log.gz", @i.generate_path_template('/path/to/file', timekey, true, :gzip, path_suffix: '.log')
928
-
929
- # time_slice_format will be appended always if it's specified
930
- assert_equal "/path/to/file.#{time_slice_format}_**.log", @i.generate_path_template('/path/to/file', timekey, false, nil, path_suffix: '.log', time_slice_format: time_slice_format)
931
- assert_equal "/path/to/file.#{time_slice_format}_**.log.gz", @i.generate_path_template('/path/to/file', timekey, false, :gzip, path_suffix: '.log', time_slice_format: time_slice_format)
932
- assert_equal "/path/to/file.#{time_slice_format}.log", @i.generate_path_template('/path/to/file', timekey, true, nil, path_suffix: '.log', time_slice_format: time_slice_format)
933
- assert_equal "/path/to/file.#{time_slice_format}.log.gz", @i.generate_path_template('/path/to/file', timekey, true, :gzip, path_suffix: '.log', time_slice_format: time_slice_format)
934
- end
935
-
936
- data(
937
- 'day' => [86400, '%Y%m%d'],
938
- 'hour' => [3600, '%Y%m%d%H'],
939
- 'minute' => [60, '%Y%m%d%H%M'],
940
- )
941
- test 'generates path with timestamp placeholder for original path without star with timekey, and path_suffix not configured' do |data|
942
- timekey, placeholder = data
943
- # with index placeholder, without compression suffix when append disabled and compression disabled
944
- assert_equal "/path/to/file.#{placeholder}_**", @i.generate_path_template('/path/to/file', timekey, false, nil)
945
- # with index placeholder, with .gz suffix when append disabled and gzip compression enabled
946
- assert_equal "/path/to/file.#{placeholder}_**.gz", @i.generate_path_template('/path/to/file', timekey, false, :gzip)
947
- # without index placeholder, without compression suffix when append enabled and compression disabled
948
- assert_equal "/path/to/file.#{placeholder}", @i.generate_path_template('/path/to/file', timekey, true, nil)
949
- # without index placeholder, with compression suffix when append enabled and gzip compression enabled
950
- assert_equal "/path/to/file.#{placeholder}.gz", @i.generate_path_template('/path/to/file', timekey, true, :gzip)
951
- end
952
-
953
- test 'generates path without adding timestamp placeholder part if original path has enough placeholders for specified timekey' do
954
- assert_equal "/path/to/file.%Y%m%d", @i.generate_path_template('/path/to/file.%Y%m%d', 86400, true, nil)
955
- assert_equal "/path/to/%Y%m%d/file", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, true, nil)
956
-
957
- assert_equal "/path/to/%Y%m%d/file_**", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, false, nil)
958
-
959
- assert_raise Fluent::ConfigError.new("insufficient timestamp placeholders in path") do
960
- @i.generate_path_template('/path/to/%Y%m/file', 86400, true, nil)
961
- end
962
- assert_raise Fluent::ConfigError.new("insufficient timestamp placeholders in path") do
963
- @i.generate_path_template('/path/to/file.%Y%m%d.log', 3600, true, nil)
964
- end
965
-
966
- assert_equal "/path/to/file.%Y%m%d_%H_**.log.gz", @i.generate_path_template('/path/to/file.%Y%m%d_%H', 7200, false, :gzip, path_suffix: '.log')
967
- assert_equal "/path/to/${tag}/file.%Y%m%d_%H_**.log.gz", @i.generate_path_template('/path/to/${tag}/file.%Y%m%d_%H', 7200, false, :gzip, path_suffix: '.log')
968
- end
969
-
970
- test 'generates path with specified time_slice_format appended even if path has sufficient timestamp placeholders' do
971
- assert_equal "/path/to/%Y%m%d/file.%Y-%m-%d_%H_**", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, false, nil, time_slice_format: '%Y-%m-%d_%H')
972
- assert_equal "/path/to/%Y%m%d/file.%Y-%m-%d_%H", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, true, nil, time_slice_format: '%Y-%m-%d_%H')
973
- assert_equal "/path/to/%Y%m%d/file.%Y-%m-%d_%H_**.log", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, false, nil, time_slice_format: '%Y-%m-%d_%H', path_suffix: '.log')
974
- assert_equal "/path/to/%Y%m%d/file.%Y-%m-%d_%H.log", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, true, nil, time_slice_format: '%Y-%m-%d_%H', path_suffix: '.log')
975
- assert_equal "/path/to/%Y%m%d/file.%Y-%m-%d_%H.log.gz", @i.generate_path_template('/path/to/%Y%m%d/file', 86400, true, :gzip, time_slice_format: '%Y-%m-%d_%H', path_suffix: '.log')
976
- end
977
-
978
- test 'generates path without timestamp placeholder when path does not include * and timekey not specified' do
979
- assert_equal '/path/to/file.log', @i.generate_path_template('/path/to/file.log', nil, true, nil)
980
- assert_equal '/path/to/file.log_**', @i.generate_path_template('/path/to/file.log', nil, false, nil)
981
- assert_equal '/path/to/file.${tag}.log_**', @i.generate_path_template('/path/to/file.${tag}.log', nil, false, nil)
982
- assert_equal '/path/to/file.${tag}_**.log', @i.generate_path_template('/path/to/file.${tag}', nil, false, nil, path_suffix: '.log')
983
- end
984
- end
985
-
986
- sub_test_case '#find_filepath_available' do
987
- setup do
988
- @tmp = File.join(TMP_DIR, 'find_filepath_test')
989
- FileUtils.mkdir_p @tmp
990
- @i = create_driver.instance
991
- end
992
-
993
- teardown do
994
- FileUtils.rm_rf @tmp
995
- end
996
-
997
- test 'raise error if argument path does not include index placeholder' do
998
- assert_raise RuntimeError.new("BUG: index placeholder not found in path: #{@tmp}/myfile") do
999
- @i.find_filepath_available("#{@tmp}/myfile") do |path|
1000
- # ...
1001
- end
1002
- end
1003
- end
1004
-
1005
- data(
1006
- 'without suffix' => ['myfile_0', 'myfile_**'],
1007
- 'with timestamp' => ['myfile_20161003_0', 'myfile_20161003_**'],
1008
- 'with base suffix' => ['myfile_0.log', 'myfile_**.log'],
1009
- 'with compression suffix' => ['myfile_0.log.gz', 'myfile_**.log.gz'],
1010
- )
1011
- test 'returns filepath with _0 at first' do |data|
1012
- expected, argument = data
1013
- @i.find_filepath_available(File.join(@tmp, argument)) do |path|
1014
- assert_equal File.join(@tmp, expected), path
1015
- end
1016
- end
1017
-
1018
- test 'returns filepath with index which does not exist yet' do
1019
- 5.times do |i|
1020
- Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
1021
- end
1022
- @i.find_filepath_available(File.join(@tmp, "exist_**.log")) do |path|
1023
- assert_equal File.join(@tmp, "exist_5.log"), path
1024
- end
1025
- end
1026
-
1027
- test 'creates lock directory when with_lock is true to exclude operations of other worker process' do
1028
- 5.times do |i|
1029
- Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a')
1030
- end
1031
- Dir.mkdir(File.join(@tmp, "exist_5.log.lock"))
1032
- @i.find_filepath_available(File.join(@tmp, "exist_**.log"), with_lock: true) do |path|
1033
- assert Dir.exist?(File.join(@tmp, "exist_6.log.lock"))
1034
- assert_equal File.join(@tmp, "exist_6.log"), path
1035
- end
1036
- end
1037
- end
1038
- end