fluentd 1.17.0-x64-mingw32 → 1.17.1-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
@@ -1,3429 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/input'
3
- require 'fluent/plugin/in_tail'
4
- require 'fluent/plugin/buffer'
5
- require 'fluent/system_config'
6
- require 'fluent/file_wrapper'
7
- require 'net/http'
8
- require 'flexmock/test_unit'
9
- require 'timecop'
10
- require 'tmpdir'
11
- require 'securerandom'
12
-
13
- class TailInputTest < Test::Unit::TestCase
14
- include FlexMock::TestCase
15
-
16
- def tmp_dir
17
- File.join(File.dirname(__FILE__), "..", "tmp", "tail#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
18
- end
19
-
20
- def setup
21
- Fluent::Test.setup
22
- @tmp_dir = tmp_dir
23
- cleanup_directory(@tmp_dir)
24
- end
25
-
26
- def teardown
27
- super
28
- cleanup_directory(@tmp_dir)
29
- Fluent::Engine.stop
30
- Timecop.return
31
- end
32
-
33
- def cleanup_directory(path)
34
- unless Dir.exist?(path)
35
- FileUtils.mkdir_p(path)
36
- return
37
- end
38
-
39
- FileUtils.remove_entry_secure(path, true)
40
- end
41
-
42
- def cleanup_file(path)
43
- FileUtils.remove_entry_secure(path, true)
44
- end
45
-
46
- def create_target_info(path)
47
- Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
48
- end
49
-
50
- ROOT_CONFIG = config_element("ROOT", "", {
51
- "tag" => "t1",
52
- "rotate_wait" => "2s",
53
- "refresh_interval" => "1s"
54
- })
55
-
56
- def base_config
57
- ROOT_CONFIG + config_element("", "", { "path" => "#{@tmp_dir}/tail.txt" })
58
- end
59
-
60
- def common_config
61
- base_config + config_element("", "", { "pos_file" => "#{@tmp_dir}/tail.pos" })
62
- end
63
-
64
- def common_follow_inode_config
65
- config_element("ROOT", "", {
66
- "path" => "#{@tmp_dir}/tail.txt*",
67
- "pos_file" => "#{@tmp_dir}/tail.pos",
68
- "tag" => "t1",
69
- "refresh_interval" => "1s",
70
- "read_from_head" => "true",
71
- "format" => "none",
72
- "rotate_wait" => "1s",
73
- "follow_inodes" => "true"
74
- })
75
- end
76
-
77
- CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
78
- CONFIG_DISABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
79
- CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
80
- CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
81
- SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
82
- PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
83
- MULTILINE_CONFIG = config_element(
84
- "", "", {
85
- "format" => "multiline",
86
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
87
- "format_firstline" => "/^[s]/"
88
- })
89
- PARSE_MULTILINE_CONFIG = config_element(
90
- "", "", {},
91
- [config_element("parse", "", {
92
- "@type" => "multiline",
93
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
94
- "format_firstline" => "/^[s]/"
95
- })
96
- ])
97
-
98
- MULTILINE_CONFIG_WITH_NEWLINE = config_element(
99
- "", "", {
100
- "format" => "multiline",
101
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.[^\\n]+))?/",
102
- "format_firstline" => "/^[s]/"
103
- })
104
- PARSE_MULTILINE_CONFIG_WITH_NEWLINE = config_element(
105
- "", "", {},
106
- [config_element("parse", "", {
107
- "@type" => "multiline",
108
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.[^\\n]+))?/",
109
- "format_firstline" => "/^[s]/"
110
- })
111
- ])
112
-
113
- EX_ROTATE_WAIT = 0
114
- EX_FOLLOW_INODES = false
115
-
116
- def ex_config
117
- config_element("", "", {
118
- "tag" => "tail",
119
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
120
- "format" => "none",
121
- "pos_file" => "#{@tmp_dir}/tail.pos",
122
- "read_from_head" => true,
123
- "refresh_interval" => 30,
124
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
125
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
126
- })
127
- end
128
-
129
- def tailing_group_pattern
130
- "/#{@tmp_dir}\/(?<podname>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\/[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container>.+)-(?<docker_id>[a-z0-9]{6})\.log$/"
131
- end
132
-
133
- DEBUG_LOG_LEVEL = config_element("", "", {
134
- "@log_level" => "debug"
135
- })
136
-
137
- def create_group_directive(pattern, rate_period, *rules)
138
- config_element("", "", {}, [
139
- config_element("group", "", {
140
- "pattern" => pattern,
141
- "rate_period" => rate_period
142
- }, rules)
143
- ])
144
- end
145
-
146
- def create_rule_directive(match_named_captures, limit)
147
- params = {
148
- "limit" => limit,
149
- "match" => match_named_captures,
150
- }
151
-
152
- config_element("rule", "", params)
153
- end
154
-
155
- def create_path_element(path)
156
- config_element("source", "", { "path" => "#{@tmp_dir}/#{path}" })
157
- end
158
-
159
- def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
160
- config = use_common_conf ? common_config + conf : conf
161
- Fluent::Test::Driver::Input.new(Fluent::Plugin::TailInput).configure(config)
162
- end
163
-
164
- sub_test_case "configure" do
165
- test "plain single line" do
166
- d = create_driver
167
- assert_equal(["#{@tmp_dir}/tail.txt"], d.instance.paths)
168
- assert_equal("t1", d.instance.tag)
169
- assert_equal(2, d.instance.rotate_wait)
170
- assert_equal("#{@tmp_dir}/tail.pos", d.instance.pos_file)
171
- assert_equal(1000, d.instance.read_lines_limit)
172
- assert_equal(-1, d.instance.read_bytes_limit_per_second)
173
- assert_equal(false, d.instance.ignore_repeated_permission_error)
174
- assert_nothing_raised do
175
- d.instance.have_read_capability?
176
- end
177
- end
178
-
179
- data("empty" => config_element,
180
- "w/o @type" => config_element("", "", {}, [config_element("parse", "", {})]))
181
- test "w/o parse section" do |conf|
182
- assert_raise(Fluent::ConfigError) do
183
- create_driver(conf)
184
- end
185
- end
186
-
187
- test "multi paths with path_delimiter" do
188
- c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "|" })
189
- d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
190
- assert_equal(["tail.txt", "test2", "tmp,dev"], d.instance.paths)
191
- end
192
-
193
- test "multi paths with same path configured twice" do
194
- c = config_element("ROOT", "", { "path" => "test1.txt,test2.txt,test1.txt", "tag" => "t1", "path_delimiter" => "," })
195
- d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
196
- assert_equal(["test2.txt","test1.txt"].sort, d.instance.paths.sort)
197
- end
198
-
199
- test "multi paths with invaid path_delimiter" do
200
- c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "*" })
201
- assert_raise(Fluent::ConfigError) do
202
- create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
203
- end
204
- end
205
-
206
- test "follow_inodes w/o pos file" do
207
- assert_raise(Fluent::ConfigError) do
208
- create_driver(base_config + config_element('', '', {'follow_inodes' => 'true'}))
209
- end
210
- end
211
-
212
- sub_test_case "log throttling per file" do
213
- test "w/o watcher timer is invalid" do
214
- conf = CONFIG_DISABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
215
- assert_raise(Fluent::ConfigError) do
216
- create_driver(conf)
217
- end
218
- end
219
-
220
- test "valid" do
221
- conf = config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
222
- assert_raise(Fluent::ConfigError) do
223
- create_driver(conf)
224
- end
225
- end
226
- end
227
-
228
- test "both enable_watch_timer and enable_stat_watcher are false" do
229
- assert_raise(Fluent::ConfigError) do
230
- create_driver(CONFIG_DISABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
231
- end
232
- end
233
-
234
- sub_test_case "encoding" do
235
- test "valid" do
236
- conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" })
237
- d = create_driver(conf)
238
- assert_equal(Encoding::UTF_8, d.instance.encoding)
239
- end
240
-
241
- test "invalid" do
242
- conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "no-such-encoding" })
243
- assert_raise(Fluent::ConfigError) do
244
- create_driver(conf)
245
- end
246
- end
247
- end
248
-
249
- sub_test_case "from_encoding" do
250
- test "only specified from_encoding raise ConfigError" do
251
- conf = SINGLE_LINE_CONFIG + config_element("", "", { "from_encoding" => "utf-8" })
252
- assert_raise(Fluent::ConfigError) do
253
- create_driver(conf)
254
- end
255
- end
256
-
257
- test "valid" do
258
- conf = SINGLE_LINE_CONFIG + config_element("", "", {
259
- "from_encoding" => "utf-8",
260
- "encoding" => "utf-8"
261
- })
262
- d = create_driver(conf)
263
- assert_equal(Encoding::UTF_8, d.instance.from_encoding)
264
- end
265
-
266
- test "invalid" do
267
- conf = SINGLE_LINE_CONFIG + config_element("", "", {
268
- "from_encoding" => "no-such-encoding",
269
- "encoding" => "utf-8"
270
- })
271
- assert_raise(Fluent::ConfigError) do
272
- create_driver(conf)
273
- end
274
- end
275
- end
276
- end
277
-
278
- sub_test_case "configure group" do
279
- test "<rule> required" do
280
- conf = create_group_directive('.', '1m') + SINGLE_LINE_CONFIG
281
- assert_raise(Fluent::ConfigError) do
282
- create_driver(conf)
283
- end
284
- end
285
-
286
- test "valid configuration" do
287
- rule1 = create_rule_directive({
288
- "namespace"=> "/namespace-a/",
289
- "podname"=> "/podname-[b|c]/"
290
- }, 100)
291
- rule2 = create_rule_directive({
292
- "namespace"=> "/namespace-[d|e]/",
293
- "podname"=> "/podname-f/",
294
- }, 50)
295
- rule3 = create_rule_directive({
296
- "podname"=> "/podname-g/",
297
- }, -1)
298
- rule4 = create_rule_directive({
299
- "namespace"=> "/namespace-h/",
300
- }, 0)
301
-
302
- conf = create_group_directive(tailing_group_pattern, '1m', rule1, rule2, rule3, rule4) + SINGLE_LINE_CONFIG
303
- assert_nothing_raised do
304
- create_driver(conf)
305
- end
306
- end
307
-
308
- test "limit should be greater than DEFAULT_LIMIT (-1)" do
309
- rule1 = create_rule_directive({
310
- "namespace"=> "/namespace-a/",
311
- "podname"=> "/podname-[b|c]/",
312
- }, -100)
313
- rule2 = create_rule_directive({
314
- "namespace"=> "/namespace-[d|e]/",
315
- "podname"=> "/podname-f/",
316
- }, 50)
317
- conf = create_group_directive(tailing_group_pattern, '1m', rule1, rule2) + SINGLE_LINE_CONFIG
318
- assert_raise(RuntimeError) do
319
- create_driver(conf)
320
- end
321
- end
322
- end
323
-
324
- sub_test_case "group rules line limit resolution" do
325
- test "valid" do
326
- rule1 = create_rule_directive({
327
- "namespace"=> "/namespace-a/",
328
- "podname"=> "/podname-[b|c]/",
329
- }, 50)
330
- rule2 = create_rule_directive({
331
- "podname"=> "/podname-[b|c]/",
332
- }, 400)
333
- rule3 = create_rule_directive({
334
- "namespace"=> "/namespace-a/",
335
- }, 100)
336
-
337
- conf = create_group_directive(tailing_group_pattern, '1m', rule3, rule1, rule2) + SINGLE_LINE_CONFIG
338
- assert_nothing_raised do
339
- d = create_driver(conf)
340
- instance = d.instance
341
-
342
- metadata = {
343
- "namespace"=> "namespace-a",
344
- "podname"=> "podname-b",
345
- }
346
- assert_equal(50, instance.find_group(metadata).limit)
347
-
348
- metadata = {
349
- "namespace" => "namespace-a",
350
- "podname" => "podname-c",
351
- }
352
- assert_equal(50, instance.find_group(metadata).limit)
353
-
354
- metadata = {
355
- "namespace" => "namespace-a",
356
- "podname" => "podname-d",
357
- }
358
- assert_equal(100, instance.find_group(metadata).limit)
359
-
360
- metadata = {
361
- "namespace" => "namespace-f",
362
- "podname" => "podname-b",
363
- }
364
- assert_equal(400, instance.find_group(metadata).limit)
365
-
366
- metadata = {
367
- "podname" => "podname-c",
368
- }
369
- assert_equal(400, instance.find_group(metadata).limit)
370
-
371
- assert_equal(-1, instance.find_group({}).limit)
372
- end
373
- end
374
- end
375
-
376
- sub_test_case "files should be placed in groups" do
377
- test "invalid regex pattern places files in default group" do
378
- rule1 = create_rule_directive({}, 100) ## limits default groups
379
- conf = ROOT_CONFIG + DEBUG_LOG_LEVEL + create_group_directive(tailing_group_pattern, '1m', rule1) + create_path_element("test*.txt") + SINGLE_LINE_CONFIG
380
-
381
- d = create_driver(conf, false)
382
- Fluent::FileWrapper.open("#{@tmp_dir}/test1.txt", 'w')
383
- Fluent::FileWrapper.open("#{@tmp_dir}/test2.txt", 'w')
384
- Fluent::FileWrapper.open("#{@tmp_dir}/test3.txt", 'w')
385
-
386
- d.run do
387
- ## checking default group_watcher's paths
388
- instance = d.instance
389
- key = instance.default_group_key
390
-
391
- assert_equal(3, instance.log.logs.count{|a| a.match?("Cannot find group from metadata, Adding file in the default group\n")})
392
- assert_equal(3, instance.group_watchers[key].size)
393
- assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test1.txt'))
394
- assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test2.txt'))
395
- assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test3.txt'))
396
- end
397
- end
398
-
399
- test "valid regex pattern places file in their respective groups" do
400
- rule1 = create_rule_directive({
401
- "namespace"=> "/test-namespace1/",
402
- "podname"=> "/test-podname1/",
403
- }, 100)
404
- rule2 = create_rule_directive({
405
- "namespace"=> "/test-namespace1/",
406
- }, 200)
407
- rule3 = create_rule_directive({
408
- "podname"=> "/test-podname2/",
409
- }, 300)
410
- rule4 = create_rule_directive({}, 400)
411
-
412
- path_element = create_path_element("test-podname*.log")
413
-
414
- conf = ROOT_CONFIG + create_group_directive(tailing_group_pattern, '1m', rule4, rule3, rule2, rule1) + path_element + SINGLE_LINE_CONFIG
415
- d = create_driver(conf, false)
416
-
417
- file1 = File.join(@tmp_dir, "test-podname1_test-namespace1_test-container-15fabq.log")
418
- file2 = File.join(@tmp_dir, "test-podname3_test-namespace1_test-container-15fabq.log")
419
- file3 = File.join(@tmp_dir, "test-podname2_test-namespace2_test-container-15fabq.log")
420
- file4 = File.join(@tmp_dir, "test-podname4_test-namespace3_test-container-15fabq.log")
421
-
422
- d.run do
423
- Fluent::FileWrapper.open(file1, 'w')
424
- Fluent::FileWrapper.open(file2, 'w')
425
- Fluent::FileWrapper.open(file3, 'w')
426
- Fluent::FileWrapper.open(file4, 'w')
427
-
428
- instance = d.instance
429
- assert_equal(100, instance.find_group_from_metadata(file1).limit)
430
- assert_equal(200, instance.find_group_from_metadata(file2).limit)
431
- assert_equal(300, instance.find_group_from_metadata(file3).limit)
432
- assert_equal(400, instance.find_group_from_metadata(file4).limit)
433
- end
434
- end
435
- end
436
-
437
- sub_test_case "singleline" do
438
- data(flat: SINGLE_LINE_CONFIG,
439
- parse: PARSE_SINGLE_LINE_CONFIG)
440
- def test_emit(data)
441
- config = data
442
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
443
- f.puts "test1"
444
- f.puts "test2"
445
- }
446
-
447
- d = create_driver(config)
448
-
449
- d.run(expect_emits: 1) do
450
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
451
- f.puts "test3\ntest4"
452
- }
453
- end
454
-
455
- events = d.events
456
- assert_equal(true, events.length > 0)
457
- assert_equal({"message" => "test3"}, events[0][2])
458
- assert_equal({"message" => "test4"}, events[1][2])
459
- assert(events[0][1].is_a?(Fluent::EventTime))
460
- assert(events[1][1].is_a?(Fluent::EventTime))
461
- assert_equal(1, d.emit_count)
462
- end
463
-
464
- def test_emit_with_emit_unmatched_lines_true
465
- config = config_element("", "", { "format" => "/^(?<message>test.*)/", "emit_unmatched_lines" => true })
466
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
467
-
468
- d = create_driver(config)
469
- d.run(expect_emits: 1) do
470
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
471
- f.puts "test line 1"
472
- f.puts "test line 2"
473
- f.puts "bad line 1"
474
- f.puts "test line 3"
475
- }
476
- end
477
-
478
- events = d.events
479
- assert_equal(4, events.length)
480
- assert_equal({"message" => "test line 1"}, events[0][2])
481
- assert_equal({"message" => "test line 2"}, events[1][2])
482
- assert_equal({"unmatched_line" => "bad line 1"}, events[2][2])
483
- assert_equal({"message" => "test line 3"}, events[3][2])
484
- end
485
-
486
- data('flat 1' => [:flat, 1, 2],
487
- 'flat 10' => [:flat, 10, 1],
488
- 'parse 1' => [:parse, 1, 2],
489
- 'parse 10' => [:parse, 10, 1])
490
- def test_emit_with_read_lines_limit(data)
491
- config_style, limit, num_events = data
492
- case config_style
493
- when :flat
494
- config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit })
495
- when :parse
496
- config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit }) + PARSE_SINGLE_LINE_CONFIG
497
- end
498
- d = create_driver(config)
499
- msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
500
-
501
- d.run(expect_emits: num_events, timeout: 2) do
502
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
503
- f.puts msg
504
- f.puts msg
505
- }
506
- end
507
-
508
- events = d.events
509
- assert_equal(true, events.length > 0)
510
- assert_equal({"message" => msg}, events[0][2])
511
- assert_equal({"message" => msg}, events[1][2])
512
- assert num_events <= d.emit_count
513
- end
514
-
515
- sub_test_case "log throttling per file" do
516
- teardown do
517
- cleanup_file("#{@tmp_dir}/tail.txt")
518
- end
519
-
520
- sub_test_case "reads_bytes_per_second w/o throttled" do
521
- data("flat 8192 bytes, 2 events" => [:flat, 100, 8192, 2],
522
- "flat 8192 bytes, 2 events w/o stat watcher" => [:flat_without_stat, 100, 8192, 2],
523
- "flat #{8192*10} bytes, 20 events" => [:flat, 100, (8192 * 10), 20],
524
- "flat #{8192*10} bytes, 20 events w/o stat watcher" => [:flat_without_stat, 100, (8192 * 10), 20],
525
- "parse #{8192*4} bytes, 8 events" => [:parse, 100, (8192 * 4), 8],
526
- "parse #{8192*4} bytes, 8 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 4), 8],
527
- "parse #{8192*10} bytes, 20 events" => [:parse, 100, (8192 * 10), 20],
528
- "parse #{8192*10} bytes, 20 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 10), 20],
529
- "flat 8k bytes with unit, 2 events" => [:flat, 100, "8k", 2])
530
- def test_emit_with_read_bytes_limit_per_second(data)
531
- config_style, limit, limit_bytes, num_events = data
532
- case config_style
533
- when :flat
534
- config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
535
- when :parse
536
- config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
537
- when :flat_without_stat
538
- config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
539
- when :parse_without_stat
540
- config = CONFIG_READ_FROM_HEAD + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
541
- end
542
-
543
- msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
544
- start_time = Fluent::Clock.now
545
-
546
- d = create_driver(config)
547
- d.run(expect_emits: 2) do
548
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
549
- 100.times do
550
- f.puts msg
551
- end
552
- }
553
- end
554
-
555
- assert_true(Fluent::Clock.now - start_time > 1)
556
- assert_equal(num_events.times.map { {"message" => msg} },
557
- d.events.collect { |event| event[2] })
558
- end
559
-
560
- def test_read_bytes_limit_precede_read_lines_limit
561
- config = CONFIG_READ_FROM_HEAD +
562
- SINGLE_LINE_CONFIG +
563
- config_element("", "", {
564
- "read_lines_limit" => 1000,
565
- "read_bytes_limit_per_second" => 8192
566
- })
567
- msg = 'abc'
568
- start_time = Fluent::Clock.now
569
- d = create_driver(config)
570
- d.run(expect_emits: 2) do
571
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
572
- 8000.times do
573
- f.puts msg
574
- end
575
- }
576
- end
577
-
578
- assert_true(Fluent::Clock.now - start_time > 1)
579
- assert_equal(4096.times.map { {"message" => msg} },
580
- d.events.collect { |event| event[2] })
581
- end
582
- end
583
-
584
- sub_test_case "reads_bytes_per_second w/ throttled already" do
585
- data("flat 8192 bytes" => [:flat, 100, 8192],
586
- "parse 8192 bytes" => [:parse, 100, 8192])
587
- def test_emit_with_read_bytes_limit_per_second(data)
588
- config_style, limit, limit_bytes = data
589
- case config_style
590
- when :flat
591
- config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
592
- when :parse
593
- config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
594
- end
595
- d = create_driver(config)
596
- msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
597
-
598
- mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
599
- require 'fluent/config/types'
600
- limit_bytes_value = Fluent::Config.size_value(limit_bytes)
601
- io_handler.instance_variable_set(:@number_bytes_read, limit_bytes_value)
602
- if Fluent.linux?
603
- mock.proxy(io_handler).handle_notify.at_least(5)
604
- else
605
- mock.proxy(io_handler).handle_notify.twice
606
- end
607
- io_handler
608
- end
609
-
610
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") do |f|
611
- 100.times do
612
- f.puts msg
613
- end
614
- end
615
-
616
- # We should not do shutdown here due to hard timeout.
617
- d.run do
618
- start_time = Fluent::Clock.now
619
- while Fluent::Clock.now - start_time < 0.8 do
620
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") do |f|
621
- f.puts msg
622
- f.flush
623
- end
624
- sleep 0.05
625
- end
626
- end
627
-
628
- assert_equal([], d.events)
629
- end
630
- end
631
-
632
- sub_test_case "EOF with reads_bytes_per_second" do
633
- def test_longer_than_rotate_wait
634
- limit_bytes = 8192
635
- num_lines = 1024 * 3
636
- msg = "08bytes"
637
-
638
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") do |f|
639
- f.write("#{msg}\n" * num_lines)
640
- end
641
-
642
- config = CONFIG_READ_FROM_HEAD +
643
- SINGLE_LINE_CONFIG +
644
- config_element("", "", {
645
- "read_bytes_limit_per_second" => limit_bytes,
646
- "rotate_wait" => 0.1,
647
- "refresh_interval" => 0.5,
648
- })
649
-
650
- rotated = false
651
- d = create_driver(config)
652
- d.run(timeout: 10) do
653
- while d.events.size < num_lines do
654
- if d.events.size > 0 && !rotated
655
- cleanup_file("#{@tmp_dir}/tail.txt")
656
- FileUtils.touch("#{@tmp_dir}/tail.txt")
657
- rotated = true
658
- end
659
- sleep 0.3
660
- end
661
- end
662
-
663
- assert_equal(num_lines,
664
- d.events.count do |event|
665
- event[2]["message"] == msg
666
- end)
667
- end
668
-
669
- def test_shorter_than_rotate_wait
670
- limit_bytes = 8192
671
- num_lines = 1024 * 2
672
- msg = "08bytes"
673
-
674
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") do |f|
675
- f.write("#{msg}\n" * num_lines)
676
- end
677
-
678
- config = CONFIG_READ_FROM_HEAD +
679
- SINGLE_LINE_CONFIG +
680
- config_element("", "", {
681
- "read_bytes_limit_per_second" => limit_bytes,
682
- "rotate_wait" => 2,
683
- "refresh_interval" => 0.5,
684
- })
685
-
686
- start_time = Fluent::Clock.now
687
- rotated = false
688
- detached = false
689
- d = create_driver(config)
690
- mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
691
- mock.proxy(tw).detach(anything) do |v|
692
- detached = true
693
- v
694
- end
695
- tw
696
- end.twice
697
-
698
- d.run(timeout: 10) do
699
- until detached do
700
- if d.events.size > 0 && !rotated
701
- cleanup_file("#{@tmp_dir}/tail.txt")
702
- FileUtils.touch("#{@tmp_dir}/tail.txt")
703
- rotated = true
704
- end
705
- sleep 0.3
706
- end
707
- end
708
-
709
- assert_true(Fluent::Clock.now - start_time > 2)
710
- assert_equal(num_lines,
711
- d.events.count do |event|
712
- event[2]["message"] == msg
713
- end)
714
- end
715
- end
716
- end
717
-
718
- data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
719
- parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
720
- def test_emit_with_read_from_head(data)
721
- config = data
722
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
723
- f.puts "test1"
724
- f.puts "test2"
725
- }
726
-
727
- d = create_driver(config)
728
-
729
- d.run(expect_emits: 2) do
730
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
731
- f.puts "test3"
732
- f.puts "test4"
733
- }
734
- end
735
-
736
- events = d.events
737
- assert(events.length > 0)
738
- assert_equal({"message" => "test1"}, events[0][2])
739
- assert_equal({"message" => "test2"}, events[1][2])
740
- assert_equal({"message" => "test3"}, events[2][2])
741
- assert_equal({"message" => "test4"}, events[3][2])
742
- end
743
-
744
- data(flat: CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG,
745
- parse: CONFIG_DISABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG)
746
- def test_emit_without_watch_timer(data)
747
- config = data
748
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
749
- f.puts "test1"
750
- f.puts "test2"
751
- }
752
-
753
- d = create_driver(config)
754
-
755
- d.run(expect_emits: 1) do
756
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
757
- f.puts "test3"
758
- f.puts "test4"
759
- }
760
- # according to cool.io's stat_watcher.c, systems without inotify will use
761
- # an "automatic" value, typically around 5 seconds
762
- end
763
-
764
- events = d.events
765
- assert(events.length > 0)
766
- assert_equal({"message" => "test3"}, events[0][2])
767
- assert_equal({"message" => "test4"}, events[1][2])
768
- end
769
-
770
- # https://github.com/fluent/fluentd/pull/3541#discussion_r740197711
771
- def test_watch_wildcard_path_without_watch_timer
772
- omit "need inotify" unless Fluent.linux?
773
-
774
- config = config_element("ROOT", "", {
775
- "path" => "#{@tmp_dir}/tail*.txt",
776
- "tag" => "t1",
777
- })
778
- config = config + CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG
779
-
780
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
781
- f.puts "test1"
782
- f.puts "test2"
783
- }
784
-
785
- d = create_driver(config, false)
786
-
787
- d.run(expect_emits: 1, timeout: 1) do
788
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
789
- f.puts "test3"
790
- f.puts "test4"
791
- }
792
- end
793
-
794
- assert_equal(
795
- [
796
- {"message" => "test3"},
797
- {"message" => "test4"},
798
- ],
799
- d.events.collect { |event| event[2] })
800
- end
801
-
802
- data(flat: CONFIG_DISABLE_STAT_WATCHER + SINGLE_LINE_CONFIG,
803
- parse: CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
804
- def test_emit_with_disable_stat_watcher(data)
805
- config = data
806
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
807
- f.puts "test1"
808
- f.puts "test2"
809
- }
810
-
811
- d = create_driver(config)
812
-
813
- d.run(expect_emits: 1) do
814
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
815
- f.puts "test3"
816
- f.puts "test4"
817
- }
818
- end
819
-
820
- events = d.events
821
- assert(events.length > 0)
822
- assert_equal({"message" => "test3"}, events[0][2])
823
- assert_equal({"message" => "test4"}, events[1][2])
824
- end
825
-
826
- def test_always_read_from_head_on_detecting_a_new_file
827
- d = create_driver(SINGLE_LINE_CONFIG)
828
-
829
- d.run(expect_emits: 1, timeout: 3) do
830
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
831
- f.puts "test1\ntest2\n"
832
- }
833
- end
834
-
835
- assert_equal(
836
- [
837
- {"message" => "test1"},
838
- {"message" => "test2"},
839
- ],
840
- d.events.collect { |event| event[2] })
841
- end
842
- end
843
-
844
- class TestWithSystem < self
845
- include Fluent::SystemConfig::Mixin
846
-
847
- OVERRIDE_FILE_PERMISSION = 0620
848
- CONFIG_SYSTEM = %[
849
- <system>
850
- file_permission #{OVERRIDE_FILE_PERMISSION}
851
- </system>
852
- ]
853
-
854
- def setup
855
- omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
856
- super
857
- # Store default permission
858
- @default_permission = system_config.instance_variable_get(:@file_permission)
859
- end
860
-
861
- def teardown
862
- return if Fluent.windows?
863
- super
864
- # Restore default permission
865
- system_config.instance_variable_set(:@file_permission, @default_permission)
866
- end
867
-
868
- def parse_system(text)
869
- basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
870
- Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
871
- end
872
-
873
- def test_emit_with_system
874
- system_conf = parse_system(CONFIG_SYSTEM)
875
- sc = Fluent::SystemConfig.new(system_conf)
876
- Fluent::Engine.init(sc)
877
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
878
- f.puts "test1"
879
- f.puts "test2"
880
- }
881
-
882
- d = create_driver
883
-
884
- d.run(expect_emits: 1) do
885
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
886
- f.puts "test3"
887
- f.puts "test4"
888
- }
889
- end
890
-
891
- events = d.events
892
- assert_equal(true, events.length > 0)
893
- assert_equal({"message" => "test3"}, events[0][2])
894
- assert_equal({"message" => "test4"}, events[1][2])
895
- assert(events[0][1].is_a?(Fluent::EventTime))
896
- assert(events[1][1].is_a?(Fluent::EventTime))
897
- assert_equal(1, d.emit_count)
898
- pos = d.instance.instance_variable_get(:@pf_file)
899
- mode = "%o" % File.stat(pos).mode
900
- assert_equal OVERRIDE_FILE_PERMISSION, mode[-3, 3].to_i
901
- end
902
- end
903
-
904
- sub_test_case "rotate file" do
905
- def create_driver(conf = SINGLE_LINE_CONFIG)
906
- config = common_config + conf
907
- Fluent::Test::Driver::Input.new(Fluent::Plugin::TailInput).configure(config)
908
- end
909
-
910
- data(flat: SINGLE_LINE_CONFIG,
911
- parse: PARSE_SINGLE_LINE_CONFIG)
912
- def test_rotate_file(data)
913
- config = data
914
- events = sub_test_rotate_file(config, expect_emits: 2)
915
- assert_equal(3.upto(6).collect { |i| {"message" => "test#{i}"} },
916
- events.collect { |event| event[2] })
917
- end
918
-
919
- data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
920
- parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
921
- def test_rotate_file_with_read_from_head(data)
922
- config = data
923
- events = sub_test_rotate_file(config, expect_records: 6)
924
- assert_equal(1.upto(6).collect { |i| {"message" => "test#{i}"} },
925
- events.collect { |event| event[2] })
926
- end
927
-
928
- data(flat: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
929
- parse: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
930
- def test_rotate_file_with_open_on_every_update(data)
931
- config = data
932
- events = sub_test_rotate_file(config, expect_records: 6)
933
- assert_equal(1.upto(6).collect { |i| {"message" => "test#{i}"} },
934
- events.collect { |event| event[2] })
935
- end
936
-
937
- data(flat: SINGLE_LINE_CONFIG,
938
- parse: PARSE_SINGLE_LINE_CONFIG)
939
- def test_rotate_file_with_write_old(data)
940
- config = data
941
- events = sub_test_rotate_file(config, expect_emits: 3) { |rotated_file|
942
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
943
- rotated_file.puts "test7"
944
- rotated_file.puts "test8"
945
- rotated_file.flush
946
-
947
- sleep 1
948
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
949
- f.puts "test5"
950
- f.puts "test6"
951
- }
952
- }
953
- # This test sometimes fails and it shows a potential bug of in_tail
954
- # https://github.com/fluent/fluentd/issues/1434
955
- assert_equal([3, 4, 7, 8, 5, 6].collect { |i| {"message" => "test#{i}"} },
956
- events.collect { |event| event[2] })
957
- end
958
-
959
- data(flat: SINGLE_LINE_CONFIG,
960
- parse: PARSE_SINGLE_LINE_CONFIG)
961
- def test_rotate_file_with_write_old_and_no_new_file(data)
962
- config = data
963
- events = sub_test_rotate_file(config, expect_emits: 2) { |rotated_file|
964
- rotated_file.puts "test7"
965
- rotated_file.puts "test8"
966
- rotated_file.flush
967
- }
968
- assert_equal([3, 4, 7, 8].collect { |i| {"message" => "test#{i}"} },
969
- events.collect { |event| event[2] })
970
- end
971
-
972
- def sub_test_rotate_file(config = nil, expect_emits: nil, expect_records: nil, timeout: 5)
973
- file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb")
974
- file.puts "test1"
975
- file.puts "test2"
976
- file.flush
977
-
978
- d = create_driver(config)
979
- d.run(expect_emits: expect_emits, expect_records: expect_records, timeout: timeout) do
980
- sleep(0.1) while d.instance.instance_variable_get(:@startup)
981
- size = d.emit_count
982
- file.puts "test3"
983
- file.puts "test4"
984
- file.flush
985
- sleep(0.1) until d.emit_count >= size + 1
986
- size = d.emit_count
987
-
988
- if Fluent.windows?
989
- file.close
990
- FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt", force: true)
991
- file = Fluent::FileWrapper.open("#{@tmp_dir}/tail2.txt", "ab")
992
- else
993
- FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt")
994
- end
995
- if block_given?
996
- yield file
997
- else
998
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
999
- sleep 1
1000
-
1001
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1002
- f.puts "test5"
1003
- f.puts "test6"
1004
- }
1005
- end
1006
- end
1007
-
1008
- d.events
1009
- ensure
1010
- file.close if file && !file.closed?
1011
- end
1012
- end
1013
-
1014
- def test_truncate_file
1015
- config = SINGLE_LINE_CONFIG
1016
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1017
- f.puts "test1"
1018
- f.puts "test2"
1019
- f.flush
1020
- }
1021
-
1022
- d = create_driver(config)
1023
-
1024
- d.run(expect_emits: 2) do
1025
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1026
- f.puts "test3\ntest4"
1027
- f.flush
1028
- }
1029
- waiting(2) { sleep 0.1 until d.events.length == 2 }
1030
- if Fluent.windows?
1031
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f|
1032
- f.puts("test1");
1033
- }
1034
- else
1035
- File.truncate("#{@tmp_dir}/tail.txt", 6)
1036
- end
1037
- end
1038
-
1039
- expected = {
1040
- emit_count: 2,
1041
- events: [
1042
- [Fluent::EventTime, {"message" => "test3"}],
1043
- [Fluent::EventTime, {"message" => "test4"}],
1044
- [Fluent::EventTime, {"message" => "test1"}],
1045
- ]
1046
- }
1047
- actual = {
1048
- emit_count: d.emit_count,
1049
- events: d.events.collect{|event| [event[1].class, event[2]]}
1050
- }
1051
- assert_equal(expected, actual)
1052
- end
1053
-
1054
- def test_move_truncate_move_back
1055
- config = SINGLE_LINE_CONFIG
1056
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1057
- f.puts "test1"
1058
- f.puts "test2"
1059
- }
1060
-
1061
- d = create_driver(config)
1062
-
1063
- d.run(expect_emits: 1) do
1064
- if Fluent.windows?
1065
- FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt", force: true)
1066
- else
1067
- FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt")
1068
- end
1069
- sleep(1)
1070
- if Fluent.windows?
1071
- Fluent::FileWrapper.open("#{@tmp_dir}/tail2.txt", "wb") { |f|
1072
- f.puts("test1");
1073
- }
1074
- else
1075
- File.truncate("#{@tmp_dir}/tail2.txt", 6)
1076
- end
1077
- sleep(1)
1078
- if Fluent.windows?
1079
- FileUtils.mv("#{@tmp_dir}/tail2.txt", "#{@tmp_dir}/tail.txt", force: true)
1080
- else
1081
- FileUtils.mv("#{@tmp_dir}/tail2.txt", "#{@tmp_dir}/tail.txt")
1082
- end
1083
- end
1084
-
1085
- events = d.events
1086
- assert_equal(1, events.length)
1087
- assert_equal({"message" => "test1"}, events[0][2])
1088
- assert(events[0][1].is_a?(Fluent::EventTime))
1089
- assert_equal(1, d.emit_count)
1090
- end
1091
-
1092
- def test_lf
1093
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| }
1094
-
1095
- d = create_driver
1096
-
1097
- d.run(expect_emits: 1) do
1098
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1099
- f.print "test3"
1100
- }
1101
- sleep 1
1102
-
1103
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1104
- f.puts "test4"
1105
- }
1106
- end
1107
-
1108
- events = d.events
1109
- assert_equal(true, events.length > 0)
1110
- assert_equal({"message" => "test3test4"}, events[0][2])
1111
- end
1112
-
1113
- def test_whitespace
1114
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| }
1115
-
1116
- d = create_driver
1117
-
1118
- d.run(expect_emits: 1) do
1119
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1120
- f.puts " " # 4 spaces
1121
- f.puts " 4 spaces"
1122
- f.puts "4 spaces "
1123
- f.puts " " # tab
1124
- f.puts " tab"
1125
- f.puts "tab "
1126
- }
1127
- end
1128
-
1129
- events = d.events
1130
- assert_equal(true, events.length > 0)
1131
- assert_equal({"message" => " "}, events[0][2])
1132
- assert_equal({"message" => " 4 spaces"}, events[1][2])
1133
- assert_equal({"message" => "4 spaces "}, events[2][2])
1134
- assert_equal({"message" => " "}, events[3][2])
1135
- assert_equal({"message" => " tab"}, events[4][2])
1136
- assert_equal({"message" => "tab "}, events[5][2])
1137
- end
1138
-
1139
- data(
1140
- 'flat default encoding' => [SINGLE_LINE_CONFIG, Encoding::ASCII_8BIT],
1141
- 'flat explicit encoding config' => [SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8],
1142
- 'parse default encoding' => [PARSE_SINGLE_LINE_CONFIG, Encoding::ASCII_8BIT],
1143
- 'parse explicit encoding config' => [PARSE_SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8])
1144
- def test_encoding(data)
1145
- encoding_config, encoding = data
1146
-
1147
- d = create_driver(CONFIG_READ_FROM_HEAD + encoding_config)
1148
-
1149
- d.run(expect_emits: 1) do
1150
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1151
- f.puts "test"
1152
- }
1153
- end
1154
-
1155
- events = d.events
1156
- assert_equal(encoding, events[0][2]['message'].encoding)
1157
- end
1158
-
1159
- def test_from_encoding
1160
- conf = config_element(
1161
- "", "", {
1162
- "format" => "/(?<message>.*)/",
1163
- "read_from_head" => "true",
1164
- "from_encoding" => "cp932",
1165
- "encoding" => "utf-8"
1166
- })
1167
- d = create_driver(conf)
1168
- cp932_message = "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
1169
- utf8_message = cp932_message.encode(Encoding::UTF_8)
1170
-
1171
- d.run(expect_emits: 1) do
1172
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w:cp932") {|f|
1173
- f.puts cp932_message
1174
- }
1175
- end
1176
-
1177
- events = d.events
1178
- assert_equal(utf8_message, events[0][2]['message'])
1179
- assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
1180
- end
1181
-
1182
- def test_from_encoding_utf16
1183
- conf = config_element(
1184
- "", "", {
1185
- "format" => "/(?<message>.*)/",
1186
- "read_from_head" => "true",
1187
- "from_encoding" => "utf-16le",
1188
- "encoding" => "utf-8"
1189
- })
1190
- d = create_driver(conf)
1191
- utf16_message = "\u306F\u308D\u30FC\u308F\u30FC\u308B\u3069\n".encode(Encoding::UTF_16LE)
1192
- utf8_message = utf16_message.encode(Encoding::UTF_8).strip
1193
-
1194
- d.run(expect_emits: 1) do
1195
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w:utf-16le") { |f|
1196
- f.write utf16_message
1197
- }
1198
- end
1199
-
1200
- events = d.events
1201
- assert_equal(utf8_message, events[0][2]['message'])
1202
- assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
1203
- end
1204
-
1205
- def test_encoding_with_bad_character
1206
- conf = config_element(
1207
- "", "", {
1208
- "format" => "/(?<message>.*)/",
1209
- "read_from_head" => "true",
1210
- "from_encoding" => "ASCII-8BIT",
1211
- "encoding" => "utf-8"
1212
- })
1213
- d = create_driver(conf)
1214
-
1215
- d.run(expect_emits: 1) do
1216
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w") { |f|
1217
- f.write "te\x86st\n"
1218
- }
1219
- end
1220
-
1221
- events = d.events
1222
- assert_equal("te\uFFFDst", events[0][2]['message'])
1223
- assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
1224
- end
1225
-
1226
- sub_test_case "multiline" do
1227
- data(flat: MULTILINE_CONFIG,
1228
- parse: PARSE_MULTILINE_CONFIG)
1229
- def test_multiline(data)
1230
- config = data
1231
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1232
-
1233
- d = create_driver(config)
1234
- d.run(expect_emits: 1) do
1235
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1236
- f.puts "f test1"
1237
- f.puts "s test2"
1238
- f.puts "f test3"
1239
- f.puts "f test4"
1240
- f.puts "s test5"
1241
- f.puts "s test6"
1242
- f.puts "f test7"
1243
- f.puts "s test8"
1244
- }
1245
- end
1246
-
1247
- events = d.events
1248
- assert_equal(4, events.length)
1249
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
1250
- assert_equal({"message1" => "test5"}, events[1][2])
1251
- assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
1252
- assert_equal({"message1" => "test8"}, events[3][2])
1253
- end
1254
-
1255
- data(flat: MULTILINE_CONFIG,
1256
- parse: PARSE_MULTILINE_CONFIG)
1257
- def test_multiline_with_emit_unmatched_lines_true(data)
1258
- config = data + config_element("", "", { "emit_unmatched_lines" => true })
1259
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1260
-
1261
- d = create_driver(config)
1262
- d.run(expect_emits: 1) do
1263
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1264
- f.puts "f test1"
1265
- f.puts "s test2"
1266
- f.puts "f test3"
1267
- f.puts "f test4"
1268
- f.puts "s test5"
1269
- f.puts "s test6"
1270
- f.puts "f test7"
1271
- f.puts "s test8"
1272
- }
1273
- end
1274
-
1275
- events = d.events
1276
- assert_equal(5, events.length)
1277
- assert_equal({"unmatched_line" => "f test1"}, events[0][2])
1278
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[1][2])
1279
- assert_equal({"message1" => "test5"}, events[2][2])
1280
- assert_equal({"message1" => "test6", "message2" => "test7"}, events[3][2])
1281
- assert_equal({"message1" => "test8"}, events[4][2])
1282
- end
1283
-
1284
- data(
1285
- flat: MULTILINE_CONFIG_WITH_NEWLINE,
1286
- parse: PARSE_MULTILINE_CONFIG_WITH_NEWLINE)
1287
- def test_multiline_with_emit_unmatched_lines2(data)
1288
- config = data + config_element("", "", { "emit_unmatched_lines" => true })
1289
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1290
-
1291
- d = create_driver(config)
1292
- d.run(expect_emits: 0, timeout: 1) do
1293
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1294
- f.puts "s test0"
1295
- f.puts "f test1"
1296
- f.puts "f test2"
1297
-
1298
- f.puts "f test3"
1299
-
1300
- f.puts "s test4"
1301
- f.puts "f test5"
1302
- f.puts "f test6"
1303
- }
1304
- end
1305
-
1306
- events = d.events
1307
- assert_equal({"message1" => "test0", "message2" => "test1", "message3" => "test2"}, events[0][2])
1308
- assert_equal({ 'unmatched_line' => "f test3" }, events[1][2])
1309
- assert_equal({"message1" => "test4", "message2" => "test5", "message3" => "test6"}, events[2][2])
1310
- end
1311
-
1312
- data(flat: MULTILINE_CONFIG,
1313
- parse: PARSE_MULTILINE_CONFIG)
1314
- def test_multiline_with_flush_interval(data)
1315
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1316
-
1317
- config = data + config_element("", "", { "multiline_flush_interval" => "2s" })
1318
- d = create_driver(config)
1319
-
1320
- assert_equal(2, d.instance.multiline_flush_interval)
1321
-
1322
- d.run(expect_emits: 1) do
1323
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1324
- f.puts "f test1"
1325
- f.puts "s test2"
1326
- f.puts "f test3"
1327
- f.puts "f test4"
1328
- f.puts "s test5"
1329
- f.puts "s test6"
1330
- f.puts "f test7"
1331
- f.puts "s test8"
1332
- }
1333
- end
1334
-
1335
- events = d.events
1336
- assert_equal(4, events.length)
1337
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
1338
- assert_equal({"message1" => "test5"}, events[1][2])
1339
- assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
1340
- assert_equal({"message1" => "test8"}, events[3][2])
1341
- end
1342
-
1343
- data(
1344
- 'flat default encoding' => [MULTILINE_CONFIG, Encoding::ASCII_8BIT],
1345
- 'flat explicit encoding config' => [MULTILINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8],
1346
- 'parse default encoding' => [PARSE_MULTILINE_CONFIG, Encoding::ASCII_8BIT],
1347
- 'parse explicit encoding config' => [PARSE_MULTILINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8])
1348
- def test_multiline_encoding_of_flushed_record(data)
1349
- encoding_config, encoding = data
1350
-
1351
- config = config_element("", "", {
1352
- "multiline_flush_interval" => "2s",
1353
- "read_from_head" => "true",
1354
- })
1355
- d = create_driver(config + encoding_config)
1356
-
1357
- d.run(expect_emits: 1) do
1358
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f|
1359
- f.puts "s test"
1360
- }
1361
- end
1362
- events = d.events
1363
- assert_equal(1, events.length)
1364
- assert_equal(encoding, events[0][2]['message1'].encoding)
1365
- end
1366
-
1367
- def test_multiline_from_encoding_of_flushed_record
1368
- conf = MULTILINE_CONFIG + config_element(
1369
- "", "",
1370
- {
1371
- "multiline_flush_interval" => "1s",
1372
- "read_from_head" => "true",
1373
- "from_encoding" => "cp932",
1374
- "encoding" => "utf-8"
1375
- })
1376
- d = create_driver(conf)
1377
-
1378
- cp932_message = "s \x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
1379
- utf8_message = "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".encode(Encoding::UTF_8, Encoding::CP932)
1380
- d.run(expect_emits: 1) do
1381
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w:cp932") { |f|
1382
- f.puts cp932_message
1383
- }
1384
- end
1385
-
1386
- events = d.events
1387
- assert_equal(1, events.length)
1388
- assert_equal(utf8_message, events[0][2]['message1'])
1389
- assert_equal(Encoding::UTF_8, events[0][2]['message1'].encoding)
1390
- end
1391
-
1392
- data(flat: config_element(
1393
- "", "", {
1394
- "format" => "multiline",
1395
- "format1" => "/^s (?<message1>[^\\n]+)\\n?/",
1396
- "format2" => "/(f (?<message2>[^\\n]+)\\n?)?/",
1397
- "format3" => "/(f (?<message3>.*))?/",
1398
- "format_firstline" => "/^[s]/"
1399
- }),
1400
- parse: config_element(
1401
- "", "", {},
1402
- [config_element("parse", "", {
1403
- "@type" => "multiline",
1404
- "format1" => "/^s (?<message1>[^\\n]+)\\n?/",
1405
- "format2" => "/(f (?<message2>[^\\n]+)\\n?)?/",
1406
- "format3" => "/(f (?<message3>.*))?/",
1407
- "format_firstline" => "/^[s]/"
1408
- })
1409
- ])
1410
- )
1411
- def test_multiline_with_multiple_formats(data)
1412
- config = data
1413
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1414
-
1415
- d = create_driver(config)
1416
- d.run(expect_emits: 1) do
1417
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1418
- f.puts "f test1"
1419
- f.puts "s test2"
1420
- f.puts "f test3"
1421
- f.puts "f test4"
1422
- f.puts "s test5"
1423
- f.puts "s test6"
1424
- f.puts "f test7"
1425
- f.puts "s test8"
1426
- }
1427
- end
1428
-
1429
- events = d.events
1430
- assert(events.length > 0)
1431
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
1432
- assert_equal({"message1" => "test5"}, events[1][2])
1433
- assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
1434
- assert_equal({"message1" => "test8"}, events[3][2])
1435
- end
1436
-
1437
- data(flat: config_element(
1438
- "", "", {
1439
- "format" => "multiline",
1440
- "format1" => "/^[s|f] (?<message>.*)/",
1441
- "format_firstline" => "/^[s]/"
1442
- }),
1443
- parse: config_element(
1444
- "", "", {},
1445
- [config_element("parse", "", {
1446
- "@type" => "multiline",
1447
- "format1" => "/^[s|f] (?<message>.*)/",
1448
- "format_firstline" => "/^[s]/"
1449
- })
1450
- ])
1451
- )
1452
- def test_multilinelog_with_multiple_paths(data)
1453
- files = ["#{@tmp_dir}/tail1.txt", "#{@tmp_dir}/tail2.txt"]
1454
- files.each { |file| Fluent::FileWrapper.open(file, "wb") { |f| } }
1455
-
1456
- config = data + config_element("", "", {
1457
- "path" => "#{files[0]},#{files[1]}",
1458
- "tag" => "t1",
1459
- })
1460
- d = create_driver(config, false)
1461
- d.run(expect_emits: 2) do
1462
- files.each do |file|
1463
- Fluent::FileWrapper.open(file, 'ab') { |f|
1464
- f.puts "f #{file} line should be ignored"
1465
- f.puts "s test1"
1466
- f.puts "f test2"
1467
- f.puts "f test3"
1468
- f.puts "s test4"
1469
- }
1470
- end
1471
- end
1472
-
1473
- events = d.events
1474
- assert_equal({"message" => "test1\nf test2\nf test3"}, events[0][2])
1475
- assert_equal({"message" => "test1\nf test2\nf test3"}, events[1][2])
1476
- # "test4" events are here because these events are flushed at shutdown phase
1477
- assert_equal({"message" => "test4"}, events[2][2])
1478
- assert_equal({"message" => "test4"}, events[3][2])
1479
- end
1480
-
1481
- data(flat: config_element("", "", {
1482
- "format" => "multiline",
1483
- "format1" => "/(?<var1>foo \\d)\\n/",
1484
- "format2" => "/(?<var2>bar \\d)\\n/",
1485
- "format3" => "/(?<var3>baz \\d)/"
1486
- }),
1487
- parse: config_element(
1488
- "", "", {},
1489
- [config_element("parse", "", {
1490
- "@type" => "multiline",
1491
- "format1" => "/(?<var1>foo \\d)\\n/",
1492
- "format2" => "/(?<var2>bar \\d)\\n/",
1493
- "format3" => "/(?<var3>baz \\d)/"
1494
- })
1495
- ])
1496
- )
1497
- def test_multiline_without_firstline(data)
1498
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1499
-
1500
- config = data
1501
- d = create_driver(config)
1502
- d.run(expect_emits: 1) do
1503
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1504
- f.puts "foo 1"
1505
- f.puts "bar 1"
1506
- f.puts "baz 1"
1507
- f.puts "foo 2"
1508
- f.puts "bar 2"
1509
- f.puts "baz 2"
1510
- }
1511
- end
1512
-
1513
- events = d.events
1514
- assert_equal(2, events.length)
1515
- assert_equal({"var1" => "foo 1", "var2" => "bar 1", "var3" => "baz 1"}, events[0][2])
1516
- assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, events[1][2])
1517
- end
1518
- end
1519
-
1520
- sub_test_case "path" do
1521
- # * path test
1522
- # TODO: Clean up tests
1523
- def test_expand_paths
1524
- ex_paths = [
1525
- create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1526
- create_target_info('test/plugin/data/log/foo/bar.log'),
1527
- create_target_info('test/plugin/data/log/test.log')
1528
- ]
1529
- plugin = create_driver(ex_config, false).instance
1530
- flexstub(Time) do |timeclass|
1531
- timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
1532
- assert_equal(ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1533
- end
1534
-
1535
- # Test exclusion
1536
- exclude_config = ex_config + config_element("", "", { "exclude_path" => %Q(["#{ex_paths.last.path}"]) })
1537
- plugin = create_driver(exclude_config, false).instance
1538
- assert_equal(ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1539
- end
1540
-
1541
- sub_test_case "expand_paths with glob" do |data|
1542
- sub_test_case "extended_glob" do
1543
- data("curly braces" => [true, "always", "test/plugin/data/log_numeric/{0,1}*.log"],
1544
- "square brackets" => [true, "always", "test/plugin/data/log_numeric/[0-1][2-4].log"],
1545
- "asterisk" => [true, "always", "test/plugin/data/log/*.log"],
1546
- "one character matcher" => [true, "always", "test/plugin/data/log/tes?.log"],
1547
- )
1548
- def test_expand_paths_with_use_glob_p_and_almost_set_of_patterns
1549
- result, option, path = data
1550
- config = config_element("", "", {
1551
- "tag" => "tail",
1552
- "path" => path,
1553
- "format" => "none",
1554
- "pos_file" => "#{@tmp_dir}/tail.pos",
1555
- "read_from_head" => true,
1556
- "refresh_interval" => 30,
1557
- "glob_policy" => option,
1558
- "path_delimiter" => "|",
1559
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1560
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1561
- })
1562
- plugin = create_driver(config, false).instance
1563
- assert_equal(result, !!plugin.use_glob?(path))
1564
- end
1565
-
1566
- data("curly braces" => [true, false, "extended", "test/plugin/data/log_numeric/{0,1}*.log"],
1567
- "square brackets" => [false, true, "extended", "test/plugin/data/log_numeric/[0-1][2-4].log"],
1568
- "asterisk" => [false, true, "extended", "test/plugin/data/log/*.log"],
1569
- "one character matcher" => [false, true, "extended", "test/plugin/data/log/tes?.log"],
1570
- )
1571
- def test_expand_paths_with_use_glob_p
1572
- emit_exception_p, result, option, path = data
1573
- config = config_element("", "", {
1574
- "tag" => "tail",
1575
- "path" => path,
1576
- "format" => "none",
1577
- "pos_file" => "#{@tmp_dir}/tail.pos",
1578
- "read_from_head" => true,
1579
- "refresh_interval" => 30,
1580
- "glob_policy" => option,
1581
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1582
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1583
- })
1584
- if emit_exception_p
1585
- assert_raise(Fluent::ConfigError) do
1586
- plugin = create_driver(config, false).instance
1587
- end
1588
- else
1589
- plugin = create_driver(config, false).instance
1590
- assert_equal(result, !!plugin.use_glob?(path))
1591
- end
1592
- end
1593
- end
1594
-
1595
- sub_test_case "only_use_backward_compatible" do
1596
- data("square brackets" => [false, "backward_compatible", "test/plugin/data/log_numeric/[0-1][2-4].log"],
1597
- "asterisk" => [true, "backward_compatible", "test/plugin/data/log/*.log"],
1598
- "one character matcher" => [false, "backward_compatible", "test/plugin/data/log/tes?.log"],
1599
- )
1600
- def test_expand_paths_with_use_glob_p
1601
- result, option, path = data
1602
- config = config_element("", "", {
1603
- "tag" => "tail",
1604
- "path" => path,
1605
- "format" => "none",
1606
- "pos_file" => "#{@tmp_dir}/tail.pos",
1607
- "read_from_head" => true,
1608
- "refresh_interval" => 30,
1609
- "glob_policy" => option,
1610
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1611
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1612
- })
1613
- plugin = create_driver(config, false).instance
1614
- assert_equal(result, !!plugin.use_glob?(path))
1615
- end
1616
- end
1617
- end
1618
-
1619
- def ex_config_with_brackets
1620
- config_element("", "", {
1621
- "tag" => "tail",
1622
- "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
1623
- "format" => "none",
1624
- "pos_file" => "#{@tmp_dir}/tail.pos",
1625
- "read_from_head" => true,
1626
- "refresh_interval" => 30,
1627
- "glob_policy" => "extended",
1628
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1629
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1630
- })
1631
- end
1632
-
1633
- def test_config_with_always_with_default_delimiter
1634
- assert_raise(Fluent::ConfigError) do
1635
- config = config_element("", "", {
1636
- "tag" => "tail",
1637
- "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
1638
- "format" => "none",
1639
- "pos_file" => "#{@tmp_dir}/tail.pos",
1640
- "read_from_head" => true,
1641
- "refresh_interval" => 30,
1642
- "glob_policy" => "always",
1643
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1644
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1645
- })
1646
-
1647
- create_driver(config, false).instance
1648
- end
1649
- end
1650
-
1651
- def test_config_with_always_with_custom_delimiter
1652
- assert_nothing_raised do
1653
- config = config_element("", "", {
1654
- "tag" => "tail",
1655
- "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
1656
- "format" => "none",
1657
- "pos_file" => "#{@tmp_dir}/tail.pos",
1658
- "read_from_head" => true,
1659
- "refresh_interval" => 30,
1660
- "glob_policy" => "always",
1661
- "path_delimiter" => "|",
1662
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1663
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1664
- })
1665
-
1666
- create_driver(config, false).instance
1667
- end
1668
- end
1669
-
1670
- def test_expand_paths_with_brackets
1671
- expanded_paths = [
1672
- create_target_info('test/plugin/data/log_numeric/01.log'),
1673
- create_target_info('test/plugin/data/log_numeric/02.log'),
1674
- create_target_info('test/plugin/data/log_numeric/12.log'),
1675
- create_target_info('test/plugin/data/log_numeric/14.log'),
1676
- ]
1677
-
1678
- plugin = create_driver(ex_config_with_brackets, false).instance
1679
- assert_equal(expanded_paths - [expanded_paths.first], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1680
- end
1681
-
1682
- def test_expand_paths_with_duplicate_configuration
1683
- expanded_paths = [
1684
- create_target_info('test/plugin/data/log/foo/bar.log'),
1685
- create_target_info('test/plugin/data/log/test.log')
1686
- ]
1687
- duplicate_config = ex_config.dup
1688
- duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
1689
- plugin = create_driver(ex_config, false).instance
1690
- assert_equal(expanded_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1691
- end
1692
-
1693
- def test_expand_paths_with_timezone
1694
- ex_paths = [
1695
- create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1696
- create_target_info('test/plugin/data/log/foo/bar.log'),
1697
- create_target_info('test/plugin/data/log/test.log')
1698
- ]
1699
- ['Asia/Taipei', '+08'].each do |tz_type|
1700
- taipei_config = ex_config + config_element("", "", {"path_timezone" => tz_type})
1701
- plugin = create_driver(taipei_config, false).instance
1702
-
1703
- # Test exclude
1704
- exclude_config = taipei_config + config_element("", "", { "exclude_path" => %Q(["test/plugin/**/%Y%m%d-%H%M%S.log"]) })
1705
- exclude_plugin = create_driver(exclude_config, false).instance
1706
-
1707
- with_timezone('utc') do
1708
- flexstub(Time) do |timeclass|
1709
- # env : 2010-01-01 19:04:05 (UTC), tail path : 2010-01-02 03:04:05 (Asia/Taipei)
1710
- timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 1, 19, 4, 5))
1711
-
1712
- assert_equal(ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1713
- assert_equal(ex_paths - [ex_paths.first], exclude_plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1714
- end
1715
- end
1716
- end
1717
- end
1718
-
1719
- def test_log_file_without_extension
1720
- expected_files = [
1721
- create_target_info('test/plugin/data/log/bar'),
1722
- create_target_info('test/plugin/data/log/foo/bar.log'),
1723
- create_target_info('test/plugin/data/log/foo/bar2'),
1724
- create_target_info('test/plugin/data/log/test.log')
1725
- ]
1726
-
1727
- config = config_element("", "", {
1728
- "tag" => "tail",
1729
- "path" => "test/plugin/data/log/**/*",
1730
- "format" => "none",
1731
- "pos_file" => "#{@tmp_dir}/tail.pos"
1732
- })
1733
-
1734
- plugin = create_driver(config, false).instance
1735
- assert_equal(expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1736
- end
1737
-
1738
- def test_unwatched_files_should_be_removed
1739
- config = config_element("", "", {
1740
- "tag" => "tail",
1741
- "path" => "#{@tmp_dir}/*.txt",
1742
- "format" => "none",
1743
- "pos_file" => "#{@tmp_dir}/tail.pos",
1744
- "read_from_head" => true,
1745
- "refresh_interval" => 1,
1746
- })
1747
- d = create_driver(config, false)
1748
- d.end_if { d.instance.instance_variable_get(:@tails).keys.size >= 1 }
1749
- d.run(expect_emits: 1, shutdown: false) do
1750
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f| f.puts "test3\n" }
1751
- end
1752
-
1753
- cleanup_file("#{@tmp_dir}/tail.txt")
1754
- waiting(20) { sleep 0.1 until Dir.glob("#{@tmp_dir}/*.txt").size == 0 } # Ensure file is deleted on Windows
1755
- waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size <= 0 }
1756
-
1757
- assert_equal(
1758
- {
1759
- files: [],
1760
- tails: []
1761
- },
1762
- {
1763
- files: Dir.glob("#{@tmp_dir}/*.txt"),
1764
- tails: d.instance.instance_variable_get(:@tails).keys
1765
- }
1766
- )
1767
- ensure
1768
- d.instance_shutdown if d && d.instance
1769
- cleanup_directory(@tmp_dir)
1770
- end
1771
-
1772
- def count_timer_object
1773
- num = 0
1774
- ObjectSpace.each_object(Fluent::PluginHelper::Timer::TimerWatcher) { |obj|
1775
- num += 1
1776
- }
1777
- num
1778
- end
1779
- end
1780
-
1781
- sub_test_case "path w/ Linux capability" do
1782
- def capability_enabled?
1783
- if Fluent.linux?
1784
- begin
1785
- require 'capng'
1786
- true
1787
- rescue LoadError
1788
- false
1789
- end
1790
- else
1791
- false
1792
- end
1793
- end
1794
-
1795
- setup do
1796
- omit "This environment is not enabled Linux capability handling feature" unless capability_enabled?
1797
-
1798
- @capng = CapNG.new(:current_process)
1799
- flexstub(Fluent::Capability) do |klass|
1800
- klass.should_receive(:new).with(:current_process).and_return(@capng)
1801
- end
1802
- end
1803
-
1804
- data("dac_read_search" => [:dac_read_search, true, 1],
1805
- "dac_override" => [:dac_override, true, 1],
1806
- "chown" => [:chown, false, 0],
1807
- )
1808
- test "with partially elevated privileges" do |data|
1809
- cap, result, readable_paths = data
1810
- @capng.update(:add, :effective, cap)
1811
-
1812
- d = create_driver(
1813
- config_element("ROOT", "", {
1814
- "path" => "/var/log/ker*.log", # Use /var/log/kern.log
1815
- "tag" => "t1",
1816
- "rotate_wait" => "2s"
1817
- }) + PARSE_SINGLE_LINE_CONFIG, false)
1818
-
1819
- assert_equal(readable_paths, d.instance.expand_paths.length)
1820
- assert_equal(result, d.instance.have_read_capability?)
1821
- end
1822
- end
1823
-
1824
- def test_pos_file_dir_creation
1825
- config = config_element("", "", {
1826
- "tag" => "tail",
1827
- "path" => "#{@tmp_dir}/*.txt",
1828
- "format" => "none",
1829
- "pos_file" => "#{@tmp_dir}/pos/tail.pos",
1830
- "read_from_head" => true,
1831
- "refresh_interval" => 1
1832
- })
1833
-
1834
- assert_path_not_exist("#{@tmp_dir}/pos")
1835
- d = create_driver(config, false)
1836
- d.run
1837
- assert_path_exist("#{@tmp_dir}/pos")
1838
- assert_equal('755', File.stat("#{@tmp_dir}/pos").mode.to_s(8)[-3, 3])
1839
- ensure
1840
- cleanup_directory(@tmp_dir)
1841
- end
1842
-
1843
- def test_pos_file_dir_creation_with_system_dir_permission
1844
- config = config_element("", "", {
1845
- "tag" => "tail",
1846
- "path" => "#{@tmp_dir}/*.txt",
1847
- "format" => "none",
1848
- "pos_file" => "#{@tmp_dir}/pos/tail.pos",
1849
- "read_from_head" => true,
1850
- "refresh_interval" => 1
1851
- })
1852
-
1853
- assert_path_not_exist("#{@tmp_dir}/pos")
1854
-
1855
- Fluent::SystemConfig.overwrite_system_config({ "dir_permission" => "744" }) do
1856
- d = create_driver(config, false)
1857
- d.run
1858
- end
1859
-
1860
- assert_path_exist("#{@tmp_dir}/pos")
1861
- if Fluent.windows?
1862
- assert_equal('755', File.stat("#{@tmp_dir}/pos").mode.to_s(8)[-3, 3])
1863
- else
1864
- assert_equal('744', File.stat("#{@tmp_dir}/pos").mode.to_s(8)[-3, 3])
1865
- end
1866
- ensure
1867
- cleanup_directory(@tmp_dir)
1868
- end
1869
-
1870
- def test_z_refresh_watchers
1871
- ex_paths = [
1872
- create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1873
- create_target_info('test/plugin/data/log/foo/bar.log'),
1874
- create_target_info('test/plugin/data/log/test.log'),
1875
- ]
1876
- plugin = create_driver(ex_config, false).instance
1877
- sio = StringIO.new
1878
- plugin.instance_eval do
1879
- @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
1880
- @loop = Coolio::Loop.new
1881
- opened_file_metrics = Fluent::Plugin::LocalMetrics.new
1882
- opened_file_metrics.configure(config_element('metrics', '', {}))
1883
- closed_file_metrics = Fluent::Plugin::LocalMetrics.new
1884
- closed_file_metrics.configure(config_element('metrics', '', {}))
1885
- rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
1886
- rotated_file_metrics.configure(config_element('metrics', '', {}))
1887
- @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
1888
- end
1889
-
1890
- Timecop.freeze(2010, 1, 2, 3, 4, 5) do
1891
- ex_paths.each do |target_info|
1892
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
1893
- end
1894
-
1895
- plugin.refresh_watchers
1896
- end
1897
-
1898
- path = 'test/plugin/data/2010/01/20100102-030405.log'
1899
- mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)[path], Fluent::FileWrapper.stat(path).ino)
1900
-
1901
- Timecop.freeze(2010, 1, 2, 3, 4, 6) do
1902
- path = "test/plugin/data/2010/01/20100102-030406.log"
1903
- inode = Fluent::FileWrapper.stat(path).ino
1904
- target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
1905
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
1906
- plugin.refresh_watchers
1907
-
1908
- flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
1909
- watcherclass.should_receive(:new).never
1910
- plugin.refresh_watchers
1911
- end
1912
- end
1913
- end
1914
-
1915
- sub_test_case "refresh of pos file" do
1916
- test 'type of pos_file_compaction_interval is time' do
1917
- tail = {
1918
- "tag" => "tail",
1919
- "path" => "#{@tmp_dir}/*.txt",
1920
- "format" => "none",
1921
- "pos_file" => "#{@tmp_dir}/pos/tail.pos",
1922
- "refresh_interval" => 1,
1923
- "read_from_head" => true,
1924
- 'pos_file_compaction_interval' => '24h',
1925
- }
1926
- config = config_element("", "", tail)
1927
- d = create_driver(config, false)
1928
- mock(d.instance).timer_execute(:in_tail_refresh_watchers, 1.0).once
1929
- mock(d.instance).timer_execute(:in_tail_refresh_compact_pos_file, 60 * 60 * 24).once
1930
- d.run # call start
1931
- end
1932
- end
1933
-
1934
- sub_test_case "receive_lines" do
1935
- DummyWatcher = Struct.new("DummyWatcher", :tag)
1936
-
1937
- def test_tag
1938
- d = create_driver(ex_config, false)
1939
- d.run {}
1940
- plugin = d.instance
1941
- mock(plugin.router).emit_stream('tail', anything).once
1942
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1943
- end
1944
-
1945
- def test_tag_with_only_star
1946
- config = config_element("", "", {
1947
- "tag" => "*",
1948
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1949
- "format" => "none",
1950
- "read_from_head" => true
1951
- })
1952
- d = create_driver(config, false)
1953
- d.run {}
1954
- plugin = d.instance
1955
- mock(plugin.router).emit_stream('foo.bar.log', anything).once
1956
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1957
- end
1958
-
1959
- def test_tag_prefix
1960
- config = config_element("", "", {
1961
- "tag" => "pre.*",
1962
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1963
- "format" => "none",
1964
- "read_from_head" => true
1965
- })
1966
- d = create_driver(config, false)
1967
- d.run {}
1968
- plugin = d.instance
1969
- mock(plugin.router).emit_stream('pre.foo.bar.log', anything).once
1970
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1971
- end
1972
-
1973
- def test_tag_suffix
1974
- config = config_element("", "", {
1975
- "tag" => "*.post",
1976
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1977
- "format" => "none",
1978
- "read_from_head" => true
1979
- })
1980
- d = create_driver(config, false)
1981
- d.run {}
1982
- plugin = d.instance
1983
- mock(plugin.router).emit_stream('foo.bar.log.post', anything).once
1984
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1985
- end
1986
-
1987
- def test_tag_prefix_and_suffix
1988
- config = config_element("", "", {
1989
- "tag" => "pre.*.post",
1990
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1991
- "format" => "none",
1992
- "read_from_head" => true
1993
- })
1994
- d = create_driver(config, false)
1995
- d.run {}
1996
- plugin = d.instance
1997
- mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once
1998
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1999
- end
2000
-
2001
- def test_tag_prefix_and_suffix_ignore
2002
- config = config_element("", "", {
2003
- "tag" => "pre.*.post*ignore",
2004
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
2005
- "format" => "none",
2006
- "read_from_head" => true
2007
- })
2008
- d = create_driver(config, false)
2009
- d.run {}
2010
- plugin = d.instance
2011
- mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once
2012
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
2013
- end
2014
-
2015
- data(
2016
- small: ["128", 128],
2017
- KiB: ["1k", 1024]
2018
- )
2019
- test 'max_line_size' do |(label, size)|
2020
- config = config_element("", "", {
2021
- "tag" => "max_line_size",
2022
- "path" => "#{@tmp_dir}/with_long_lines.txt",
2023
- "format" => "none",
2024
- "read_from_head" => true,
2025
- "max_line_size" => label,
2026
- "log_level" => "debug"
2027
- })
2028
- Fluent::FileWrapper.open("#{@tmp_dir}/with_long_lines.txt", "w+") do |f|
2029
- f.puts "foo"
2030
- f.puts "x" * size # 'x' * size + \n > @max_line_size
2031
- f.puts "bar"
2032
- end
2033
- d = create_driver(config, false)
2034
- timestamp = Time.parse("Mon Nov 29 11:22:33 UTC 2021")
2035
- Timecop.freeze(timestamp)
2036
- d.run(expect_records: 2)
2037
- assert_equal([
2038
- [{"message" => "foo"},{"message" => "bar"}],
2039
- [
2040
- "2021-11-29 11:22:33 +0000 [warn]: received line length is longer than #{size}\n",
2041
- "2021-11-29 11:22:33 +0000 [debug]: skipped line: #{'x' * size}\n"
2042
- ]
2043
- ],
2044
- [
2045
- d.events.collect { |event| event.last },
2046
- d.logs[-2..]
2047
- ])
2048
- end
2049
- end
2050
-
2051
- # Ensure that no fatal exception is raised when a file is missing and that
2052
- # files that do exist are still tailed as expected.
2053
- def test_missing_file
2054
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2055
- f.puts "test1"
2056
- f.puts "test2"
2057
- }
2058
-
2059
- # Try two different configs - one with read_from_head and one without,
2060
- # since their interactions with the filesystem differ.
2061
- config1 = config_element("", "", {
2062
- "tag" => "t1",
2063
- "path" => "#{@tmp_dir}/non_existent_file.txt,#{@tmp_dir}/tail.txt",
2064
- "format" => "none",
2065
- "rotate_wait" => "2s",
2066
- "pos_file" => "#{@tmp_dir}/tail.pos"
2067
- })
2068
- config2 = config1 + config_element("", "", { "read_from_head" => true })
2069
- [config1, config2].each do |config|
2070
- d = create_driver(config, false)
2071
- d.run(expect_emits: 1) do
2072
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
2073
- f.puts "test3"
2074
- f.puts "test4"
2075
- }
2076
- end
2077
- # This test sometimes fails and it shows a potential bug of in_tail
2078
- # https://github.com/fluent/fluentd/issues/1434
2079
- events = d.events
2080
- assert_equal(2, events.length)
2081
- assert_equal({"message" => "test3"}, events[0][2])
2082
- assert_equal({"message" => "test4"}, events[1][2])
2083
- end
2084
- end
2085
-
2086
- sub_test_case 'inode_processing' do
2087
- def test_should_delete_file_pos_entry_for_non_existing_file_with_follow_inodes
2088
- config = common_follow_inode_config
2089
-
2090
- path = "#{@tmp_dir}/tail.txt"
2091
- ino = 1
2092
- pos = 1234
2093
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "wb") {|f|
2094
- f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
2095
- }
2096
-
2097
- d = create_driver(config, false)
2098
- d.run
2099
-
2100
- pos_file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r")
2101
- pos_file.pos = 0
2102
-
2103
- assert_raise(EOFError) do
2104
- pos_file.readline
2105
- end
2106
- end
2107
-
2108
- def test_should_write_latest_offset_after_rotate_wait
2109
- config = common_follow_inode_config
2110
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2111
- f.puts "test1"
2112
- f.puts "test2"
2113
- }
2114
-
2115
- d = create_driver(config, false)
2116
- d.run(expect_emits: 2, shutdown: false) do
2117
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2118
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
2119
- sleep 1
2120
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt" + "1", "ab") {|f| f.puts "test4\n"}
2121
- end
2122
-
2123
- pos_file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r")
2124
- pos_file.pos = 0
2125
- line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
2126
- waiting(5) {
2127
- while line_parts[2].to_i(16) != 24
2128
- sleep(0.1)
2129
- pos_file.pos = 0
2130
- line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
2131
- end
2132
- }
2133
- assert_equal(24, line_parts[2].to_i(16))
2134
- d.instance_shutdown
2135
- end
2136
-
2137
- def test_should_remove_deleted_file
2138
- config = config_element("", "", {"format" => "none"})
2139
-
2140
- path = "#{@tmp_dir}/tail.txt"
2141
- ino = 1
2142
- pos = 1234
2143
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "wb") {|f|
2144
- f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
2145
- }
2146
-
2147
- d = create_driver(config)
2148
- d.run do
2149
- pos_file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r")
2150
- pos_file.pos = 0
2151
- assert_equal([], pos_file.readlines)
2152
- end
2153
- end
2154
-
2155
- def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
2156
- config = config_element("ROOT", "", {
2157
- "path" => "#{@tmp_dir}/tail.txt*",
2158
- "pos_file" => "#{@tmp_dir}/tail.pos",
2159
- "tag" => "t1",
2160
- "rotate_wait" => "1s",
2161
- "refresh_interval" => "1s",
2162
- "limit_recently_modified" => "1s",
2163
- "read_from_head" => "true",
2164
- "format" => "none",
2165
- "follow_inodes" => "true",
2166
- })
2167
-
2168
- d = create_driver(config, false)
2169
-
2170
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2171
- f.puts "test1"
2172
- f.puts "test2"
2173
- }
2174
- target_info = create_target_info("#{@tmp_dir}/tail.txt")
2175
-
2176
- d.run(expect_emits: 1, shutdown: false) do
2177
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2178
- end
2179
-
2180
-
2181
- Timecop.travel(Time.now + 10) do
2182
- waiting(5) {
2183
- # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
2184
- sleep 0.1 until d.instance.instance_variable_get(:@pf)[target_info].read_pos == 0
2185
- }
2186
- end
2187
-
2188
- assert_equal(0, d.instance.instance_variable_get(:@pf)[target_info].read_pos)
2189
-
2190
- d.instance_shutdown
2191
- end
2192
-
2193
- def test_should_read_from_head_on_file_renaming_with_star_in_pattern
2194
- config = config_element("ROOT", "", {
2195
- "path" => "#{@tmp_dir}/tail.txt*",
2196
- "pos_file" => "#{@tmp_dir}/tail.pos",
2197
- "tag" => "t1",
2198
- "rotate_wait" => "10s",
2199
- "refresh_interval" => "1s",
2200
- "limit_recently_modified" => "60s",
2201
- "read_from_head" => "true",
2202
- "format" => "none",
2203
- "follow_inodes" => "true"
2204
- })
2205
-
2206
- d = create_driver(config, false)
2207
-
2208
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2209
- f.puts "test1"
2210
- f.puts "test2"
2211
- }
2212
-
2213
- d.run(expect_emits: 2, shutdown: false) do
2214
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2215
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1")
2216
- end
2217
-
2218
- events = d.events
2219
- assert_equal(3, events.length)
2220
- d.instance_shutdown
2221
- end
2222
-
2223
- def test_should_not_read_from_head_on_rotation_when_watching_inodes
2224
- config = common_follow_inode_config
2225
-
2226
- d = create_driver(config, false)
2227
-
2228
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2229
- f.puts "test1"
2230
- f.puts "test2"
2231
- }
2232
-
2233
- d.run(expect_emits: 1, shutdown: false) do
2234
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2235
- end
2236
-
2237
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1")
2238
- Timecop.travel(Time.now + 10) do
2239
- sleep 2
2240
- events = d.events
2241
- assert_equal(3, events.length)
2242
- end
2243
-
2244
- d.instance_shutdown
2245
- end
2246
-
2247
- def test_should_mark_file_unwatched_if_same_name_file_created_with_different_inode
2248
- config = common_follow_inode_config
2249
-
2250
- d = create_driver(config, false)
2251
-
2252
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2253
- f.puts "test1"
2254
- f.puts "test2"
2255
- }
2256
- target_info = create_target_info("#{@tmp_dir}/tail.txt")
2257
-
2258
- d.run(expect_emits: 2, shutdown: false) do
2259
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2260
- cleanup_file("#{@tmp_dir}/tail.txt")
2261
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "test4\n"}
2262
- end
2263
-
2264
- new_target_info = create_target_info("#{@tmp_dir}/tail.txt")
2265
-
2266
- pos_file = d.instance.instance_variable_get(:@pf)
2267
-
2268
- waiting(10) {
2269
- # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
2270
- sleep 0.1 until pos_file[target_info].read_pos == 0
2271
- }
2272
- new_position = pos_file[new_target_info].read_pos
2273
- assert_equal(6, new_position)
2274
-
2275
- d.instance_shutdown
2276
- end
2277
-
2278
- def test_should_close_watcher_after_rotate_wait
2279
- now = Time.now
2280
- config = common_follow_inode_config + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
2281
-
2282
- d = create_driver(config, false)
2283
- d.instance.instance_eval do
2284
- opened_file_metrics = Fluent::Plugin::LocalMetrics.new
2285
- opened_file_metrics.configure(config_element('metrics', '', {}))
2286
- closed_file_metrics = Fluent::Plugin::LocalMetrics.new
2287
- closed_file_metrics.configure(config_element('metrics', '', {}))
2288
- rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
2289
- rotated_file_metrics.configure(config_element('metrics', '', {}))
2290
- @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
2291
- end
2292
-
2293
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2294
- f.puts "test1"
2295
- f.puts "test2"
2296
- }
2297
- target_info = create_target_info("#{@tmp_dir}/tail.txt")
2298
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything, anything).once
2299
- d.run(shutdown: false)
2300
- assert d.instance.instance_variable_get(:@tails)[target_info.path]
2301
-
2302
- Timecop.travel(now + 10) do
2303
- d.instance.instance_eval do
2304
- sleep 0.1 until @tails[target_info.path] == nil
2305
- end
2306
- assert_nil d.instance.instance_variable_get(:@tails)[target_info.path]
2307
- end
2308
- d.instance_shutdown
2309
- end
2310
-
2311
- def test_should_create_new_watcher_for_new_file_with_same_name
2312
- now = Time.now
2313
- config = common_follow_inode_config + config_element('', '', {"limit_recently_modified" => "2s"})
2314
-
2315
- d = create_driver(config, false)
2316
-
2317
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2318
- f.puts "test1"
2319
- f.puts "test2"
2320
- }
2321
- path_ino = create_target_info("#{@tmp_dir}/tail.txt")
2322
-
2323
- d.run(expect_emits: 1, shutdown: false) do
2324
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2325
- end
2326
-
2327
- cleanup_file("#{@tmp_dir}/tail.txt")
2328
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2329
- f.puts "test3"
2330
- f.puts "test4"
2331
- }
2332
- new_path_ino = create_target_info("#{@tmp_dir}/tail.txt")
2333
-
2334
- Timecop.travel(now + 10) do
2335
- sleep 3
2336
- d.instance.instance_eval do
2337
- @tails[path_ino.path] == nil
2338
- @tails[new_path_ino.path] != nil
2339
- end
2340
- end
2341
-
2342
- events = d.events
2343
-
2344
- assert_equal(5, events.length)
2345
-
2346
- d.instance_shutdown
2347
- end
2348
-
2349
- def test_truncate_file_with_follow_inodes
2350
- config = common_follow_inode_config
2351
-
2352
- d = create_driver(config, false)
2353
-
2354
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2355
- f.puts "test1"
2356
- f.puts "test2"
2357
- }
2358
-
2359
- d.run(expect_emits: 3, shutdown: false) do
2360
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2361
- sleep 2
2362
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w+b") {|f| f.puts "test4\n"}
2363
- end
2364
-
2365
- events = d.events
2366
- assert_equal(4, events.length)
2367
- assert_equal({"message" => "test1"}, events[0][2])
2368
- assert_equal({"message" => "test2"}, events[1][2])
2369
- assert_equal({"message" => "test3"}, events[2][2])
2370
- assert_equal({"message" => "test4"}, events[3][2])
2371
- d.instance_shutdown
2372
- end
2373
-
2374
- # issue #3464
2375
- def test_should_replace_target_info
2376
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2377
- f.puts "test1\n"
2378
- }
2379
- target_info = create_target_info("#{@tmp_dir}/tail.txt")
2380
- inodes = []
2381
-
2382
- config = config_element("ROOT", "", {
2383
- "path" => "#{@tmp_dir}/tail.txt*",
2384
- "pos_file" => "#{@tmp_dir}/tail.pos",
2385
- "tag" => "t1",
2386
- "refresh_interval" => "60s",
2387
- "read_from_head" => "true",
2388
- "format" => "none",
2389
- "rotate_wait" => "1s",
2390
- "follow_inodes" => "true"
2391
- })
2392
- d = create_driver(config, false)
2393
- d.run(timeout: 5) do
2394
- while d.events.size < 1 do
2395
- sleep 0.1
2396
- end
2397
- inodes = d.instance.instance_variable_get(:@tails).values.collect do |tw|
2398
- tw.ino
2399
- end
2400
- assert_equal([target_info.ino], inodes)
2401
-
2402
- cleanup_file("#{@tmp_dir}/tail.txt")
2403
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "test2\n"}
2404
-
2405
- while d.events.size < 2 do
2406
- sleep 0.1
2407
- end
2408
- inodes = d.instance.instance_variable_get(:@tails).values.collect do |tw|
2409
- tw.ino
2410
- end
2411
- new_target_info = create_target_info("#{@tmp_dir}/tail.txt")
2412
- assert_not_equal(target_info.ino, new_target_info.ino)
2413
- assert_equal([new_target_info.ino], inodes)
2414
- end
2415
- end
2416
- end
2417
-
2418
- sub_test_case "tail_path" do
2419
- def test_tail_path_with_singleline
2420
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2421
- f.puts "test1"
2422
- f.puts "test2"
2423
- }
2424
-
2425
- d = create_driver(SINGLE_LINE_CONFIG + config_element("", "", { "path_key" => "path" }))
2426
-
2427
- d.run(expect_emits: 1) do
2428
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
2429
- f.puts "test3"
2430
- f.puts "test4"
2431
- }
2432
- end
2433
-
2434
- events = d.events
2435
- assert_equal(true, events.length > 0)
2436
- events.each do |emit|
2437
- assert_equal("#{@tmp_dir}/tail.txt", emit[2]["path"])
2438
- end
2439
- end
2440
-
2441
- def test_tail_path_with_multiline_with_firstline
2442
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
2443
-
2444
- config = config_element("", "", {
2445
- "path_key" => "path",
2446
- "format" => "multiline",
2447
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
2448
- "format_firstline" => "/^[s]/"
2449
- })
2450
- d = create_driver(config)
2451
- d.run(expect_emits: 1) do
2452
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
2453
- f.puts "f test1"
2454
- f.puts "s test2"
2455
- f.puts "f test3"
2456
- f.puts "f test4"
2457
- f.puts "s test5"
2458
- f.puts "s test6"
2459
- f.puts "f test7"
2460
- f.puts "s test8"
2461
- }
2462
- end
2463
-
2464
- events = d.events
2465
- assert_equal(4, events.length)
2466
- events.each do |emit|
2467
- assert_equal("#{@tmp_dir}/tail.txt", emit[2]["path"])
2468
- end
2469
- end
2470
-
2471
- def test_tail_path_with_multiline_without_firstline
2472
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
2473
-
2474
- config = config_element("", "", {
2475
- "path_key" => "path",
2476
- "format" => "multiline",
2477
- "format1" => "/(?<var1>foo \\d)\\n/",
2478
- "format2" => "/(?<var2>bar \\d)\\n/",
2479
- "format3" => "/(?<var3>baz \\d)/",
2480
- })
2481
- d = create_driver(config)
2482
- d.run(expect_emits: 1) do
2483
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
2484
- f.puts "foo 1"
2485
- f.puts "bar 1"
2486
- f.puts "baz 1"
2487
- }
2488
- end
2489
-
2490
- events = d.events
2491
- assert(events.length > 0)
2492
- events.each do |emit|
2493
- assert_equal("#{@tmp_dir}/tail.txt", emit[2]["path"])
2494
- end
2495
- end
2496
-
2497
- def test_tail_path_with_multiline_with_multiple_paths
2498
- if ENV["APPVEYOR"] && Fluent.windows?
2499
- omit "This testcase is unstable on AppVeyor."
2500
- end
2501
- files = ["#{@tmp_dir}/tail1.txt", "#{@tmp_dir}/tail2.txt"]
2502
- files.each { |file| Fluent::FileWrapper.open(file, "wb") { |f| } }
2503
-
2504
- config = config_element("", "", {
2505
- "path" => "#{files[0]},#{files[1]}",
2506
- "path_key" => "path",
2507
- "tag" => "t1",
2508
- "format" => "multiline",
2509
- "format1" => "/^[s|f] (?<message>.*)/",
2510
- "format_firstline" => "/^[s]/"
2511
- })
2512
- d = create_driver(config, false)
2513
- d.run(expect_emits: 2) do
2514
- files.each do |file|
2515
- Fluent::FileWrapper.open(file, 'ab') { |f|
2516
- f.puts "f #{file} line should be ignored"
2517
- f.puts "s test1"
2518
- f.puts "f test2"
2519
- f.puts "f test3"
2520
- f.puts "s test4"
2521
- }
2522
- end
2523
- end
2524
-
2525
- events = d.events
2526
- assert_equal(4, events.length)
2527
- assert_equal(files, [events[0][2]["path"], events[1][2]["path"]].sort)
2528
- # "test4" events are here because these events are flushed at shutdown phase
2529
- assert_equal(files, [events[2][2]["path"], events[3][2]["path"]].sort)
2530
- end
2531
- end
2532
-
2533
- def test_limit_recently_modified
2534
- now = Time.new(2010, 1, 2, 3, 4, 5)
2535
- FileUtils.touch("#{@tmp_dir}/tail_unwatch.txt", mtime: (now - 3601))
2536
- FileUtils.touch("#{@tmp_dir}/tail_watch1.txt", mtime: (now - 3600))
2537
- FileUtils.touch("#{@tmp_dir}/tail_watch2.txt", mtime: now)
2538
-
2539
- config = config_element('', '', {
2540
- 'tag' => 'tail',
2541
- 'path' => "#{@tmp_dir}/*.txt",
2542
- 'format' => 'none',
2543
- 'limit_recently_modified' => '3600s'
2544
- })
2545
-
2546
- expected_files = [
2547
- create_target_info("#{@tmp_dir}/tail_watch1.txt"),
2548
- create_target_info("#{@tmp_dir}/tail_watch2.txt")
2549
- ]
2550
-
2551
- Timecop.freeze(now) do
2552
- plugin = create_driver(config, false).instance
2553
- assert_equal(expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
2554
- end
2555
- end
2556
-
2557
- def test_skip_refresh_on_startup
2558
- FileUtils.touch("#{@tmp_dir}/tail.txt")
2559
- config = config_element('', '', {
2560
- 'format' => 'none',
2561
- 'refresh_interval' => 1,
2562
- 'skip_refresh_on_startup' => true
2563
- })
2564
- d = create_driver(config)
2565
- d.run(shutdown: false) {}
2566
- assert_equal(0, d.instance.instance_variable_get(:@tails).keys.size)
2567
- # detect a file at first execution of in_tail_refresh_watchers timer
2568
- waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1 }
2569
- d.instance_shutdown
2570
- end
2571
-
2572
- def test_ENOENT_error_after_setup_watcher
2573
- path = "#{@tmp_dir}/tail.txt"
2574
- FileUtils.touch(path)
2575
- config = config_element('', '', {
2576
- 'format' => 'none',
2577
- })
2578
- d = create_driver(config)
2579
- file_deleted = false
2580
- mock.proxy(d.instance).existence_path do |hash|
2581
- unless file_deleted
2582
- cleanup_file(path)
2583
- file_deleted = true
2584
- end
2585
- hash
2586
- end.twice
2587
- assert_nothing_raised do
2588
- d.run(shutdown: false) {}
2589
- end
2590
- assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed. Continuing without tailing it.\n") },
2591
- $log.out.logs.join("\n"))
2592
- ensure
2593
- d.instance_shutdown if d && d.instance
2594
- end
2595
-
2596
- def test_EACCES_error_after_setup_watcher
2597
- omit "Cannot test with root user" if Process::UID.eid == 0
2598
- path = "#{@tmp_dir}/noaccess/tail.txt"
2599
- begin
2600
- FileUtils.mkdir_p("#{@tmp_dir}/noaccess")
2601
- FileUtils.chmod(0755, "#{@tmp_dir}/noaccess")
2602
- FileUtils.touch(path)
2603
- config = config_element('', '', {
2604
- 'tag' => "tail",
2605
- 'path' => path,
2606
- 'format' => 'none',
2607
- })
2608
- d = create_driver(config, false)
2609
- mock.proxy(d.instance).existence_path do |hash|
2610
- FileUtils.chmod(0000, "#{@tmp_dir}/noaccess")
2611
- hash
2612
- end.twice
2613
- assert_nothing_raised do
2614
- d.run(shutdown: false) {}
2615
- end
2616
- assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed. Continuing without tailing it.\n") },
2617
- $log.out.logs.join("\n"))
2618
- end
2619
- ensure
2620
- d.instance_shutdown if d && d.instance
2621
- if File.exist?("#{@tmp_dir}/noaccess")
2622
- FileUtils.chmod(0755, "#{@tmp_dir}/noaccess")
2623
- FileUtils.rm_rf("#{@tmp_dir}/noaccess")
2624
- end
2625
- end unless Fluent.windows?
2626
-
2627
- def test_EACCES
2628
- path = "#{@tmp_dir}/tail.txt"
2629
- FileUtils.touch(path)
2630
- config = config_element('', '', {
2631
- 'format' => 'none',
2632
- })
2633
- d = create_driver(config)
2634
- mock.proxy(Fluent::FileWrapper).stat(path) do |stat|
2635
- raise Errno::EACCES
2636
- end.at_least(1)
2637
- assert_nothing_raised do
2638
- d.run(shutdown: false) {}
2639
- end
2640
- assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
2641
- ensure
2642
- d.instance_shutdown if d && d.instance
2643
- end
2644
-
2645
- def test_shutdown_timeout
2646
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") do |f|
2647
- # Should be large enough to take too long time to consume
2648
- (1024 * 1024 * 5).times do
2649
- f.puts "{\"test\":\"fizzbuzz\"}"
2650
- end
2651
- end
2652
-
2653
- config =
2654
- CONFIG_READ_FROM_HEAD +
2655
- config_element('', '', {
2656
- 'format' => 'json',
2657
- 'skip_refresh_on_startup' => true,
2658
- })
2659
- shutdown_start_time = 0
2660
-
2661
- d = create_driver(config)
2662
- mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
2663
- mock.proxy(io_handler).ready_to_shutdown(anything) do
2664
- shutdown_start_time = Fluent::Clock.now
2665
- end
2666
- io_handler.shutdown_timeout = 0.5
2667
- io_handler
2668
- end
2669
-
2670
- assert_nothing_raised do
2671
- d.run(expect_emits: 1)
2672
- end
2673
-
2674
- elapsed = Fluent::Clock.now - shutdown_start_time
2675
- assert_true(elapsed > 0.5 && elapsed < 2.0,
2676
- "elapsed time: #{elapsed}")
2677
- end
2678
-
2679
- sub_test_case "throttling logs at in_tail level" do
2680
- data("file test1.log no_limit 5120 text: msg" => ["test1.log", 5120, "msg"],
2681
- "file test2.log no_limit 1024 text: test" => ["test2.log", 1024, "test"])
2682
- def test_lines_collected_with_no_throttling(data)
2683
- file, num_lines, msg = data
2684
-
2685
- pattern = "/^#{@tmp_dir}\/(?<file>.+)\.log$/"
2686
- rule = create_rule_directive({
2687
- "file" => "/test.*/",
2688
- }, -1)
2689
- group = create_group_directive(pattern, "1s", rule)
2690
- path_element = create_path_element(file)
2691
-
2692
- conf = ROOT_CONFIG + group + path_element + CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG
2693
-
2694
- Fluent::FileWrapper.open("#{@tmp_dir}/#{file}", 'wb') do |f|
2695
- num_lines.times do
2696
- f.puts "#{msg}\n"
2697
- end
2698
- end
2699
-
2700
-
2701
- d = create_driver(conf, false)
2702
- d.run(timeout: 3) do
2703
- start_time = Fluent::Clock.now
2704
-
2705
- assert_equal(num_lines, d.record_count)
2706
- assert_equal({ "message" => msg }, d.events[0][2])
2707
-
2708
- prev_count = d.record_count
2709
- sleep(0.1) while d.emit_count < 1
2710
- assert_true(Fluent::Clock.now - start_time < 2)
2711
- ## after waiting for 1 (+ jitter) secs, limit will reset
2712
- ## Plugin will start reading but it will encounter EOF Error
2713
- ## since no logs are left to be read
2714
- ## Hence, d.record_count = prev_count
2715
- tail_watcher_interval = 1.0 # hard coded value in in_tail
2716
- safety_ratio = 1.2
2717
- jitter = tail_watcher_interval * safety_ratio
2718
- sleep(1.0 + jitter)
2719
- assert_equal(0, d.record_count - prev_count)
2720
- end
2721
- end
2722
-
2723
- test "lines collected with throttling" do
2724
- file = "podname1_namespace12_container-123456.log"
2725
- limit = 1000
2726
- rate_period = 2
2727
- num_lines = 3000
2728
- msg = "a" * 8190 # Total size = 8190 bytes + 2 (\n) bytes
2729
-
2730
- rule = create_rule_directive({
2731
- "namespace"=> "/namespace.+/",
2732
- "podname"=> "/podname.+/",
2733
- }, limit)
2734
- path_element = create_path_element(file)
2735
- conf = ROOT_CONFIG + create_group_directive(tailing_group_pattern, "#{rate_period}s", rule) + path_element + SINGLE_LINE_CONFIG + CONFIG_READ_FROM_HEAD
2736
-
2737
- d = create_driver(conf, false)
2738
- file_path = "#{@tmp_dir}/#{file}"
2739
-
2740
- Fluent::FileWrapper.open(file_path, 'wb') do |f|
2741
- num_lines.times do
2742
- f.puts msg
2743
- end
2744
- end
2745
-
2746
- d.run(timeout: 15) do
2747
- sleep_interval = 0.1
2748
- tail_watcher_interval = 1.0 # hard coded value in in_tail
2749
- safety_ratio = 1.2
2750
- lower_jitter = sleep_interval * safety_ratio
2751
- upper_jitter = (tail_watcher_interval + sleep_interval) * safety_ratio
2752
- lower_interval = rate_period - lower_jitter
2753
- upper_interval = rate_period + upper_jitter
2754
-
2755
- emit_count = 0
2756
- prev_count = 0
2757
-
2758
- while emit_count < 3 do
2759
- start_time = Fluent::Clock.now
2760
- sleep(sleep_interval) while d.emit_count <= emit_count
2761
- elapsed_seconds = Fluent::Clock.now - start_time
2762
- if emit_count > 0
2763
- assert_true(elapsed_seconds > lower_interval && elapsed_seconds < upper_interval,
2764
- "elapsed_seconds #{elapsed_seconds} is out of allowed range:\n" +
2765
- " lower: #{lower_interval} [sec]\n" +
2766
- " upper: #{upper_interval} [sec]")
2767
- end
2768
- assert_equal(limit, d.record_count - prev_count)
2769
- emit_count = d.emit_count
2770
- prev_count = d.record_count
2771
- end
2772
-
2773
- ## When all the lines are read and rate_period seconds are over
2774
- ## limit will reset and since there are no more logs to be read,
2775
- ## number_lines_read will be 0
2776
- sleep upper_interval
2777
- gw = d.instance.find_group_from_metadata(file_path)
2778
- assert_equal(0, gw.current_paths[file_path].number_lines_read)
2779
- end
2780
- end
2781
- end
2782
-
2783
- sub_test_case "Update watchers for rotation with follow_inodes" do
2784
- def test_updateTW_before_refreshTW_and_detach_before_refreshTW
2785
- config = config_element(
2786
- "ROOT",
2787
- "",
2788
- {
2789
- "path" => "#{@tmp_dir}/tail.txt*",
2790
- "pos_file" => "#{@tmp_dir}/tail.pos",
2791
- "tag" => "t1",
2792
- "format" => "none",
2793
- "read_from_head" => "true",
2794
- "follow_inodes" => "true",
2795
- # In order to detach the old watcher quickly.
2796
- "rotate_wait" => "1s",
2797
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
2798
- # called by a timer.
2799
- "refresh_interval" => "1h",
2800
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
2801
- # so disable it in order to reproduce the same condition stably.
2802
- "enable_stat_watcher" => "false",
2803
- }
2804
- )
2805
- d = create_driver(config, false)
2806
-
2807
- tail_watchers = []
2808
- stub.proxy(d.instance).setup_watcher do |tw|
2809
- tail_watchers.append(tw)
2810
- tw
2811
- end
2812
-
2813
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file1 log1"}
2814
-
2815
- d.run(expect_records: 4, timeout: 10) do
2816
- # Rotate (If the timing is bad, `TailWatcher::on_notify` might be called between mv and new-file-creation)
2817
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file1 log2"}
2818
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
2819
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file2 log1"}
2820
-
2821
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` updates the TailWatcher:
2822
- # TailWatcher(path: "tail.txt", inode: inode_0) => TailWatcher(path: "tail.txt", inode: inode_1)
2823
- # The old TailWathcer is detached here since `rotate_wait` is just `1s`.
2824
- sleep 3
2825
-
2826
- # This reproduces the following situation:
2827
- # Rotation => update_watcher => refresh_watchers
2828
- # This adds a new TailWatcher: TailWatcher(path: "tail.txt1", inode: inode_0)
2829
- d.instance.refresh_watchers
2830
-
2831
- # Append to the new current log file.
2832
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file2 log2"}
2833
- end
2834
-
2835
- inode_0 = tail_watchers[0].ino
2836
- inode_1 = tail_watchers[1].ino
2837
- record_values = d.events.collect { |event| event[2]["message"] }.sort
2838
- position_entries = []
2839
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
2840
- f.readlines(chomp: true).each do |line|
2841
- values = line.split("\t")
2842
- position_entries.append([values[0], values[1], values[2].to_i(16)])
2843
- end
2844
- end
2845
-
2846
- assert_equal(
2847
- {
2848
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2"],
2849
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1"],
2850
- tail_watcher_inodes: [inode_0, inode_1, inode_0],
2851
- tail_watcher_io_handler_opened_statuses: [false, false, false],
2852
- position_entries: [
2853
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_0],
2854
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_1],
2855
- ],
2856
- },
2857
- {
2858
- record_values: record_values,
2859
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
2860
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
2861
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
2862
- position_entries: position_entries
2863
- },
2864
- )
2865
- end
2866
-
2867
- def test_updateTW_before_refreshTW_and_detach_after_refreshTW
2868
- config = config_element(
2869
- "ROOT",
2870
- "",
2871
- {
2872
- "path" => "#{@tmp_dir}/tail.txt*",
2873
- "pos_file" => "#{@tmp_dir}/tail.pos",
2874
- "tag" => "t1",
2875
- "format" => "none",
2876
- "read_from_head" => "true",
2877
- "follow_inodes" => "true",
2878
- # In order to detach the old watcher after refresh_watchers.
2879
- "rotate_wait" => "4s",
2880
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
2881
- # called by a timer.
2882
- "refresh_interval" => "1h",
2883
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
2884
- # so disable it in order to reproduce the same condition stably.
2885
- "enable_stat_watcher" => "false",
2886
- }
2887
- )
2888
- d = create_driver(config, false)
2889
-
2890
- tail_watchers = []
2891
- stub.proxy(d.instance).setup_watcher do |tw|
2892
- tail_watchers.append(tw)
2893
- tw
2894
- end
2895
-
2896
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file1 log1"}
2897
-
2898
- d.run(expect_records: 4, timeout: 10) do
2899
- # Rotate (If the timing is bad, `TailWatcher::on_notify` might be called between mv and new-file-creation)
2900
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file1 log2"}
2901
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
2902
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file2 log1"}
2903
-
2904
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` updates the TailWatcher:
2905
- # TailWatcher(path: "tail.txt", inode: inode_0) => TailWatcher(path: "tail.txt", inode: inode_1)
2906
- sleep 2
2907
-
2908
- # This reproduces the following situation:
2909
- # Rotation => update_watcher => refresh_watchers
2910
- # This adds a new TailWatcher: TailWatcher(path: "tail.txt1", inode: inode_0)
2911
- d.instance.refresh_watchers
2912
-
2913
- # The old TailWathcer is detached here since `rotate_wait` is `4s`.
2914
- sleep 3
2915
-
2916
- # Append to the new current log file.
2917
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file2 log2"}
2918
- end
2919
-
2920
- inode_0 = tail_watchers[0].ino
2921
- inode_1 = tail_watchers[1].ino
2922
- record_values = d.events.collect { |event| event[2]["message"] }.sort
2923
- position_entries = []
2924
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
2925
- f.readlines(chomp: true).each do |line|
2926
- values = line.split("\t")
2927
- position_entries.append([values[0], values[1], values[2].to_i(16)])
2928
- end
2929
- end
2930
-
2931
- assert_equal(
2932
- {
2933
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2"],
2934
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1"],
2935
- tail_watcher_inodes: [inode_0, inode_1, inode_0],
2936
- tail_watcher_io_handler_opened_statuses: [false, false, false],
2937
- position_entries: [
2938
- # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
2939
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_0],
2940
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_1],
2941
- ],
2942
- },
2943
- {
2944
- record_values: record_values,
2945
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
2946
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
2947
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
2948
- position_entries: position_entries
2949
- },
2950
- )
2951
- end
2952
-
2953
- # The scenario where in_tail wrongly detaches TailWatcher.
2954
- # This is reported in https://github.com/fluent/fluentd/issues/4190.
2955
- def test_updateTW_after_refreshTW
2956
- config = config_element(
2957
- "ROOT",
2958
- "",
2959
- {
2960
- "path" => "#{@tmp_dir}/tail.txt*",
2961
- "pos_file" => "#{@tmp_dir}/tail.pos",
2962
- "tag" => "t1",
2963
- "format" => "none",
2964
- "read_from_head" => "true",
2965
- "follow_inodes" => "true",
2966
- # In order to detach the old watcher quickly.
2967
- "rotate_wait" => "1s",
2968
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
2969
- # called by a timer.
2970
- "refresh_interval" => "1h",
2971
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
2972
- # so disable it in order to reproduce the same condition stably.
2973
- "enable_stat_watcher" => "false",
2974
- }
2975
- )
2976
- d = create_driver(config, false)
2977
-
2978
- tail_watchers = []
2979
- stub.proxy(d.instance).setup_watcher do |tw|
2980
- tail_watchers.append(tw)
2981
- tw
2982
- end
2983
-
2984
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file1 log1"}
2985
-
2986
- d.run(expect_records: 4, timeout: 10) do
2987
- # Rotate (If the timing is bad, `TailWatcher::on_notify` might be called between mv and new-file-creation)
2988
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file1 log2"}
2989
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
2990
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file2 log1"}
2991
-
2992
- # This reproduces the following situation:
2993
- # Rotation => refresh_watchers => update_watcher
2994
- # This add a new TailWatcher: TailWatcher(path: "tail.txt", inode: inode_1)
2995
- # This overwrites `@tails["tail.txt"]`
2996
- d.instance.refresh_watchers
2997
-
2998
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to update the TailWatcher:
2999
- # TailWatcher(path: "tail.txt", inode: inode_0) => TailWatcher(path: "tail.txt", inode: inode_1)
3000
- # However, it is already added in `refresh_watcher`, so `update_watcher` doesn't create the new TailWatcher.
3001
- # The old TailWathcer is detached here since `rotate_wait` is just `1s`.
3002
- sleep 3
3003
-
3004
- # This adds a new TailWatcher: TailWatcher(path: "tail.txt1", inode: inode_0)
3005
- d.instance.refresh_watchers
3006
-
3007
- # Append to the new current log file.
3008
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file2 log2"}
3009
- end
3010
-
3011
- inode_0 = tail_watchers[0].ino
3012
- inode_1 = tail_watchers[1].ino
3013
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3014
- position_entries = []
3015
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3016
- f.readlines(chomp: true).each do |line|
3017
- values = line.split("\t")
3018
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3019
- end
3020
- end
3021
-
3022
- assert_equal(
3023
- {
3024
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2"],
3025
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1"],
3026
- tail_watcher_inodes: [inode_0, inode_1, inode_0],
3027
- tail_watcher_io_handler_opened_statuses: [false, false, false],
3028
- position_entries: [
3029
- # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
3030
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_0],
3031
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_1],
3032
- ],
3033
- },
3034
- {
3035
- record_values: record_values,
3036
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3037
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3038
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3039
- position_entries: position_entries
3040
- },
3041
- )
3042
- end
3043
-
3044
- def test_path_resurrection
3045
- config = config_element(
3046
- "ROOT",
3047
- "",
3048
- {
3049
- "path" => "#{@tmp_dir}/tail.txt*",
3050
- "pos_file" => "#{@tmp_dir}/tail.pos",
3051
- "tag" => "t1",
3052
- "format" => "none",
3053
- "read_from_head" => "true",
3054
- "follow_inodes" => "true",
3055
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
3056
- # called by a timer.
3057
- "refresh_interval" => "1h",
3058
- # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
3059
- # Because of this problem, log duplication can occur during `rotate_wait`.
3060
- # Need to set `rotate_wait 0` for a workaround.
3061
- "rotate_wait" => "0s",
3062
- }
3063
- )
3064
- d = create_driver(config, false)
3065
-
3066
- tail_watchers = []
3067
- stub.proxy(d.instance).setup_watcher do |tw|
3068
- tail_watchers.append(tw)
3069
- tw
3070
- end
3071
-
3072
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file1 log1"}
3073
-
3074
- d.run(expect_records: 5, timeout: 10) do
3075
- # Rotate
3076
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file1 log2"}
3077
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
3078
- # TailWatcher(path: "tail.txt", inode: inode_0) detects `tail.txt` disappeared.
3079
- # Call `update_watcher` to stop and discard self.
3080
- # If not discarding, then it will be a orphan and cause leak and log duplication.
3081
- #
3082
- # This reproduces the case where the notify to TailWatcher comes before the new file for the path
3083
- # is created during rotation.
3084
- # (stat_watcher notifies faster than a new file is created)
3085
- # Overall, this is a rotation operation, but from the TailWatcher, it appears as if the file
3086
- # was resurrected once it disappeared.
3087
- sleep 2 # On Windows and macOS, StatWatcher doesn't work, so need enough interval for TimeTrigger.
3088
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file2 log1"}
3089
-
3090
- # Add new TailWatchers
3091
- # tail.txt: TailWatcher(path: "tail.txt", inode: inode_1)
3092
- # tail.txt: TailWatcher(path: "tail.txt1", inode: inode_0)
3093
- # NOTE: If not discarding the first TailWatcher on notify, this makes it a orphan because
3094
- # this overwrites the `@tails[tail.txt]` by adding TailWatcher(path: "tail.txt", inode: inode_1)
3095
- d.instance.refresh_watchers
3096
-
3097
- # This does nothing.
3098
- # NOTE: If not discarding the first TailWatcher on notify, this add
3099
- # tail.txt1: TailWatcher(path: "tail.txt1", inode: inode_0)
3100
- # because the previous refresh_watcher overwrites `@tails[tail.txt]` and the inode_0 is lost.
3101
- # This would cause log duplication.
3102
- d.instance.refresh_watchers
3103
-
3104
- # Append to the old file
3105
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt1", "ab") {|f| f.puts "file1 log3"}
3106
-
3107
- # Append to the new current log file.
3108
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file2 log2"}
3109
- end
3110
-
3111
- inode_0 = Fluent::FileWrapper.stat("#{@tmp_dir}/tail.txt1").ino
3112
- inode_1 = Fluent::FileWrapper.stat("#{@tmp_dir}/tail.txt").ino
3113
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3114
- position_entries = []
3115
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3116
- f.readlines(chomp: true).each do |line|
3117
- values = line.split("\t")
3118
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3119
- end
3120
- end
3121
-
3122
- assert_equal(
3123
- {
3124
- record_values: ["file1 log1", "file1 log2", "file1 log3", "file2 log1", "file2 log2"],
3125
- tail_watcher_set: Set[
3126
- {
3127
- path: "#{@tmp_dir}/tail.txt",
3128
- inode: inode_0,
3129
- io_handler_opened_status: false,
3130
- },
3131
- {
3132
- path: "#{@tmp_dir}/tail.txt",
3133
- inode: inode_1,
3134
- io_handler_opened_status: false,
3135
- },
3136
- {
3137
- path: "#{@tmp_dir}/tail.txt1",
3138
- inode: inode_0,
3139
- io_handler_opened_status: false,
3140
- },
3141
- ],
3142
- position_entries: [
3143
- ["#{@tmp_dir}/tail.txt", "0000000000000021", inode_0],
3144
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_1],
3145
- ],
3146
- },
3147
- {
3148
- record_values: record_values,
3149
- tail_watcher_set: Set.new(tail_watchers.collect { |tw|
3150
- {
3151
- path: tw.path,
3152
- inode: tw.ino,
3153
- io_handler_opened_status: tw.instance_variable_get(:@io_handler)&.opened? || false,
3154
- }
3155
- }),
3156
- position_entries: position_entries,
3157
- },
3158
- )
3159
- end
3160
-
3161
- def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait
3162
- config = config_element(
3163
- "ROOT",
3164
- "",
3165
- {
3166
- "path" => "#{@tmp_dir}/tail.txt*",
3167
- "pos_file" => "#{@tmp_dir}/tail.pos",
3168
- "tag" => "t1",
3169
- "format" => "none",
3170
- "read_from_head" => "true",
3171
- "follow_inodes" => "true",
3172
- "rotate_wait" => "3s",
3173
- "refresh_interval" => "1h",
3174
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
3175
- # so disable it in order to reproduce the same condition stably.
3176
- "enable_stat_watcher" => "false",
3177
- }
3178
- )
3179
- d = create_driver(config, false)
3180
-
3181
- tail_watchers = []
3182
- stub.proxy(d.instance).setup_watcher do |tw|
3183
- tail_watchers.append(tw)
3184
- mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.
3185
- tw
3186
- end
3187
-
3188
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
3189
-
3190
- d.run(expect_records: 6, timeout: 15) do
3191
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
3192
-
3193
- sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3194
-
3195
- FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
3196
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
3197
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
3198
-
3199
- sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3200
-
3201
- # Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)
3202
- [1, 0].each do |i|
3203
- FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
3204
- end
3205
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
3206
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3207
-
3208
- # Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.
3209
- # (Note: Currently, there is no harm in duplicate calls)
3210
- sleep 4
3211
- end
3212
-
3213
- inode_0 = tail_watchers[0]&.ino
3214
- inode_1 = tail_watchers[1]&.ino
3215
- inode_2 = tail_watchers[2]&.ino
3216
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3217
- position_entries = []
3218
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3219
- f.readlines(chomp: true).each do |line|
3220
- values = line.split("\t")
3221
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3222
- end
3223
- end
3224
-
3225
- assert_equal(
3226
- {
3227
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
3228
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
3229
- tail_watcher_inodes: [inode_0, inode_1, inode_2],
3230
- tail_watcher_io_handler_opened_statuses: [false, false, false],
3231
- position_entries: [
3232
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_0],
3233
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_1],
3234
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
3235
- ],
3236
- },
3237
- {
3238
- record_values: record_values,
3239
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3240
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3241
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3242
- position_entries: position_entries
3243
- },
3244
- )
3245
- end
3246
- end
3247
-
3248
- sub_test_case "Update watchers for rotation without follow_inodes" do
3249
- # The scenario where in_tail wrongly unwatches the PositionEntry.
3250
- # This is reported in https://github.com/fluent/fluentd/issues/3614.
3251
- def test_refreshTW_during_rotation
3252
- config = config_element(
3253
- "ROOT",
3254
- "",
3255
- {
3256
- "path" => "#{@tmp_dir}/tail.txt0",
3257
- "pos_file" => "#{@tmp_dir}/tail.pos",
3258
- "tag" => "t1",
3259
- "format" => "none",
3260
- "read_from_head" => "true",
3261
- # In order to detach the old watcher quickly.
3262
- "rotate_wait" => "3s",
3263
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
3264
- # called by a timer.
3265
- "refresh_interval" => "1h",
3266
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
3267
- # so disable it in order to reproduce the same condition stably.
3268
- "enable_stat_watcher" => "false",
3269
- }
3270
- )
3271
- d = create_driver(config, false)
3272
-
3273
- tail_watchers = []
3274
- stub.proxy(d.instance).setup_watcher do |tw|
3275
- tail_watchers.append(tw)
3276
- tw
3277
- end
3278
-
3279
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
3280
-
3281
- d.run(expect_records: 6, timeout: 15) do
3282
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
3283
- FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
3284
-
3285
- # This reproduces the following situation:
3286
- # `refresh_watchers` is called during the rotation process and it detects the current file being lost.
3287
- # Then it stops and unwatches the TailWatcher.
3288
- d.instance.refresh_watchers
3289
-
3290
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
3291
-
3292
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to add the new TailWatcher.
3293
- # After `rotate_wait` interval, the PositionEntry is unwatched.
3294
- # HOWEVER, the new TailWatcher is still using that PositionEntry, so this breaks the PositionFile!!
3295
- # That PositionEntry is removed from `PositionFile::map`, but it is still working and remaining in the real pos file.
3296
- sleep 5
3297
-
3298
- # Append to the new current log file.
3299
- # The PositionEntry is updated although it does not exist in `PositionFile::map`.
3300
- # `PositionFile::map`: empty
3301
- # Real pos file: `.../tail.txt 0000000000000016 (inode)`
3302
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
3303
-
3304
- # Rotate again
3305
- [1, 0].each do |i|
3306
- FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
3307
- end
3308
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
3309
-
3310
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to update the TailWatcher.
3311
- sleep 3
3312
-
3313
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3314
- end
3315
-
3316
- inode_0 = tail_watchers[0]&.ino
3317
- inode_1 = tail_watchers[1]&.ino
3318
- inode_2 = tail_watchers[2]&.ino
3319
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3320
- position_entries = []
3321
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3322
- f.readlines(chomp: true).each do |line|
3323
- values = line.split("\t")
3324
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3325
- end
3326
- end
3327
-
3328
- assert_equal(
3329
- {
3330
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
3331
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
3332
- tail_watcher_inodes: [inode_0, inode_1, inode_2],
3333
- tail_watcher_io_handler_opened_statuses: [false, false, false],
3334
- position_entries: [
3335
- # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
3336
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
3337
- ],
3338
- },
3339
- {
3340
- record_values: record_values,
3341
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3342
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3343
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3344
- position_entries: position_entries
3345
- },
3346
- )
3347
- end
3348
-
3349
- def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait
3350
- config = config_element(
3351
- "ROOT",
3352
- "",
3353
- {
3354
- "path" => "#{@tmp_dir}/tail.txt0",
3355
- "pos_file" => "#{@tmp_dir}/tail.pos",
3356
- "tag" => "t1",
3357
- "format" => "none",
3358
- "read_from_head" => "true",
3359
- "rotate_wait" => "3s",
3360
- "refresh_interval" => "1h",
3361
- }
3362
- )
3363
- d = create_driver(config, false)
3364
-
3365
- tail_watchers = []
3366
- stub.proxy(d.instance).setup_watcher do |tw|
3367
- tail_watchers.append(tw)
3368
- mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.
3369
- tw
3370
- end
3371
-
3372
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
3373
-
3374
- d.run(expect_records: 6, timeout: 15) do
3375
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
3376
-
3377
- sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3378
-
3379
- FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
3380
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
3381
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
3382
-
3383
- sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3384
-
3385
- # Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)
3386
- [1, 0].each do |i|
3387
- FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
3388
- end
3389
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
3390
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3391
-
3392
- # Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.
3393
- # (Note: Currently, there is no harm in duplicate calls)
3394
- sleep 4
3395
- end
3396
-
3397
- inode_0 = tail_watchers[0]&.ino
3398
- inode_1 = tail_watchers[1]&.ino
3399
- inode_2 = tail_watchers[2]&.ino
3400
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3401
- position_entries = []
3402
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3403
- f.readlines(chomp: true).each do |line|
3404
- values = line.split("\t")
3405
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3406
- end
3407
- end
3408
-
3409
- assert_equal(
3410
- {
3411
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
3412
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
3413
- tail_watcher_inodes: [inode_0, inode_1, inode_2],
3414
- tail_watcher_io_handler_opened_statuses: [false, false, false],
3415
- position_entries: [
3416
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
3417
- ],
3418
- },
3419
- {
3420
- record_values: record_values,
3421
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3422
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3423
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3424
- position_entries: position_entries
3425
- },
3426
- )
3427
- end
3428
- end
3429
- end