fluentd 1.16.5 → 1.17.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (268) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +88 -0
  3. data/README.md +2 -1
  4. data/Rakefile +1 -1
  5. data/SECURITY.md +2 -2
  6. data/fluent.conf +14 -14
  7. data/lib/fluent/command/binlog_reader.rb +1 -1
  8. data/lib/fluent/command/cap_ctl.rb +4 -4
  9. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  10. data/lib/fluent/compat/propagate_default.rb +4 -4
  11. data/lib/fluent/config/configure_proxy.rb +2 -2
  12. data/lib/fluent/config/types.rb +1 -1
  13. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  14. data/lib/fluent/configurable.rb +2 -2
  15. data/lib/fluent/counter/mutex_hash.rb +1 -1
  16. data/lib/fluent/fluent_log_event_router.rb +0 -2
  17. data/lib/fluent/log/console_adapter.rb +4 -2
  18. data/lib/fluent/plugin/buf_file.rb +1 -1
  19. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  20. data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
  21. data/lib/fluent/plugin/filter_parser.rb +26 -8
  22. data/lib/fluent/plugin/in_exec.rb +14 -2
  23. data/lib/fluent/plugin/in_http.rb +19 -54
  24. data/lib/fluent/plugin/in_sample.rb +13 -7
  25. data/lib/fluent/plugin/in_tail.rb +99 -25
  26. data/lib/fluent/plugin/out_copy.rb +1 -1
  27. data/lib/fluent/plugin/out_file.rb +8 -0
  28. data/lib/fluent/plugin/out_http.rb +137 -13
  29. data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
  30. data/lib/fluent/plugin/parser_json.rb +26 -17
  31. data/lib/fluent/plugin/parser_msgpack.rb +24 -3
  32. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  33. data/lib/fluent/plugin_helper/metrics.rb +2 -2
  34. data/lib/fluent/registry.rb +6 -6
  35. data/lib/fluent/test/output_test.rb +1 -1
  36. data/lib/fluent/unique_id.rb +1 -1
  37. data/lib/fluent/version.rb +1 -1
  38. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  39. metadata +109 -459
  40. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  41. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  42. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  43. data/.github/ISSUE_TEMPLATE.md +0 -17
  44. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  45. data/.github/workflows/stale-actions.yml +0 -24
  46. data/.github/workflows/test.yml +0 -32
  47. data/.gitignore +0 -30
  48. data/Gemfile +0 -9
  49. data/fluentd.gemspec +0 -54
  50. data/test/command/test_binlog_reader.rb +0 -362
  51. data/test/command/test_ca_generate.rb +0 -70
  52. data/test/command/test_cap_ctl.rb +0 -100
  53. data/test/command/test_cat.rb +0 -128
  54. data/test/command/test_ctl.rb +0 -56
  55. data/test/command/test_fluentd.rb +0 -1291
  56. data/test/command/test_plugin_config_formatter.rb +0 -397
  57. data/test/command/test_plugin_generator.rb +0 -109
  58. data/test/compat/test_calls_super.rb +0 -166
  59. data/test/compat/test_parser.rb +0 -92
  60. data/test/config/assertions.rb +0 -42
  61. data/test/config/test_config_parser.rb +0 -551
  62. data/test/config/test_configurable.rb +0 -1784
  63. data/test/config/test_configure_proxy.rb +0 -604
  64. data/test/config/test_dsl.rb +0 -415
  65. data/test/config/test_element.rb +0 -518
  66. data/test/config/test_literal_parser.rb +0 -309
  67. data/test/config/test_plugin_configuration.rb +0 -56
  68. data/test/config/test_section.rb +0 -191
  69. data/test/config/test_system_config.rb +0 -195
  70. data/test/config/test_types.rb +0 -408
  71. data/test/counter/test_client.rb +0 -563
  72. data/test/counter/test_error.rb +0 -44
  73. data/test/counter/test_mutex_hash.rb +0 -179
  74. data/test/counter/test_server.rb +0 -589
  75. data/test/counter/test_store.rb +0 -258
  76. data/test/counter/test_validator.rb +0 -137
  77. data/test/helper.rb +0 -155
  78. data/test/helpers/fuzzy_assert.rb +0 -89
  79. data/test/helpers/process_extenstion.rb +0 -33
  80. data/test/log/test_console_adapter.rb +0 -110
  81. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  82. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  83. data/test/plugin/data/2010/01/20100102.log +0 -0
  84. data/test/plugin/data/log/bar +0 -0
  85. data/test/plugin/data/log/foo/bar.log +0 -0
  86. data/test/plugin/data/log/foo/bar2 +0 -0
  87. data/test/plugin/data/log/test.log +0 -0
  88. data/test/plugin/data/sd_file/config +0 -11
  89. data/test/plugin/data/sd_file/config.json +0 -17
  90. data/test/plugin/data/sd_file/config.yaml +0 -11
  91. data/test/plugin/data/sd_file/config.yml +0 -11
  92. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  93. data/test/plugin/in_tail/test_fifo.rb +0 -121
  94. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  95. data/test/plugin/in_tail/test_position_file.rb +0 -346
  96. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  97. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  98. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  99. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  100. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  101. data/test/plugin/test_bare_output.rb +0 -131
  102. data/test/plugin/test_base.rb +0 -247
  103. data/test/plugin/test_buf_file.rb +0 -1314
  104. data/test/plugin/test_buf_file_single.rb +0 -898
  105. data/test/plugin/test_buf_memory.rb +0 -42
  106. data/test/plugin/test_buffer.rb +0 -1493
  107. data/test/plugin/test_buffer_chunk.rb +0 -209
  108. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  109. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  110. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  111. data/test/plugin/test_compressable.rb +0 -87
  112. data/test/plugin/test_file_util.rb +0 -96
  113. data/test/plugin/test_filter.rb +0 -368
  114. data/test/plugin/test_filter_grep.rb +0 -697
  115. data/test/plugin/test_filter_parser.rb +0 -731
  116. data/test/plugin/test_filter_record_transformer.rb +0 -577
  117. data/test/plugin/test_filter_stdout.rb +0 -207
  118. data/test/plugin/test_formatter_csv.rb +0 -136
  119. data/test/plugin/test_formatter_hash.rb +0 -38
  120. data/test/plugin/test_formatter_json.rb +0 -61
  121. data/test/plugin/test_formatter_ltsv.rb +0 -70
  122. data/test/plugin/test_formatter_msgpack.rb +0 -28
  123. data/test/plugin/test_formatter_out_file.rb +0 -116
  124. data/test/plugin/test_formatter_single_value.rb +0 -44
  125. data/test/plugin/test_formatter_tsv.rb +0 -76
  126. data/test/plugin/test_in_debug_agent.rb +0 -49
  127. data/test/plugin/test_in_exec.rb +0 -261
  128. data/test/plugin/test_in_forward.rb +0 -1178
  129. data/test/plugin/test_in_gc_stat.rb +0 -62
  130. data/test/plugin/test_in_http.rb +0 -1102
  131. data/test/plugin/test_in_monitor_agent.rb +0 -922
  132. data/test/plugin/test_in_object_space.rb +0 -66
  133. data/test/plugin/test_in_sample.rb +0 -190
  134. data/test/plugin/test_in_syslog.rb +0 -505
  135. data/test/plugin/test_in_tail.rb +0 -3288
  136. data/test/plugin/test_in_tcp.rb +0 -328
  137. data/test/plugin/test_in_udp.rb +0 -296
  138. data/test/plugin/test_in_unix.rb +0 -181
  139. data/test/plugin/test_input.rb +0 -137
  140. data/test/plugin/test_metadata.rb +0 -89
  141. data/test/plugin/test_metrics.rb +0 -294
  142. data/test/plugin/test_metrics_local.rb +0 -96
  143. data/test/plugin/test_multi_output.rb +0 -204
  144. data/test/plugin/test_out_copy.rb +0 -308
  145. data/test/plugin/test_out_exec.rb +0 -312
  146. data/test/plugin/test_out_exec_filter.rb +0 -606
  147. data/test/plugin/test_out_file.rb +0 -1038
  148. data/test/plugin/test_out_forward.rb +0 -1349
  149. data/test/plugin/test_out_http.rb +0 -429
  150. data/test/plugin/test_out_null.rb +0 -105
  151. data/test/plugin/test_out_relabel.rb +0 -28
  152. data/test/plugin/test_out_roundrobin.rb +0 -146
  153. data/test/plugin/test_out_secondary_file.rb +0 -458
  154. data/test/plugin/test_out_stdout.rb +0 -205
  155. data/test/plugin/test_out_stream.rb +0 -103
  156. data/test/plugin/test_output.rb +0 -1334
  157. data/test/plugin/test_output_as_buffered.rb +0 -2024
  158. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  159. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  160. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  161. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  162. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  163. data/test/plugin/test_output_as_standard.rb +0 -374
  164. data/test/plugin/test_owned_by.rb +0 -35
  165. data/test/plugin/test_parser.rb +0 -399
  166. data/test/plugin/test_parser_apache.rb +0 -42
  167. data/test/plugin/test_parser_apache2.rb +0 -47
  168. data/test/plugin/test_parser_apache_error.rb +0 -45
  169. data/test/plugin/test_parser_csv.rb +0 -200
  170. data/test/plugin/test_parser_json.rb +0 -138
  171. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  172. data/test/plugin/test_parser_multiline.rb +0 -111
  173. data/test/plugin/test_parser_nginx.rb +0 -88
  174. data/test/plugin/test_parser_none.rb +0 -52
  175. data/test/plugin/test_parser_regexp.rb +0 -284
  176. data/test/plugin/test_parser_syslog.rb +0 -650
  177. data/test/plugin/test_parser_tsv.rb +0 -122
  178. data/test/plugin/test_sd_file.rb +0 -228
  179. data/test/plugin/test_sd_srv.rb +0 -230
  180. data/test/plugin/test_storage.rb +0 -167
  181. data/test/plugin/test_storage_local.rb +0 -335
  182. data/test/plugin/test_string_util.rb +0 -26
  183. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  184. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  185. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  186. data/test/plugin_helper/data/cert/cert.pem +0 -19
  187. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  188. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  189. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  191. data/test/plugin_helper/data/cert/empty.pem +0 -0
  192. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  193. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  194. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  195. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  196. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  197. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  198. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  199. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  200. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  201. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  202. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  203. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  204. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  205. data/test/plugin_helper/http_server/test_app.rb +0 -65
  206. data/test/plugin_helper/http_server/test_route.rb +0 -32
  207. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  208. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  209. data/test/plugin_helper/test_cert_option.rb +0 -25
  210. data/test/plugin_helper/test_child_process.rb +0 -862
  211. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  212. data/test/plugin_helper/test_event_emitter.rb +0 -80
  213. data/test/plugin_helper/test_event_loop.rb +0 -52
  214. data/test/plugin_helper/test_extract.rb +0 -194
  215. data/test/plugin_helper/test_formatter.rb +0 -255
  216. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  217. data/test/plugin_helper/test_inject.rb +0 -561
  218. data/test/plugin_helper/test_metrics.rb +0 -137
  219. data/test/plugin_helper/test_parser.rb +0 -264
  220. data/test/plugin_helper/test_record_accessor.rb +0 -238
  221. data/test/plugin_helper/test_retry_state.rb +0 -1006
  222. data/test/plugin_helper/test_server.rb +0 -1895
  223. data/test/plugin_helper/test_service_discovery.rb +0 -165
  224. data/test/plugin_helper/test_socket.rb +0 -146
  225. data/test/plugin_helper/test_storage.rb +0 -542
  226. data/test/plugin_helper/test_thread.rb +0 -164
  227. data/test/plugin_helper/test_timer.rb +0 -130
  228. data/test/scripts/exec_script.rb +0 -32
  229. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  230. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  231. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  232. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  233. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  234. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  235. data/test/test_capability.rb +0 -74
  236. data/test/test_clock.rb +0 -164
  237. data/test/test_config.rb +0 -369
  238. data/test/test_configdsl.rb +0 -148
  239. data/test/test_daemonizer.rb +0 -91
  240. data/test/test_engine.rb +0 -203
  241. data/test/test_event.rb +0 -531
  242. data/test/test_event_router.rb +0 -348
  243. data/test/test_event_time.rb +0 -199
  244. data/test/test_file_wrapper.rb +0 -53
  245. data/test/test_filter.rb +0 -121
  246. data/test/test_fluent_log_event_router.rb +0 -99
  247. data/test/test_formatter.rb +0 -369
  248. data/test/test_input.rb +0 -31
  249. data/test/test_log.rb +0 -1076
  250. data/test/test_match.rb +0 -148
  251. data/test/test_mixin.rb +0 -351
  252. data/test/test_msgpack_factory.rb +0 -50
  253. data/test/test_oj_options.rb +0 -55
  254. data/test/test_output.rb +0 -278
  255. data/test/test_plugin.rb +0 -251
  256. data/test/test_plugin_classes.rb +0 -370
  257. data/test/test_plugin_helper.rb +0 -81
  258. data/test/test_plugin_id.rb +0 -119
  259. data/test/test_process.rb +0 -14
  260. data/test/test_root_agent.rb +0 -951
  261. data/test/test_static_config_analysis.rb +0 -177
  262. data/test/test_supervisor.rb +0 -821
  263. data/test/test_test_drivers.rb +0 -136
  264. data/test/test_time_formatter.rb +0 -301
  265. data/test/test_time_parser.rb +0 -362
  266. data/test/test_tls.rb +0 -65
  267. data/test/test_unique_id.rb +0 -47
  268. data/test/test_variable_store.rb +0 -65
@@ -1,3288 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/input'
3
- require 'fluent/plugin/in_tail'
4
- require 'fluent/plugin/buffer'
5
- require 'fluent/system_config'
6
- require 'fluent/file_wrapper'
7
- require 'net/http'
8
- require 'flexmock/test_unit'
9
- require 'timecop'
10
- require 'tmpdir'
11
- require 'securerandom'
12
-
13
- class TailInputTest < Test::Unit::TestCase
14
- include FlexMock::TestCase
15
-
16
- def tmp_dir
17
- File.join(File.dirname(__FILE__), "..", "tmp", "tail#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
18
- end
19
-
20
- def setup
21
- Fluent::Test.setup
22
- @tmp_dir = tmp_dir
23
- cleanup_directory(@tmp_dir)
24
- end
25
-
26
- def teardown
27
- super
28
- cleanup_directory(@tmp_dir)
29
- Fluent::Engine.stop
30
- Timecop.return
31
- end
32
-
33
- def cleanup_directory(path)
34
- unless Dir.exist?(path)
35
- FileUtils.mkdir_p(path)
36
- return
37
- end
38
-
39
- FileUtils.remove_entry_secure(path, true)
40
- end
41
-
42
- def cleanup_file(path)
43
- FileUtils.remove_entry_secure(path, true)
44
- end
45
-
46
- def create_target_info(path)
47
- Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
48
- end
49
-
50
- ROOT_CONFIG = config_element("ROOT", "", {
51
- "tag" => "t1",
52
- "rotate_wait" => "2s",
53
- "refresh_interval" => "1s"
54
- })
55
-
56
- def base_config
57
- ROOT_CONFIG + config_element("", "", { "path" => "#{@tmp_dir}/tail.txt" })
58
- end
59
-
60
- def common_config
61
- base_config + config_element("", "", { "pos_file" => "#{@tmp_dir}/tail.pos" })
62
- end
63
-
64
- def common_follow_inode_config
65
- config_element("ROOT", "", {
66
- "path" => "#{@tmp_dir}/tail.txt*",
67
- "pos_file" => "#{@tmp_dir}/tail.pos",
68
- "tag" => "t1",
69
- "refresh_interval" => "1s",
70
- "read_from_head" => "true",
71
- "format" => "none",
72
- "rotate_wait" => "1s",
73
- "follow_inodes" => "true"
74
- })
75
- end
76
-
77
- CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
78
- CONFIG_DISABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
79
- CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
80
- CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
81
- SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
82
- PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
83
- MULTILINE_CONFIG = config_element(
84
- "", "", {
85
- "format" => "multiline",
86
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
87
- "format_firstline" => "/^[s]/"
88
- })
89
- PARSE_MULTILINE_CONFIG = config_element(
90
- "", "", {},
91
- [config_element("parse", "", {
92
- "@type" => "multiline",
93
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
94
- "format_firstline" => "/^[s]/"
95
- })
96
- ])
97
-
98
- MULTILINE_CONFIG_WITH_NEWLINE = config_element(
99
- "", "", {
100
- "format" => "multiline",
101
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.[^\\n]+))?/",
102
- "format_firstline" => "/^[s]/"
103
- })
104
- PARSE_MULTILINE_CONFIG_WITH_NEWLINE = config_element(
105
- "", "", {},
106
- [config_element("parse", "", {
107
- "@type" => "multiline",
108
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.[^\\n]+))?/",
109
- "format_firstline" => "/^[s]/"
110
- })
111
- ])
112
-
113
- EX_ROTATE_WAIT = 0
114
- EX_FOLLOW_INODES = false
115
-
116
- def ex_config
117
- config_element("", "", {
118
- "tag" => "tail",
119
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
120
- "format" => "none",
121
- "pos_file" => "#{@tmp_dir}/tail.pos",
122
- "read_from_head" => true,
123
- "refresh_interval" => 30,
124
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
125
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
126
- })
127
- end
128
-
129
- def tailing_group_pattern
130
- "/#{@tmp_dir}\/(?<podname>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\/[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container>.+)-(?<docker_id>[a-z0-9]{6})\.log$/"
131
- end
132
-
133
- DEBUG_LOG_LEVEL = config_element("", "", {
134
- "@log_level" => "debug"
135
- })
136
-
137
- def create_group_directive(pattern, rate_period, *rules)
138
- config_element("", "", {}, [
139
- config_element("group", "", {
140
- "pattern" => pattern,
141
- "rate_period" => rate_period
142
- }, rules)
143
- ])
144
- end
145
-
146
- def create_rule_directive(match_named_captures, limit)
147
- params = {
148
- "limit" => limit,
149
- "match" => match_named_captures,
150
- }
151
-
152
- config_element("rule", "", params)
153
- end
154
-
155
- def create_path_element(path)
156
- config_element("source", "", { "path" => "#{@tmp_dir}/#{path}" })
157
- end
158
-
159
- def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
160
- config = use_common_conf ? common_config + conf : conf
161
- Fluent::Test::Driver::Input.new(Fluent::Plugin::TailInput).configure(config)
162
- end
163
-
164
- sub_test_case "configure" do
165
- test "plain single line" do
166
- d = create_driver
167
- assert_equal(["#{@tmp_dir}/tail.txt"], d.instance.paths)
168
- assert_equal("t1", d.instance.tag)
169
- assert_equal(2, d.instance.rotate_wait)
170
- assert_equal("#{@tmp_dir}/tail.pos", d.instance.pos_file)
171
- assert_equal(1000, d.instance.read_lines_limit)
172
- assert_equal(-1, d.instance.read_bytes_limit_per_second)
173
- assert_equal(false, d.instance.ignore_repeated_permission_error)
174
- assert_nothing_raised do
175
- d.instance.have_read_capability?
176
- end
177
- end
178
-
179
- data("empty" => config_element,
180
- "w/o @type" => config_element("", "", {}, [config_element("parse", "", {})]))
181
- test "w/o parse section" do |conf|
182
- assert_raise(Fluent::ConfigError) do
183
- create_driver(conf)
184
- end
185
- end
186
-
187
- test "multi paths with path_delimiter" do
188
- c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "|" })
189
- d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
190
- assert_equal(["tail.txt", "test2", "tmp,dev"], d.instance.paths)
191
- end
192
-
193
- test "multi paths with same path configured twice" do
194
- c = config_element("ROOT", "", { "path" => "test1.txt,test2.txt,test1.txt", "tag" => "t1", "path_delimiter" => "," })
195
- d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
196
- assert_equal(["test2.txt","test1.txt"].sort, d.instance.paths.sort)
197
- end
198
-
199
- test "multi paths with invaid path_delimiter" do
200
- c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "*" })
201
- assert_raise(Fluent::ConfigError) do
202
- create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
203
- end
204
- end
205
-
206
- test "follow_inodes w/o pos file" do
207
- assert_raise(Fluent::ConfigError) do
208
- create_driver(base_config + config_element('', '', {'follow_inodes' => 'true'}))
209
- end
210
- end
211
-
212
- sub_test_case "log throttling per file" do
213
- test "w/o watcher timer is invalid" do
214
- conf = CONFIG_DISABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
215
- assert_raise(Fluent::ConfigError) do
216
- create_driver(conf)
217
- end
218
- end
219
-
220
- test "valid" do
221
- conf = config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
222
- assert_raise(Fluent::ConfigError) do
223
- create_driver(conf)
224
- end
225
- end
226
- end
227
-
228
- test "both enable_watch_timer and enable_stat_watcher are false" do
229
- assert_raise(Fluent::ConfigError) do
230
- create_driver(CONFIG_DISABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
231
- end
232
- end
233
-
234
- sub_test_case "encoding" do
235
- test "valid" do
236
- conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" })
237
- d = create_driver(conf)
238
- assert_equal(Encoding::UTF_8, d.instance.encoding)
239
- end
240
-
241
- test "invalid" do
242
- conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "no-such-encoding" })
243
- assert_raise(Fluent::ConfigError) do
244
- create_driver(conf)
245
- end
246
- end
247
- end
248
-
249
- sub_test_case "from_encoding" do
250
- test "only specified from_encoding raise ConfigError" do
251
- conf = SINGLE_LINE_CONFIG + config_element("", "", { "from_encoding" => "utf-8" })
252
- assert_raise(Fluent::ConfigError) do
253
- create_driver(conf)
254
- end
255
- end
256
-
257
- test "valid" do
258
- conf = SINGLE_LINE_CONFIG + config_element("", "", {
259
- "from_encoding" => "utf-8",
260
- "encoding" => "utf-8"
261
- })
262
- d = create_driver(conf)
263
- assert_equal(Encoding::UTF_8, d.instance.from_encoding)
264
- end
265
-
266
- test "invalid" do
267
- conf = SINGLE_LINE_CONFIG + config_element("", "", {
268
- "from_encoding" => "no-such-encoding",
269
- "encoding" => "utf-8"
270
- })
271
- assert_raise(Fluent::ConfigError) do
272
- create_driver(conf)
273
- end
274
- end
275
- end
276
- end
277
-
278
- sub_test_case "configure group" do
279
- test "<rule> required" do
280
- conf = create_group_directive('.', '1m') + SINGLE_LINE_CONFIG
281
- assert_raise(Fluent::ConfigError) do
282
- create_driver(conf)
283
- end
284
- end
285
-
286
- test "valid configuration" do
287
- rule1 = create_rule_directive({
288
- "namespace"=> "/namespace-a/",
289
- "podname"=> "/podname-[b|c]/"
290
- }, 100)
291
- rule2 = create_rule_directive({
292
- "namespace"=> "/namespace-[d|e]/",
293
- "podname"=> "/podname-f/",
294
- }, 50)
295
- rule3 = create_rule_directive({
296
- "podname"=> "/podname-g/",
297
- }, -1)
298
- rule4 = create_rule_directive({
299
- "namespace"=> "/namespace-h/",
300
- }, 0)
301
-
302
- conf = create_group_directive(tailing_group_pattern, '1m', rule1, rule2, rule3, rule4) + SINGLE_LINE_CONFIG
303
- assert_nothing_raised do
304
- create_driver(conf)
305
- end
306
- end
307
-
308
- test "limit should be greater than DEFAULT_LIMIT (-1)" do
309
- rule1 = create_rule_directive({
310
- "namespace"=> "/namespace-a/",
311
- "podname"=> "/podname-[b|c]/",
312
- }, -100)
313
- rule2 = create_rule_directive({
314
- "namespace"=> "/namespace-[d|e]/",
315
- "podname"=> "/podname-f/",
316
- }, 50)
317
- conf = create_group_directive(tailing_group_pattern, '1m', rule1, rule2) + SINGLE_LINE_CONFIG
318
- assert_raise(RuntimeError) do
319
- create_driver(conf)
320
- end
321
- end
322
- end
323
-
324
- sub_test_case "group rules line limit resolution" do
325
- test "valid" do
326
- rule1 = create_rule_directive({
327
- "namespace"=> "/namespace-a/",
328
- "podname"=> "/podname-[b|c]/",
329
- }, 50)
330
- rule2 = create_rule_directive({
331
- "podname"=> "/podname-[b|c]/",
332
- }, 400)
333
- rule3 = create_rule_directive({
334
- "namespace"=> "/namespace-a/",
335
- }, 100)
336
-
337
- conf = create_group_directive(tailing_group_pattern, '1m', rule3, rule1, rule2) + SINGLE_LINE_CONFIG
338
- assert_nothing_raised do
339
- d = create_driver(conf)
340
- instance = d.instance
341
-
342
- metadata = {
343
- "namespace"=> "namespace-a",
344
- "podname"=> "podname-b",
345
- }
346
- assert_equal(50, instance.find_group(metadata).limit)
347
-
348
- metadata = {
349
- "namespace" => "namespace-a",
350
- "podname" => "podname-c",
351
- }
352
- assert_equal(50, instance.find_group(metadata).limit)
353
-
354
- metadata = {
355
- "namespace" => "namespace-a",
356
- "podname" => "podname-d",
357
- }
358
- assert_equal(100, instance.find_group(metadata).limit)
359
-
360
- metadata = {
361
- "namespace" => "namespace-f",
362
- "podname" => "podname-b",
363
- }
364
- assert_equal(400, instance.find_group(metadata).limit)
365
-
366
- metadata = {
367
- "podname" => "podname-c",
368
- }
369
- assert_equal(400, instance.find_group(metadata).limit)
370
-
371
- assert_equal(-1, instance.find_group({}).limit)
372
- end
373
- end
374
- end
375
-
376
- sub_test_case "files should be placed in groups" do
377
- test "invalid regex pattern places files in default group" do
378
- rule1 = create_rule_directive({}, 100) ## limits default groups
379
- conf = ROOT_CONFIG + DEBUG_LOG_LEVEL + create_group_directive(tailing_group_pattern, '1m', rule1) + create_path_element("test*.txt") + SINGLE_LINE_CONFIG
380
-
381
- d = create_driver(conf, false)
382
- Fluent::FileWrapper.open("#{@tmp_dir}/test1.txt", 'w')
383
- Fluent::FileWrapper.open("#{@tmp_dir}/test2.txt", 'w')
384
- Fluent::FileWrapper.open("#{@tmp_dir}/test3.txt", 'w')
385
-
386
- d.run do
387
- ## checking default group_watcher's paths
388
- instance = d.instance
389
- key = instance.default_group_key
390
-
391
- assert_equal(3, instance.log.logs.count{|a| a.match?("Cannot find group from metadata, Adding file in the default group\n")})
392
- assert_equal(3, instance.group_watchers[key].size)
393
- assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test1.txt'))
394
- assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test2.txt'))
395
- assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test3.txt'))
396
- end
397
- end
398
-
399
- test "valid regex pattern places file in their respective groups" do
400
- rule1 = create_rule_directive({
401
- "namespace"=> "/test-namespace1/",
402
- "podname"=> "/test-podname1/",
403
- }, 100)
404
- rule2 = create_rule_directive({
405
- "namespace"=> "/test-namespace1/",
406
- }, 200)
407
- rule3 = create_rule_directive({
408
- "podname"=> "/test-podname2/",
409
- }, 300)
410
- rule4 = create_rule_directive({}, 400)
411
-
412
- path_element = create_path_element("test-podname*.log")
413
-
414
- conf = ROOT_CONFIG + create_group_directive(tailing_group_pattern, '1m', rule4, rule3, rule2, rule1) + path_element + SINGLE_LINE_CONFIG
415
- d = create_driver(conf, false)
416
-
417
- file1 = File.join(@tmp_dir, "test-podname1_test-namespace1_test-container-15fabq.log")
418
- file2 = File.join(@tmp_dir, "test-podname3_test-namespace1_test-container-15fabq.log")
419
- file3 = File.join(@tmp_dir, "test-podname2_test-namespace2_test-container-15fabq.log")
420
- file4 = File.join(@tmp_dir, "test-podname4_test-namespace3_test-container-15fabq.log")
421
-
422
- d.run do
423
- Fluent::FileWrapper.open(file1, 'w')
424
- Fluent::FileWrapper.open(file2, 'w')
425
- Fluent::FileWrapper.open(file3, 'w')
426
- Fluent::FileWrapper.open(file4, 'w')
427
-
428
- instance = d.instance
429
- assert_equal(100, instance.find_group_from_metadata(file1).limit)
430
- assert_equal(200, instance.find_group_from_metadata(file2).limit)
431
- assert_equal(300, instance.find_group_from_metadata(file3).limit)
432
- assert_equal(400, instance.find_group_from_metadata(file4).limit)
433
- end
434
- end
435
- end
436
-
437
- sub_test_case "singleline" do
438
- data(flat: SINGLE_LINE_CONFIG,
439
- parse: PARSE_SINGLE_LINE_CONFIG)
440
- def test_emit(data)
441
- config = data
442
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
443
- f.puts "test1"
444
- f.puts "test2"
445
- }
446
-
447
- d = create_driver(config)
448
-
449
- d.run(expect_emits: 1) do
450
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
451
- f.puts "test3\ntest4"
452
- }
453
- end
454
-
455
- events = d.events
456
- assert_equal(true, events.length > 0)
457
- assert_equal({"message" => "test3"}, events[0][2])
458
- assert_equal({"message" => "test4"}, events[1][2])
459
- assert(events[0][1].is_a?(Fluent::EventTime))
460
- assert(events[1][1].is_a?(Fluent::EventTime))
461
- assert_equal(1, d.emit_count)
462
- end
463
-
464
- def test_emit_with_emit_unmatched_lines_true
465
- config = config_element("", "", { "format" => "/^(?<message>test.*)/", "emit_unmatched_lines" => true })
466
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
467
-
468
- d = create_driver(config)
469
- d.run(expect_emits: 1) do
470
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
471
- f.puts "test line 1"
472
- f.puts "test line 2"
473
- f.puts "bad line 1"
474
- f.puts "test line 3"
475
- }
476
- end
477
-
478
- events = d.events
479
- assert_equal(4, events.length)
480
- assert_equal({"message" => "test line 1"}, events[0][2])
481
- assert_equal({"message" => "test line 2"}, events[1][2])
482
- assert_equal({"unmatched_line" => "bad line 1"}, events[2][2])
483
- assert_equal({"message" => "test line 3"}, events[3][2])
484
- end
485
-
486
- data('flat 1' => [:flat, 1, 2],
487
- 'flat 10' => [:flat, 10, 1],
488
- 'parse 1' => [:parse, 1, 2],
489
- 'parse 10' => [:parse, 10, 1])
490
- def test_emit_with_read_lines_limit(data)
491
- config_style, limit, num_events = data
492
- case config_style
493
- when :flat
494
- config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit })
495
- when :parse
496
- config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit }) + PARSE_SINGLE_LINE_CONFIG
497
- end
498
- d = create_driver(config)
499
- msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
500
-
501
- d.run(expect_emits: num_events, timeout: 2) do
502
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
503
- f.puts msg
504
- f.puts msg
505
- }
506
- end
507
-
508
- events = d.events
509
- assert_equal(true, events.length > 0)
510
- assert_equal({"message" => msg}, events[0][2])
511
- assert_equal({"message" => msg}, events[1][2])
512
- assert num_events <= d.emit_count
513
- end
514
-
515
- sub_test_case "log throttling per file" do
516
- teardown do
517
- cleanup_file("#{@tmp_dir}/tail.txt")
518
- end
519
-
520
- sub_test_case "reads_bytes_per_second w/o throttled" do
521
- data("flat 8192 bytes, 2 events" => [:flat, 100, 8192, 2],
522
- "flat 8192 bytes, 2 events w/o stat watcher" => [:flat_without_stat, 100, 8192, 2],
523
- "flat #{8192*10} bytes, 20 events" => [:flat, 100, (8192 * 10), 20],
524
- "flat #{8192*10} bytes, 20 events w/o stat watcher" => [:flat_without_stat, 100, (8192 * 10), 20],
525
- "parse #{8192*4} bytes, 8 events" => [:parse, 100, (8192 * 4), 8],
526
- "parse #{8192*4} bytes, 8 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 4), 8],
527
- "parse #{8192*10} bytes, 20 events" => [:parse, 100, (8192 * 10), 20],
528
- "parse #{8192*10} bytes, 20 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 10), 20],
529
- "flat 8k bytes with unit, 2 events" => [:flat, 100, "8k", 2])
530
- def test_emit_with_read_bytes_limit_per_second(data)
531
- config_style, limit, limit_bytes, num_events = data
532
- case config_style
533
- when :flat
534
- config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
535
- when :parse
536
- config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
537
- when :flat_without_stat
538
- config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
539
- when :parse_without_stat
540
- config = CONFIG_READ_FROM_HEAD + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
541
- end
542
-
543
- msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
544
- start_time = Fluent::Clock.now
545
-
546
- d = create_driver(config)
547
- d.run(expect_emits: 2) do
548
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
549
- 100.times do
550
- f.puts msg
551
- end
552
- }
553
- end
554
-
555
- assert_true(Fluent::Clock.now - start_time > 1)
556
- assert_equal(num_events.times.map { {"message" => msg} },
557
- d.events.collect { |event| event[2] })
558
- end
559
-
560
- def test_read_bytes_limit_precede_read_lines_limit
561
- config = CONFIG_READ_FROM_HEAD +
562
- SINGLE_LINE_CONFIG +
563
- config_element("", "", {
564
- "read_lines_limit" => 1000,
565
- "read_bytes_limit_per_second" => 8192
566
- })
567
- msg = 'abc'
568
- start_time = Fluent::Clock.now
569
- d = create_driver(config)
570
- d.run(expect_emits: 2) do
571
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
572
- 8000.times do
573
- f.puts msg
574
- end
575
- }
576
- end
577
-
578
- assert_true(Fluent::Clock.now - start_time > 1)
579
- assert_equal(4096.times.map { {"message" => msg} },
580
- d.events.collect { |event| event[2] })
581
- end
582
- end
583
-
584
- sub_test_case "reads_bytes_per_second w/ throttled already" do
585
- data("flat 8192 bytes" => [:flat, 100, 8192],
586
- "parse 8192 bytes" => [:parse, 100, 8192])
587
- def test_emit_with_read_bytes_limit_per_second(data)
588
- config_style, limit, limit_bytes = data
589
- case config_style
590
- when :flat
591
- config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
592
- when :parse
593
- config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
594
- end
595
- d = create_driver(config)
596
- msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
597
-
598
- mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
599
- require 'fluent/config/types'
600
- limit_bytes_value = Fluent::Config.size_value(limit_bytes)
601
- io_handler.instance_variable_set(:@number_bytes_read, limit_bytes_value)
602
- if Fluent.linux?
603
- mock.proxy(io_handler).handle_notify.at_least(5)
604
- else
605
- mock.proxy(io_handler).handle_notify.twice
606
- end
607
- io_handler
608
- end
609
-
610
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") do |f|
611
- 100.times do
612
- f.puts msg
613
- end
614
- end
615
-
616
- # We should not do shutdown here due to hard timeout.
617
- d.run do
618
- start_time = Fluent::Clock.now
619
- while Fluent::Clock.now - start_time < 0.8 do
620
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") do |f|
621
- f.puts msg
622
- f.flush
623
- end
624
- sleep 0.05
625
- end
626
- end
627
-
628
- assert_equal([], d.events)
629
- end
630
- end
631
-
632
- sub_test_case "EOF with reads_bytes_per_second" do
633
- def test_longer_than_rotate_wait
634
- limit_bytes = 8192
635
- num_lines = 1024 * 3
636
- msg = "08bytes"
637
-
638
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") do |f|
639
- f.write("#{msg}\n" * num_lines)
640
- end
641
-
642
- config = CONFIG_READ_FROM_HEAD +
643
- SINGLE_LINE_CONFIG +
644
- config_element("", "", {
645
- "read_bytes_limit_per_second" => limit_bytes,
646
- "rotate_wait" => 0.1,
647
- "refresh_interval" => 0.5,
648
- })
649
-
650
- rotated = false
651
- d = create_driver(config)
652
- d.run(timeout: 10) do
653
- while d.events.size < num_lines do
654
- if d.events.size > 0 && !rotated
655
- cleanup_file("#{@tmp_dir}/tail.txt")
656
- FileUtils.touch("#{@tmp_dir}/tail.txt")
657
- rotated = true
658
- end
659
- sleep 0.3
660
- end
661
- end
662
-
663
- assert_equal(num_lines,
664
- d.events.count do |event|
665
- event[2]["message"] == msg
666
- end)
667
- end
668
-
669
- def test_shorter_than_rotate_wait
670
- limit_bytes = 8192
671
- num_lines = 1024 * 2
672
- msg = "08bytes"
673
-
674
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") do |f|
675
- f.write("#{msg}\n" * num_lines)
676
- end
677
-
678
- config = CONFIG_READ_FROM_HEAD +
679
- SINGLE_LINE_CONFIG +
680
- config_element("", "", {
681
- "read_bytes_limit_per_second" => limit_bytes,
682
- "rotate_wait" => 2,
683
- "refresh_interval" => 0.5,
684
- })
685
-
686
- start_time = Fluent::Clock.now
687
- rotated = false
688
- detached = false
689
- d = create_driver(config)
690
- mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
691
- mock.proxy(tw).detach(anything) do |v|
692
- detached = true
693
- v
694
- end
695
- tw
696
- end.twice
697
-
698
- d.run(timeout: 10) do
699
- until detached do
700
- if d.events.size > 0 && !rotated
701
- cleanup_file("#{@tmp_dir}/tail.txt")
702
- FileUtils.touch("#{@tmp_dir}/tail.txt")
703
- rotated = true
704
- end
705
- sleep 0.3
706
- end
707
- end
708
-
709
- assert_true(Fluent::Clock.now - start_time > 2)
710
- assert_equal(num_lines,
711
- d.events.count do |event|
712
- event[2]["message"] == msg
713
- end)
714
- end
715
- end
716
- end
717
-
718
- data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
719
- parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
720
- def test_emit_with_read_from_head(data)
721
- config = data
722
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
723
- f.puts "test1"
724
- f.puts "test2"
725
- }
726
-
727
- d = create_driver(config)
728
-
729
- d.run(expect_emits: 2) do
730
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
731
- f.puts "test3"
732
- f.puts "test4"
733
- }
734
- end
735
-
736
- events = d.events
737
- assert(events.length > 0)
738
- assert_equal({"message" => "test1"}, events[0][2])
739
- assert_equal({"message" => "test2"}, events[1][2])
740
- assert_equal({"message" => "test3"}, events[2][2])
741
- assert_equal({"message" => "test4"}, events[3][2])
742
- end
743
-
744
- data(flat: CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG,
745
- parse: CONFIG_DISABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG)
746
- def test_emit_without_watch_timer(data)
747
- config = data
748
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
749
- f.puts "test1"
750
- f.puts "test2"
751
- }
752
-
753
- d = create_driver(config)
754
-
755
- d.run(expect_emits: 1) do
756
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
757
- f.puts "test3"
758
- f.puts "test4"
759
- }
760
- # according to cool.io's stat_watcher.c, systems without inotify will use
761
- # an "automatic" value, typically around 5 seconds
762
- end
763
-
764
- events = d.events
765
- assert(events.length > 0)
766
- assert_equal({"message" => "test3"}, events[0][2])
767
- assert_equal({"message" => "test4"}, events[1][2])
768
- end
769
-
770
- # https://github.com/fluent/fluentd/pull/3541#discussion_r740197711
771
- def test_watch_wildcard_path_without_watch_timer
772
- omit "need inotify" unless Fluent.linux?
773
-
774
- config = config_element("ROOT", "", {
775
- "path" => "#{@tmp_dir}/tail*.txt",
776
- "tag" => "t1",
777
- })
778
- config = config + CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG
779
-
780
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
781
- f.puts "test1"
782
- f.puts "test2"
783
- }
784
-
785
- d = create_driver(config, false)
786
-
787
- d.run(expect_emits: 1, timeout: 1) do
788
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
789
- f.puts "test3"
790
- f.puts "test4"
791
- }
792
- end
793
-
794
- assert_equal(
795
- [
796
- {"message" => "test3"},
797
- {"message" => "test4"},
798
- ],
799
- d.events.collect { |event| event[2] })
800
- end
801
-
802
- data(flat: CONFIG_DISABLE_STAT_WATCHER + SINGLE_LINE_CONFIG,
803
- parse: CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
804
- def test_emit_with_disable_stat_watcher(data)
805
- config = data
806
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
807
- f.puts "test1"
808
- f.puts "test2"
809
- }
810
-
811
- d = create_driver(config)
812
-
813
- d.run(expect_emits: 1) do
814
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
815
- f.puts "test3"
816
- f.puts "test4"
817
- }
818
- end
819
-
820
- events = d.events
821
- assert(events.length > 0)
822
- assert_equal({"message" => "test3"}, events[0][2])
823
- assert_equal({"message" => "test4"}, events[1][2])
824
- end
825
-
826
- def test_always_read_from_head_on_detecting_a_new_file
827
- d = create_driver(SINGLE_LINE_CONFIG)
828
-
829
- d.run(expect_emits: 1, timeout: 3) do
830
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
831
- f.puts "test1\ntest2\n"
832
- }
833
- end
834
-
835
- assert_equal(
836
- [
837
- {"message" => "test1"},
838
- {"message" => "test2"},
839
- ],
840
- d.events.collect { |event| event[2] })
841
- end
842
- end
843
-
844
- class TestWithSystem < self
845
- include Fluent::SystemConfig::Mixin
846
-
847
- OVERRIDE_FILE_PERMISSION = 0620
848
- CONFIG_SYSTEM = %[
849
- <system>
850
- file_permission #{OVERRIDE_FILE_PERMISSION}
851
- </system>
852
- ]
853
-
854
- def setup
855
- omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
856
- super
857
- # Store default permission
858
- @default_permission = system_config.instance_variable_get(:@file_permission)
859
- end
860
-
861
- def teardown
862
- return if Fluent.windows?
863
- super
864
- # Restore default permission
865
- system_config.instance_variable_set(:@file_permission, @default_permission)
866
- end
867
-
868
- def parse_system(text)
869
- basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
870
- Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
871
- end
872
-
873
- def test_emit_with_system
874
- system_conf = parse_system(CONFIG_SYSTEM)
875
- sc = Fluent::SystemConfig.new(system_conf)
876
- Fluent::Engine.init(sc)
877
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
878
- f.puts "test1"
879
- f.puts "test2"
880
- }
881
-
882
- d = create_driver
883
-
884
- d.run(expect_emits: 1) do
885
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
886
- f.puts "test3"
887
- f.puts "test4"
888
- }
889
- end
890
-
891
- events = d.events
892
- assert_equal(true, events.length > 0)
893
- assert_equal({"message" => "test3"}, events[0][2])
894
- assert_equal({"message" => "test4"}, events[1][2])
895
- assert(events[0][1].is_a?(Fluent::EventTime))
896
- assert(events[1][1].is_a?(Fluent::EventTime))
897
- assert_equal(1, d.emit_count)
898
- pos = d.instance.instance_variable_get(:@pf_file)
899
- mode = "%o" % File.stat(pos).mode
900
- assert_equal OVERRIDE_FILE_PERMISSION, mode[-3, 3].to_i
901
- end
902
- end
903
-
904
- sub_test_case "rotate file" do
905
- def create_driver(conf = SINGLE_LINE_CONFIG)
906
- config = common_config + conf
907
- Fluent::Test::Driver::Input.new(Fluent::Plugin::TailInput).configure(config)
908
- end
909
-
910
- data(flat: SINGLE_LINE_CONFIG,
911
- parse: PARSE_SINGLE_LINE_CONFIG)
912
- def test_rotate_file(data)
913
- config = data
914
- events = sub_test_rotate_file(config, expect_emits: 2)
915
- assert_equal(3.upto(6).collect { |i| {"message" => "test#{i}"} },
916
- events.collect { |event| event[2] })
917
- end
918
-
919
- data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
920
- parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
921
- def test_rotate_file_with_read_from_head(data)
922
- config = data
923
- events = sub_test_rotate_file(config, expect_records: 6)
924
- assert_equal(1.upto(6).collect { |i| {"message" => "test#{i}"} },
925
- events.collect { |event| event[2] })
926
- end
927
-
928
- data(flat: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
929
- parse: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
930
- def test_rotate_file_with_open_on_every_update(data)
931
- config = data
932
- events = sub_test_rotate_file(config, expect_records: 6)
933
- assert_equal(1.upto(6).collect { |i| {"message" => "test#{i}"} },
934
- events.collect { |event| event[2] })
935
- end
936
-
937
- data(flat: SINGLE_LINE_CONFIG,
938
- parse: PARSE_SINGLE_LINE_CONFIG)
939
- def test_rotate_file_with_write_old(data)
940
- config = data
941
- events = sub_test_rotate_file(config, expect_emits: 3) { |rotated_file|
942
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
943
- rotated_file.puts "test7"
944
- rotated_file.puts "test8"
945
- rotated_file.flush
946
-
947
- sleep 1
948
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
949
- f.puts "test5"
950
- f.puts "test6"
951
- }
952
- }
953
- # This test sometimes fails and it shows a potential bug of in_tail
954
- # https://github.com/fluent/fluentd/issues/1434
955
- assert_equal([3, 4, 7, 8, 5, 6].collect { |i| {"message" => "test#{i}"} },
956
- events.collect { |event| event[2] })
957
- end
958
-
959
- data(flat: SINGLE_LINE_CONFIG,
960
- parse: PARSE_SINGLE_LINE_CONFIG)
961
- def test_rotate_file_with_write_old_and_no_new_file(data)
962
- config = data
963
- events = sub_test_rotate_file(config, expect_emits: 2) { |rotated_file|
964
- rotated_file.puts "test7"
965
- rotated_file.puts "test8"
966
- rotated_file.flush
967
- }
968
- assert_equal([3, 4, 7, 8].collect { |i| {"message" => "test#{i}"} },
969
- events.collect { |event| event[2] })
970
- end
971
-
972
- def sub_test_rotate_file(config = nil, expect_emits: nil, expect_records: nil, timeout: 5)
973
- file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb")
974
- file.puts "test1"
975
- file.puts "test2"
976
- file.flush
977
-
978
- d = create_driver(config)
979
- d.run(expect_emits: expect_emits, expect_records: expect_records, timeout: timeout) do
980
- sleep(0.1) while d.instance.instance_variable_get(:@startup)
981
- size = d.emit_count
982
- file.puts "test3"
983
- file.puts "test4"
984
- file.flush
985
- sleep(0.1) until d.emit_count >= size + 1
986
- size = d.emit_count
987
-
988
- if Fluent.windows?
989
- file.close
990
- FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt", force: true)
991
- file = Fluent::FileWrapper.open("#{@tmp_dir}/tail2.txt", "ab")
992
- else
993
- FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt")
994
- end
995
- if block_given?
996
- yield file
997
- else
998
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
999
- sleep 1
1000
-
1001
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1002
- f.puts "test5"
1003
- f.puts "test6"
1004
- }
1005
- end
1006
- end
1007
-
1008
- d.events
1009
- ensure
1010
- file.close if file && !file.closed?
1011
- end
1012
- end
1013
-
1014
- def test_truncate_file
1015
- config = SINGLE_LINE_CONFIG
1016
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1017
- f.puts "test1"
1018
- f.puts "test2"
1019
- f.flush
1020
- }
1021
-
1022
- d = create_driver(config)
1023
-
1024
- d.run(expect_emits: 2) do
1025
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1026
- f.puts "test3\ntest4"
1027
- f.flush
1028
- }
1029
- waiting(2) { sleep 0.1 until d.events.length == 2 }
1030
- if Fluent.windows?
1031
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f|
1032
- f.puts("test1");
1033
- }
1034
- else
1035
- File.truncate("#{@tmp_dir}/tail.txt", 6)
1036
- end
1037
- end
1038
-
1039
- expected = {
1040
- emit_count: 2,
1041
- events: [
1042
- [Fluent::EventTime, {"message" => "test3"}],
1043
- [Fluent::EventTime, {"message" => "test4"}],
1044
- [Fluent::EventTime, {"message" => "test1"}],
1045
- ]
1046
- }
1047
- actual = {
1048
- emit_count: d.emit_count,
1049
- events: d.events.collect{|event| [event[1].class, event[2]]}
1050
- }
1051
- assert_equal(expected, actual)
1052
- end
1053
-
1054
- def test_move_truncate_move_back
1055
- config = SINGLE_LINE_CONFIG
1056
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1057
- f.puts "test1"
1058
- f.puts "test2"
1059
- }
1060
-
1061
- d = create_driver(config)
1062
-
1063
- d.run(expect_emits: 1) do
1064
- if Fluent.windows?
1065
- FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt", force: true)
1066
- else
1067
- FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt")
1068
- end
1069
- sleep(1)
1070
- if Fluent.windows?
1071
- Fluent::FileWrapper.open("#{@tmp_dir}/tail2.txt", "wb") { |f|
1072
- f.puts("test1");
1073
- }
1074
- else
1075
- File.truncate("#{@tmp_dir}/tail2.txt", 6)
1076
- end
1077
- sleep(1)
1078
- if Fluent.windows?
1079
- FileUtils.mv("#{@tmp_dir}/tail2.txt", "#{@tmp_dir}/tail.txt", force: true)
1080
- else
1081
- FileUtils.mv("#{@tmp_dir}/tail2.txt", "#{@tmp_dir}/tail.txt")
1082
- end
1083
- end
1084
-
1085
- events = d.events
1086
- assert_equal(1, events.length)
1087
- assert_equal({"message" => "test1"}, events[0][2])
1088
- assert(events[0][1].is_a?(Fluent::EventTime))
1089
- assert_equal(1, d.emit_count)
1090
- end
1091
-
1092
- def test_lf
1093
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| }
1094
-
1095
- d = create_driver
1096
-
1097
- d.run(expect_emits: 1) do
1098
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1099
- f.print "test3"
1100
- }
1101
- sleep 1
1102
-
1103
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1104
- f.puts "test4"
1105
- }
1106
- end
1107
-
1108
- events = d.events
1109
- assert_equal(true, events.length > 0)
1110
- assert_equal({"message" => "test3test4"}, events[0][2])
1111
- end
1112
-
1113
- def test_whitespace
1114
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| }
1115
-
1116
- d = create_driver
1117
-
1118
- d.run(expect_emits: 1) do
1119
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1120
- f.puts " " # 4 spaces
1121
- f.puts " 4 spaces"
1122
- f.puts "4 spaces "
1123
- f.puts " " # tab
1124
- f.puts " tab"
1125
- f.puts "tab "
1126
- }
1127
- end
1128
-
1129
- events = d.events
1130
- assert_equal(true, events.length > 0)
1131
- assert_equal({"message" => " "}, events[0][2])
1132
- assert_equal({"message" => " 4 spaces"}, events[1][2])
1133
- assert_equal({"message" => "4 spaces "}, events[2][2])
1134
- assert_equal({"message" => " "}, events[3][2])
1135
- assert_equal({"message" => " tab"}, events[4][2])
1136
- assert_equal({"message" => "tab "}, events[5][2])
1137
- end
1138
-
1139
- data(
1140
- 'flat default encoding' => [SINGLE_LINE_CONFIG, Encoding::ASCII_8BIT],
1141
- 'flat explicit encoding config' => [SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8],
1142
- 'parse default encoding' => [PARSE_SINGLE_LINE_CONFIG, Encoding::ASCII_8BIT],
1143
- 'parse explicit encoding config' => [PARSE_SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8])
1144
- def test_encoding(data)
1145
- encoding_config, encoding = data
1146
-
1147
- d = create_driver(CONFIG_READ_FROM_HEAD + encoding_config)
1148
-
1149
- d.run(expect_emits: 1) do
1150
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1151
- f.puts "test"
1152
- }
1153
- end
1154
-
1155
- events = d.events
1156
- assert_equal(encoding, events[0][2]['message'].encoding)
1157
- end
1158
-
1159
- def test_from_encoding
1160
- conf = config_element(
1161
- "", "", {
1162
- "format" => "/(?<message>.*)/",
1163
- "read_from_head" => "true",
1164
- "from_encoding" => "cp932",
1165
- "encoding" => "utf-8"
1166
- })
1167
- d = create_driver(conf)
1168
- cp932_message = "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
1169
- utf8_message = cp932_message.encode(Encoding::UTF_8)
1170
-
1171
- d.run(expect_emits: 1) do
1172
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w:cp932") {|f|
1173
- f.puts cp932_message
1174
- }
1175
- end
1176
-
1177
- events = d.events
1178
- assert_equal(utf8_message, events[0][2]['message'])
1179
- assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
1180
- end
1181
-
1182
- def test_from_encoding_utf16
1183
- conf = config_element(
1184
- "", "", {
1185
- "format" => "/(?<message>.*)/",
1186
- "read_from_head" => "true",
1187
- "from_encoding" => "utf-16le",
1188
- "encoding" => "utf-8"
1189
- })
1190
- d = create_driver(conf)
1191
- utf16_message = "\u306F\u308D\u30FC\u308F\u30FC\u308B\u3069\n".encode(Encoding::UTF_16LE)
1192
- utf8_message = utf16_message.encode(Encoding::UTF_8).strip
1193
-
1194
- d.run(expect_emits: 1) do
1195
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w:utf-16le") { |f|
1196
- f.write utf16_message
1197
- }
1198
- end
1199
-
1200
- events = d.events
1201
- assert_equal(utf8_message, events[0][2]['message'])
1202
- assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
1203
- end
1204
-
1205
- def test_encoding_with_bad_character
1206
- conf = config_element(
1207
- "", "", {
1208
- "format" => "/(?<message>.*)/",
1209
- "read_from_head" => "true",
1210
- "from_encoding" => "ASCII-8BIT",
1211
- "encoding" => "utf-8"
1212
- })
1213
- d = create_driver(conf)
1214
-
1215
- d.run(expect_emits: 1) do
1216
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w") { |f|
1217
- f.write "te\x86st\n"
1218
- }
1219
- end
1220
-
1221
- events = d.events
1222
- assert_equal("te\uFFFDst", events[0][2]['message'])
1223
- assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
1224
- end
1225
-
1226
- sub_test_case "multiline" do
1227
- data(flat: MULTILINE_CONFIG,
1228
- parse: PARSE_MULTILINE_CONFIG)
1229
- def test_multiline(data)
1230
- config = data
1231
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1232
-
1233
- d = create_driver(config)
1234
- d.run(expect_emits: 1) do
1235
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1236
- f.puts "f test1"
1237
- f.puts "s test2"
1238
- f.puts "f test3"
1239
- f.puts "f test4"
1240
- f.puts "s test5"
1241
- f.puts "s test6"
1242
- f.puts "f test7"
1243
- f.puts "s test8"
1244
- }
1245
- end
1246
-
1247
- events = d.events
1248
- assert_equal(4, events.length)
1249
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
1250
- assert_equal({"message1" => "test5"}, events[1][2])
1251
- assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
1252
- assert_equal({"message1" => "test8"}, events[3][2])
1253
- end
1254
-
1255
- data(flat: MULTILINE_CONFIG,
1256
- parse: PARSE_MULTILINE_CONFIG)
1257
- def test_multiline_with_emit_unmatched_lines_true(data)
1258
- config = data + config_element("", "", { "emit_unmatched_lines" => true })
1259
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1260
-
1261
- d = create_driver(config)
1262
- d.run(expect_emits: 1) do
1263
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1264
- f.puts "f test1"
1265
- f.puts "s test2"
1266
- f.puts "f test3"
1267
- f.puts "f test4"
1268
- f.puts "s test5"
1269
- f.puts "s test6"
1270
- f.puts "f test7"
1271
- f.puts "s test8"
1272
- }
1273
- end
1274
-
1275
- events = d.events
1276
- assert_equal(5, events.length)
1277
- assert_equal({"unmatched_line" => "f test1"}, events[0][2])
1278
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[1][2])
1279
- assert_equal({"message1" => "test5"}, events[2][2])
1280
- assert_equal({"message1" => "test6", "message2" => "test7"}, events[3][2])
1281
- assert_equal({"message1" => "test8"}, events[4][2])
1282
- end
1283
-
1284
- data(
1285
- flat: MULTILINE_CONFIG_WITH_NEWLINE,
1286
- parse: PARSE_MULTILINE_CONFIG_WITH_NEWLINE)
1287
- def test_multiline_with_emit_unmatched_lines2(data)
1288
- config = data + config_element("", "", { "emit_unmatched_lines" => true })
1289
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1290
-
1291
- d = create_driver(config)
1292
- d.run(expect_emits: 0, timeout: 1) do
1293
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1294
- f.puts "s test0"
1295
- f.puts "f test1"
1296
- f.puts "f test2"
1297
-
1298
- f.puts "f test3"
1299
-
1300
- f.puts "s test4"
1301
- f.puts "f test5"
1302
- f.puts "f test6"
1303
- }
1304
- end
1305
-
1306
- events = d.events
1307
- assert_equal({"message1" => "test0", "message2" => "test1", "message3" => "test2"}, events[0][2])
1308
- assert_equal({ 'unmatched_line' => "f test3" }, events[1][2])
1309
- assert_equal({"message1" => "test4", "message2" => "test5", "message3" => "test6"}, events[2][2])
1310
- end
1311
-
1312
- data(flat: MULTILINE_CONFIG,
1313
- parse: PARSE_MULTILINE_CONFIG)
1314
- def test_multiline_with_flush_interval(data)
1315
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1316
-
1317
- config = data + config_element("", "", { "multiline_flush_interval" => "2s" })
1318
- d = create_driver(config)
1319
-
1320
- assert_equal(2, d.instance.multiline_flush_interval)
1321
-
1322
- d.run(expect_emits: 1) do
1323
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1324
- f.puts "f test1"
1325
- f.puts "s test2"
1326
- f.puts "f test3"
1327
- f.puts "f test4"
1328
- f.puts "s test5"
1329
- f.puts "s test6"
1330
- f.puts "f test7"
1331
- f.puts "s test8"
1332
- }
1333
- end
1334
-
1335
- events = d.events
1336
- assert_equal(4, events.length)
1337
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
1338
- assert_equal({"message1" => "test5"}, events[1][2])
1339
- assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
1340
- assert_equal({"message1" => "test8"}, events[3][2])
1341
- end
1342
-
1343
- data(
1344
- 'flat default encoding' => [MULTILINE_CONFIG, Encoding::ASCII_8BIT],
1345
- 'flat explicit encoding config' => [MULTILINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8],
1346
- 'parse default encoding' => [PARSE_MULTILINE_CONFIG, Encoding::ASCII_8BIT],
1347
- 'parse explicit encoding config' => [PARSE_MULTILINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8])
1348
- def test_multiline_encoding_of_flushed_record(data)
1349
- encoding_config, encoding = data
1350
-
1351
- config = config_element("", "", {
1352
- "multiline_flush_interval" => "2s",
1353
- "read_from_head" => "true",
1354
- })
1355
- d = create_driver(config + encoding_config)
1356
-
1357
- d.run(expect_emits: 1) do
1358
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f|
1359
- f.puts "s test"
1360
- }
1361
- end
1362
- events = d.events
1363
- assert_equal(1, events.length)
1364
- assert_equal(encoding, events[0][2]['message1'].encoding)
1365
- end
1366
-
1367
- def test_multiline_from_encoding_of_flushed_record
1368
- conf = MULTILINE_CONFIG + config_element(
1369
- "", "",
1370
- {
1371
- "multiline_flush_interval" => "1s",
1372
- "read_from_head" => "true",
1373
- "from_encoding" => "cp932",
1374
- "encoding" => "utf-8"
1375
- })
1376
- d = create_driver(conf)
1377
-
1378
- cp932_message = "s \x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
1379
- utf8_message = "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".encode(Encoding::UTF_8, Encoding::CP932)
1380
- d.run(expect_emits: 1) do
1381
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w:cp932") { |f|
1382
- f.puts cp932_message
1383
- }
1384
- end
1385
-
1386
- events = d.events
1387
- assert_equal(1, events.length)
1388
- assert_equal(utf8_message, events[0][2]['message1'])
1389
- assert_equal(Encoding::UTF_8, events[0][2]['message1'].encoding)
1390
- end
1391
-
1392
- data(flat: config_element(
1393
- "", "", {
1394
- "format" => "multiline",
1395
- "format1" => "/^s (?<message1>[^\\n]+)\\n?/",
1396
- "format2" => "/(f (?<message2>[^\\n]+)\\n?)?/",
1397
- "format3" => "/(f (?<message3>.*))?/",
1398
- "format_firstline" => "/^[s]/"
1399
- }),
1400
- parse: config_element(
1401
- "", "", {},
1402
- [config_element("parse", "", {
1403
- "@type" => "multiline",
1404
- "format1" => "/^s (?<message1>[^\\n]+)\\n?/",
1405
- "format2" => "/(f (?<message2>[^\\n]+)\\n?)?/",
1406
- "format3" => "/(f (?<message3>.*))?/",
1407
- "format_firstline" => "/^[s]/"
1408
- })
1409
- ])
1410
- )
1411
- def test_multiline_with_multiple_formats(data)
1412
- config = data
1413
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1414
-
1415
- d = create_driver(config)
1416
- d.run(expect_emits: 1) do
1417
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1418
- f.puts "f test1"
1419
- f.puts "s test2"
1420
- f.puts "f test3"
1421
- f.puts "f test4"
1422
- f.puts "s test5"
1423
- f.puts "s test6"
1424
- f.puts "f test7"
1425
- f.puts "s test8"
1426
- }
1427
- end
1428
-
1429
- events = d.events
1430
- assert(events.length > 0)
1431
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
1432
- assert_equal({"message1" => "test5"}, events[1][2])
1433
- assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
1434
- assert_equal({"message1" => "test8"}, events[3][2])
1435
- end
1436
-
1437
- data(flat: config_element(
1438
- "", "", {
1439
- "format" => "multiline",
1440
- "format1" => "/^[s|f] (?<message>.*)/",
1441
- "format_firstline" => "/^[s]/"
1442
- }),
1443
- parse: config_element(
1444
- "", "", {},
1445
- [config_element("parse", "", {
1446
- "@type" => "multiline",
1447
- "format1" => "/^[s|f] (?<message>.*)/",
1448
- "format_firstline" => "/^[s]/"
1449
- })
1450
- ])
1451
- )
1452
- def test_multilinelog_with_multiple_paths(data)
1453
- files = ["#{@tmp_dir}/tail1.txt", "#{@tmp_dir}/tail2.txt"]
1454
- files.each { |file| Fluent::FileWrapper.open(file, "wb") { |f| } }
1455
-
1456
- config = data + config_element("", "", {
1457
- "path" => "#{files[0]},#{files[1]}",
1458
- "tag" => "t1",
1459
- })
1460
- d = create_driver(config, false)
1461
- d.run(expect_emits: 2) do
1462
- files.each do |file|
1463
- Fluent::FileWrapper.open(file, 'ab') { |f|
1464
- f.puts "f #{file} line should be ignored"
1465
- f.puts "s test1"
1466
- f.puts "f test2"
1467
- f.puts "f test3"
1468
- f.puts "s test4"
1469
- }
1470
- end
1471
- end
1472
-
1473
- events = d.events
1474
- assert_equal({"message" => "test1\nf test2\nf test3"}, events[0][2])
1475
- assert_equal({"message" => "test1\nf test2\nf test3"}, events[1][2])
1476
- # "test4" events are here because these events are flushed at shutdown phase
1477
- assert_equal({"message" => "test4"}, events[2][2])
1478
- assert_equal({"message" => "test4"}, events[3][2])
1479
- end
1480
-
1481
- data(flat: config_element("", "", {
1482
- "format" => "multiline",
1483
- "format1" => "/(?<var1>foo \\d)\\n/",
1484
- "format2" => "/(?<var2>bar \\d)\\n/",
1485
- "format3" => "/(?<var3>baz \\d)/"
1486
- }),
1487
- parse: config_element(
1488
- "", "", {},
1489
- [config_element("parse", "", {
1490
- "@type" => "multiline",
1491
- "format1" => "/(?<var1>foo \\d)\\n/",
1492
- "format2" => "/(?<var2>bar \\d)\\n/",
1493
- "format3" => "/(?<var3>baz \\d)/"
1494
- })
1495
- ])
1496
- )
1497
- def test_multiline_without_firstline(data)
1498
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1499
-
1500
- config = data
1501
- d = create_driver(config)
1502
- d.run(expect_emits: 1) do
1503
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1504
- f.puts "foo 1"
1505
- f.puts "bar 1"
1506
- f.puts "baz 1"
1507
- f.puts "foo 2"
1508
- f.puts "bar 2"
1509
- f.puts "baz 2"
1510
- }
1511
- end
1512
-
1513
- events = d.events
1514
- assert_equal(2, events.length)
1515
- assert_equal({"var1" => "foo 1", "var2" => "bar 1", "var3" => "baz 1"}, events[0][2])
1516
- assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, events[1][2])
1517
- end
1518
- end
1519
-
1520
- sub_test_case "path" do
1521
- # * path test
1522
- # TODO: Clean up tests
1523
- def test_expand_paths
1524
- ex_paths = [
1525
- create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1526
- create_target_info('test/plugin/data/log/foo/bar.log'),
1527
- create_target_info('test/plugin/data/log/test.log')
1528
- ]
1529
- plugin = create_driver(ex_config, false).instance
1530
- flexstub(Time) do |timeclass|
1531
- timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
1532
- assert_equal(ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1533
- end
1534
-
1535
- # Test exclusion
1536
- exclude_config = ex_config + config_element("", "", { "exclude_path" => %Q(["#{ex_paths.last.path}"]) })
1537
- plugin = create_driver(exclude_config, false).instance
1538
- assert_equal(ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1539
- end
1540
-
1541
- def test_expand_paths_with_duplicate_configuration
1542
- expanded_paths = [
1543
- create_target_info('test/plugin/data/log/foo/bar.log'),
1544
- create_target_info('test/plugin/data/log/test.log')
1545
- ]
1546
- duplicate_config = ex_config.dup
1547
- duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
1548
- plugin = create_driver(ex_config, false).instance
1549
- assert_equal(expanded_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1550
- end
1551
-
1552
- def test_expand_paths_with_timezone
1553
- ex_paths = [
1554
- create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1555
- create_target_info('test/plugin/data/log/foo/bar.log'),
1556
- create_target_info('test/plugin/data/log/test.log')
1557
- ]
1558
- ['Asia/Taipei', '+08'].each do |tz_type|
1559
- taipei_config = ex_config + config_element("", "", {"path_timezone" => tz_type})
1560
- plugin = create_driver(taipei_config, false).instance
1561
-
1562
- # Test exclude
1563
- exclude_config = taipei_config + config_element("", "", { "exclude_path" => %Q(["test/plugin/**/%Y%m%d-%H%M%S.log"]) })
1564
- exclude_plugin = create_driver(exclude_config, false).instance
1565
-
1566
- with_timezone('utc') do
1567
- flexstub(Time) do |timeclass|
1568
- # env : 2010-01-01 19:04:05 (UTC), tail path : 2010-01-02 03:04:05 (Asia/Taipei)
1569
- timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 1, 19, 4, 5))
1570
-
1571
- assert_equal(ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1572
- assert_equal(ex_paths - [ex_paths.first], exclude_plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1573
- end
1574
- end
1575
- end
1576
- end
1577
-
1578
- def test_log_file_without_extension
1579
- expected_files = [
1580
- create_target_info('test/plugin/data/log/bar'),
1581
- create_target_info('test/plugin/data/log/foo/bar.log'),
1582
- create_target_info('test/plugin/data/log/foo/bar2'),
1583
- create_target_info('test/plugin/data/log/test.log')
1584
- ]
1585
-
1586
- config = config_element("", "", {
1587
- "tag" => "tail",
1588
- "path" => "test/plugin/data/log/**/*",
1589
- "format" => "none",
1590
- "pos_file" => "#{@tmp_dir}/tail.pos"
1591
- })
1592
-
1593
- plugin = create_driver(config, false).instance
1594
- assert_equal(expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1595
- end
1596
-
1597
- def test_unwatched_files_should_be_removed
1598
- config = config_element("", "", {
1599
- "tag" => "tail",
1600
- "path" => "#{@tmp_dir}/*.txt",
1601
- "format" => "none",
1602
- "pos_file" => "#{@tmp_dir}/tail.pos",
1603
- "read_from_head" => true,
1604
- "refresh_interval" => 1,
1605
- })
1606
- d = create_driver(config, false)
1607
- d.end_if { d.instance.instance_variable_get(:@tails).keys.size >= 1 }
1608
- d.run(expect_emits: 1, shutdown: false) do
1609
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f| f.puts "test3\n" }
1610
- end
1611
-
1612
- cleanup_file("#{@tmp_dir}/tail.txt")
1613
- waiting(20) { sleep 0.1 until Dir.glob("#{@tmp_dir}/*.txt").size == 0 } # Ensure file is deleted on Windows
1614
- waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size <= 0 }
1615
-
1616
- assert_equal(
1617
- {
1618
- files: [],
1619
- tails: []
1620
- },
1621
- {
1622
- files: Dir.glob("#{@tmp_dir}/*.txt"),
1623
- tails: d.instance.instance_variable_get(:@tails).keys
1624
- }
1625
- )
1626
- ensure
1627
- d.instance_shutdown if d && d.instance
1628
- cleanup_directory(@tmp_dir)
1629
- end
1630
-
1631
- def count_timer_object
1632
- num = 0
1633
- ObjectSpace.each_object(Fluent::PluginHelper::Timer::TimerWatcher) { |obj|
1634
- num += 1
1635
- }
1636
- num
1637
- end
1638
- end
1639
-
1640
- sub_test_case "path w/ Linux capability" do
1641
- def capability_enabled?
1642
- if Fluent.linux?
1643
- begin
1644
- require 'capng'
1645
- true
1646
- rescue LoadError
1647
- false
1648
- end
1649
- else
1650
- false
1651
- end
1652
- end
1653
-
1654
- setup do
1655
- omit "This environment is not enabled Linux capability handling feature" unless capability_enabled?
1656
-
1657
- @capng = CapNG.new(:current_process)
1658
- flexstub(Fluent::Capability) do |klass|
1659
- klass.should_receive(:new).with(:current_process).and_return(@capng)
1660
- end
1661
- end
1662
-
1663
- data("dac_read_search" => [:dac_read_search, true, 1],
1664
- "dac_override" => [:dac_override, true, 1],
1665
- "chown" => [:chown, false, 0],
1666
- )
1667
- test "with partially elevated privileges" do |data|
1668
- cap, result, readable_paths = data
1669
- @capng.update(:add, :effective, cap)
1670
-
1671
- d = create_driver(
1672
- config_element("ROOT", "", {
1673
- "path" => "/var/log/ker*.log", # Use /var/log/kern.log
1674
- "tag" => "t1",
1675
- "rotate_wait" => "2s"
1676
- }) + PARSE_SINGLE_LINE_CONFIG, false)
1677
-
1678
- assert_equal(readable_paths, d.instance.expand_paths.length)
1679
- assert_equal(result, d.instance.have_read_capability?)
1680
- end
1681
- end
1682
-
1683
- def test_pos_file_dir_creation
1684
- config = config_element("", "", {
1685
- "tag" => "tail",
1686
- "path" => "#{@tmp_dir}/*.txt",
1687
- "format" => "none",
1688
- "pos_file" => "#{@tmp_dir}/pos/tail.pos",
1689
- "read_from_head" => true,
1690
- "refresh_interval" => 1
1691
- })
1692
-
1693
- assert_path_not_exist("#{@tmp_dir}/pos")
1694
- d = create_driver(config, false)
1695
- d.run
1696
- assert_path_exist("#{@tmp_dir}/pos")
1697
- assert_equal('755', File.stat("#{@tmp_dir}/pos").mode.to_s(8)[-3, 3])
1698
- ensure
1699
- cleanup_directory(@tmp_dir)
1700
- end
1701
-
1702
- def test_pos_file_dir_creation_with_system_dir_permission
1703
- config = config_element("", "", {
1704
- "tag" => "tail",
1705
- "path" => "#{@tmp_dir}/*.txt",
1706
- "format" => "none",
1707
- "pos_file" => "#{@tmp_dir}/pos/tail.pos",
1708
- "read_from_head" => true,
1709
- "refresh_interval" => 1
1710
- })
1711
-
1712
- assert_path_not_exist("#{@tmp_dir}/pos")
1713
-
1714
- Fluent::SystemConfig.overwrite_system_config({ "dir_permission" => "744" }) do
1715
- d = create_driver(config, false)
1716
- d.run
1717
- end
1718
-
1719
- assert_path_exist("#{@tmp_dir}/pos")
1720
- if Fluent.windows?
1721
- assert_equal('755', File.stat("#{@tmp_dir}/pos").mode.to_s(8)[-3, 3])
1722
- else
1723
- assert_equal('744', File.stat("#{@tmp_dir}/pos").mode.to_s(8)[-3, 3])
1724
- end
1725
- ensure
1726
- cleanup_directory(@tmp_dir)
1727
- end
1728
-
1729
- def test_z_refresh_watchers
1730
- ex_paths = [
1731
- create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1732
- create_target_info('test/plugin/data/log/foo/bar.log'),
1733
- create_target_info('test/plugin/data/log/test.log'),
1734
- ]
1735
- plugin = create_driver(ex_config, false).instance
1736
- sio = StringIO.new
1737
- plugin.instance_eval do
1738
- @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
1739
- @loop = Coolio::Loop.new
1740
- opened_file_metrics = Fluent::Plugin::LocalMetrics.new
1741
- opened_file_metrics.configure(config_element('metrics', '', {}))
1742
- closed_file_metrics = Fluent::Plugin::LocalMetrics.new
1743
- closed_file_metrics.configure(config_element('metrics', '', {}))
1744
- rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
1745
- rotated_file_metrics.configure(config_element('metrics', '', {}))
1746
- @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
1747
- end
1748
-
1749
- Timecop.freeze(2010, 1, 2, 3, 4, 5) do
1750
- ex_paths.each do |target_info|
1751
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
1752
- end
1753
-
1754
- plugin.refresh_watchers
1755
- end
1756
-
1757
- path = 'test/plugin/data/2010/01/20100102-030405.log'
1758
- mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)[path], Fluent::FileWrapper.stat(path).ino)
1759
-
1760
- Timecop.freeze(2010, 1, 2, 3, 4, 6) do
1761
- path = "test/plugin/data/2010/01/20100102-030406.log"
1762
- inode = Fluent::FileWrapper.stat(path).ino
1763
- target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
1764
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
1765
- plugin.refresh_watchers
1766
-
1767
- flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
1768
- watcherclass.should_receive(:new).never
1769
- plugin.refresh_watchers
1770
- end
1771
- end
1772
- end
1773
-
1774
- sub_test_case "refresh of pos file" do
1775
- test 'type of pos_file_compaction_interval is time' do
1776
- tail = {
1777
- "tag" => "tail",
1778
- "path" => "#{@tmp_dir}/*.txt",
1779
- "format" => "none",
1780
- "pos_file" => "#{@tmp_dir}/pos/tail.pos",
1781
- "refresh_interval" => 1,
1782
- "read_from_head" => true,
1783
- 'pos_file_compaction_interval' => '24h',
1784
- }
1785
- config = config_element("", "", tail)
1786
- d = create_driver(config, false)
1787
- mock(d.instance).timer_execute(:in_tail_refresh_watchers, 1.0).once
1788
- mock(d.instance).timer_execute(:in_tail_refresh_compact_pos_file, 60 * 60 * 24).once
1789
- d.run # call start
1790
- end
1791
- end
1792
-
1793
- sub_test_case "receive_lines" do
1794
- DummyWatcher = Struct.new("DummyWatcher", :tag)
1795
-
1796
- def test_tag
1797
- d = create_driver(ex_config, false)
1798
- d.run {}
1799
- plugin = d.instance
1800
- mock(plugin.router).emit_stream('tail', anything).once
1801
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1802
- end
1803
-
1804
- def test_tag_with_only_star
1805
- config = config_element("", "", {
1806
- "tag" => "*",
1807
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1808
- "format" => "none",
1809
- "read_from_head" => true
1810
- })
1811
- d = create_driver(config, false)
1812
- d.run {}
1813
- plugin = d.instance
1814
- mock(plugin.router).emit_stream('foo.bar.log', anything).once
1815
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1816
- end
1817
-
1818
- def test_tag_prefix
1819
- config = config_element("", "", {
1820
- "tag" => "pre.*",
1821
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1822
- "format" => "none",
1823
- "read_from_head" => true
1824
- })
1825
- d = create_driver(config, false)
1826
- d.run {}
1827
- plugin = d.instance
1828
- mock(plugin.router).emit_stream('pre.foo.bar.log', anything).once
1829
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1830
- end
1831
-
1832
- def test_tag_suffix
1833
- config = config_element("", "", {
1834
- "tag" => "*.post",
1835
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1836
- "format" => "none",
1837
- "read_from_head" => true
1838
- })
1839
- d = create_driver(config, false)
1840
- d.run {}
1841
- plugin = d.instance
1842
- mock(plugin.router).emit_stream('foo.bar.log.post', anything).once
1843
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1844
- end
1845
-
1846
- def test_tag_prefix_and_suffix
1847
- config = config_element("", "", {
1848
- "tag" => "pre.*.post",
1849
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1850
- "format" => "none",
1851
- "read_from_head" => true
1852
- })
1853
- d = create_driver(config, false)
1854
- d.run {}
1855
- plugin = d.instance
1856
- mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once
1857
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1858
- end
1859
-
1860
- def test_tag_prefix_and_suffix_ignore
1861
- config = config_element("", "", {
1862
- "tag" => "pre.*.post*ignore",
1863
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1864
- "format" => "none",
1865
- "read_from_head" => true
1866
- })
1867
- d = create_driver(config, false)
1868
- d.run {}
1869
- plugin = d.instance
1870
- mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once
1871
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1872
- end
1873
-
1874
- data(
1875
- small: ["128", 128],
1876
- KiB: ["1k", 1024]
1877
- )
1878
- test 'max_line_size' do |(label, size)|
1879
- config = config_element("", "", {
1880
- "tag" => "max_line_size",
1881
- "path" => "#{@tmp_dir}/with_long_lines.txt",
1882
- "format" => "none",
1883
- "read_from_head" => true,
1884
- "max_line_size" => label,
1885
- "log_level" => "debug"
1886
- })
1887
- Fluent::FileWrapper.open("#{@tmp_dir}/with_long_lines.txt", "w+") do |f|
1888
- f.puts "foo"
1889
- f.puts "x" * size # 'x' * size + \n > @max_line_size
1890
- f.puts "bar"
1891
- end
1892
- d = create_driver(config, false)
1893
- timestamp = Time.parse("Mon Nov 29 11:22:33 UTC 2021")
1894
- Timecop.freeze(timestamp)
1895
- d.run(expect_records: 2)
1896
- assert_equal([
1897
- [{"message" => "foo"},{"message" => "bar"}],
1898
- [
1899
- "2021-11-29 11:22:33 +0000 [warn]: received line length is longer than #{size}\n",
1900
- "2021-11-29 11:22:33 +0000 [debug]: skipped line: #{'x' * size}\n"
1901
- ]
1902
- ],
1903
- [
1904
- d.events.collect { |event| event.last },
1905
- d.logs[-2..]
1906
- ])
1907
- end
1908
- end
1909
-
1910
- # Ensure that no fatal exception is raised when a file is missing and that
1911
- # files that do exist are still tailed as expected.
1912
- def test_missing_file
1913
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1914
- f.puts "test1"
1915
- f.puts "test2"
1916
- }
1917
-
1918
- # Try two different configs - one with read_from_head and one without,
1919
- # since their interactions with the filesystem differ.
1920
- config1 = config_element("", "", {
1921
- "tag" => "t1",
1922
- "path" => "#{@tmp_dir}/non_existent_file.txt,#{@tmp_dir}/tail.txt",
1923
- "format" => "none",
1924
- "rotate_wait" => "2s",
1925
- "pos_file" => "#{@tmp_dir}/tail.pos"
1926
- })
1927
- config2 = config1 + config_element("", "", { "read_from_head" => true })
1928
- [config1, config2].each do |config|
1929
- d = create_driver(config, false)
1930
- d.run(expect_emits: 1) do
1931
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1932
- f.puts "test3"
1933
- f.puts "test4"
1934
- }
1935
- end
1936
- # This test sometimes fails and it shows a potential bug of in_tail
1937
- # https://github.com/fluent/fluentd/issues/1434
1938
- events = d.events
1939
- assert_equal(2, events.length)
1940
- assert_equal({"message" => "test3"}, events[0][2])
1941
- assert_equal({"message" => "test4"}, events[1][2])
1942
- end
1943
- end
1944
-
1945
- sub_test_case 'inode_processing' do
1946
- def test_should_delete_file_pos_entry_for_non_existing_file_with_follow_inodes
1947
- config = common_follow_inode_config
1948
-
1949
- path = "#{@tmp_dir}/tail.txt"
1950
- ino = 1
1951
- pos = 1234
1952
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "wb") {|f|
1953
- f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
1954
- }
1955
-
1956
- d = create_driver(config, false)
1957
- d.run
1958
-
1959
- pos_file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r")
1960
- pos_file.pos = 0
1961
-
1962
- assert_raise(EOFError) do
1963
- pos_file.readline
1964
- end
1965
- end
1966
-
1967
- def test_should_write_latest_offset_after_rotate_wait
1968
- config = common_follow_inode_config
1969
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1970
- f.puts "test1"
1971
- f.puts "test2"
1972
- }
1973
-
1974
- d = create_driver(config, false)
1975
- d.run(expect_emits: 2, shutdown: false) do
1976
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
1977
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
1978
- sleep 1
1979
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt" + "1", "ab") {|f| f.puts "test4\n"}
1980
- end
1981
-
1982
- pos_file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r")
1983
- pos_file.pos = 0
1984
- line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1985
- waiting(5) {
1986
- while line_parts[2].to_i(16) != 24
1987
- sleep(0.1)
1988
- pos_file.pos = 0
1989
- line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1990
- end
1991
- }
1992
- assert_equal(24, line_parts[2].to_i(16))
1993
- d.instance_shutdown
1994
- end
1995
-
1996
- def test_should_remove_deleted_file
1997
- config = config_element("", "", {"format" => "none"})
1998
-
1999
- path = "#{@tmp_dir}/tail.txt"
2000
- ino = 1
2001
- pos = 1234
2002
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "wb") {|f|
2003
- f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
2004
- }
2005
-
2006
- d = create_driver(config)
2007
- d.run do
2008
- pos_file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r")
2009
- pos_file.pos = 0
2010
- assert_equal([], pos_file.readlines)
2011
- end
2012
- end
2013
-
2014
- def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
2015
- config = config_element("ROOT", "", {
2016
- "path" => "#{@tmp_dir}/tail.txt*",
2017
- "pos_file" => "#{@tmp_dir}/tail.pos",
2018
- "tag" => "t1",
2019
- "rotate_wait" => "1s",
2020
- "refresh_interval" => "1s",
2021
- "limit_recently_modified" => "1s",
2022
- "read_from_head" => "true",
2023
- "format" => "none",
2024
- "follow_inodes" => "true",
2025
- })
2026
-
2027
- d = create_driver(config, false)
2028
-
2029
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2030
- f.puts "test1"
2031
- f.puts "test2"
2032
- }
2033
- target_info = create_target_info("#{@tmp_dir}/tail.txt")
2034
-
2035
- d.run(expect_emits: 1, shutdown: false) do
2036
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2037
- end
2038
-
2039
-
2040
- Timecop.travel(Time.now + 10) do
2041
- waiting(5) {
2042
- # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
2043
- sleep 0.1 until d.instance.instance_variable_get(:@pf)[target_info].read_pos == 0
2044
- }
2045
- end
2046
-
2047
- assert_equal(0, d.instance.instance_variable_get(:@pf)[target_info].read_pos)
2048
-
2049
- d.instance_shutdown
2050
- end
2051
-
2052
- def test_should_read_from_head_on_file_renaming_with_star_in_pattern
2053
- config = config_element("ROOT", "", {
2054
- "path" => "#{@tmp_dir}/tail.txt*",
2055
- "pos_file" => "#{@tmp_dir}/tail.pos",
2056
- "tag" => "t1",
2057
- "rotate_wait" => "10s",
2058
- "refresh_interval" => "1s",
2059
- "limit_recently_modified" => "60s",
2060
- "read_from_head" => "true",
2061
- "format" => "none",
2062
- "follow_inodes" => "true"
2063
- })
2064
-
2065
- d = create_driver(config, false)
2066
-
2067
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2068
- f.puts "test1"
2069
- f.puts "test2"
2070
- }
2071
-
2072
- d.run(expect_emits: 2, shutdown: false) do
2073
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2074
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1")
2075
- end
2076
-
2077
- events = d.events
2078
- assert_equal(3, events.length)
2079
- d.instance_shutdown
2080
- end
2081
-
2082
- def test_should_not_read_from_head_on_rotation_when_watching_inodes
2083
- config = common_follow_inode_config
2084
-
2085
- d = create_driver(config, false)
2086
-
2087
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2088
- f.puts "test1"
2089
- f.puts "test2"
2090
- }
2091
-
2092
- d.run(expect_emits: 1, shutdown: false) do
2093
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2094
- end
2095
-
2096
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1")
2097
- Timecop.travel(Time.now + 10) do
2098
- sleep 2
2099
- events = d.events
2100
- assert_equal(3, events.length)
2101
- end
2102
-
2103
- d.instance_shutdown
2104
- end
2105
-
2106
- def test_should_mark_file_unwatched_if_same_name_file_created_with_different_inode
2107
- config = common_follow_inode_config
2108
-
2109
- d = create_driver(config, false)
2110
-
2111
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2112
- f.puts "test1"
2113
- f.puts "test2"
2114
- }
2115
- target_info = create_target_info("#{@tmp_dir}/tail.txt")
2116
-
2117
- d.run(expect_emits: 2, shutdown: false) do
2118
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2119
- cleanup_file("#{@tmp_dir}/tail.txt")
2120
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "test4\n"}
2121
- end
2122
-
2123
- new_target_info = create_target_info("#{@tmp_dir}/tail.txt")
2124
-
2125
- pos_file = d.instance.instance_variable_get(:@pf)
2126
-
2127
- waiting(10) {
2128
- # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
2129
- sleep 0.1 until pos_file[target_info].read_pos == 0
2130
- }
2131
- new_position = pos_file[new_target_info].read_pos
2132
- assert_equal(6, new_position)
2133
-
2134
- d.instance_shutdown
2135
- end
2136
-
2137
- def test_should_close_watcher_after_rotate_wait
2138
- now = Time.now
2139
- config = common_follow_inode_config + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
2140
-
2141
- d = create_driver(config, false)
2142
- d.instance.instance_eval do
2143
- opened_file_metrics = Fluent::Plugin::LocalMetrics.new
2144
- opened_file_metrics.configure(config_element('metrics', '', {}))
2145
- closed_file_metrics = Fluent::Plugin::LocalMetrics.new
2146
- closed_file_metrics.configure(config_element('metrics', '', {}))
2147
- rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
2148
- rotated_file_metrics.configure(config_element('metrics', '', {}))
2149
- @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
2150
- end
2151
-
2152
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2153
- f.puts "test1"
2154
- f.puts "test2"
2155
- }
2156
- target_info = create_target_info("#{@tmp_dir}/tail.txt")
2157
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything, anything).once
2158
- d.run(shutdown: false)
2159
- assert d.instance.instance_variable_get(:@tails)[target_info.path]
2160
-
2161
- Timecop.travel(now + 10) do
2162
- d.instance.instance_eval do
2163
- sleep 0.1 until @tails[target_info.path] == nil
2164
- end
2165
- assert_nil d.instance.instance_variable_get(:@tails)[target_info.path]
2166
- end
2167
- d.instance_shutdown
2168
- end
2169
-
2170
- def test_should_create_new_watcher_for_new_file_with_same_name
2171
- now = Time.now
2172
- config = common_follow_inode_config + config_element('', '', {"limit_recently_modified" => "2s"})
2173
-
2174
- d = create_driver(config, false)
2175
-
2176
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2177
- f.puts "test1"
2178
- f.puts "test2"
2179
- }
2180
- path_ino = create_target_info("#{@tmp_dir}/tail.txt")
2181
-
2182
- d.run(expect_emits: 1, shutdown: false) do
2183
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2184
- end
2185
-
2186
- cleanup_file("#{@tmp_dir}/tail.txt")
2187
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2188
- f.puts "test3"
2189
- f.puts "test4"
2190
- }
2191
- new_path_ino = create_target_info("#{@tmp_dir}/tail.txt")
2192
-
2193
- Timecop.travel(now + 10) do
2194
- sleep 3
2195
- d.instance.instance_eval do
2196
- @tails[path_ino.path] == nil
2197
- @tails[new_path_ino.path] != nil
2198
- end
2199
- end
2200
-
2201
- events = d.events
2202
-
2203
- assert_equal(5, events.length)
2204
-
2205
- d.instance_shutdown
2206
- end
2207
-
2208
- def test_truncate_file_with_follow_inodes
2209
- config = common_follow_inode_config
2210
-
2211
- d = create_driver(config, false)
2212
-
2213
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2214
- f.puts "test1"
2215
- f.puts "test2"
2216
- }
2217
-
2218
- d.run(expect_emits: 3, shutdown: false) do
2219
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2220
- sleep 2
2221
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w+b") {|f| f.puts "test4\n"}
2222
- end
2223
-
2224
- events = d.events
2225
- assert_equal(4, events.length)
2226
- assert_equal({"message" => "test1"}, events[0][2])
2227
- assert_equal({"message" => "test2"}, events[1][2])
2228
- assert_equal({"message" => "test3"}, events[2][2])
2229
- assert_equal({"message" => "test4"}, events[3][2])
2230
- d.instance_shutdown
2231
- end
2232
-
2233
- # issue #3464
2234
- def test_should_replace_target_info
2235
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2236
- f.puts "test1\n"
2237
- }
2238
- target_info = create_target_info("#{@tmp_dir}/tail.txt")
2239
- inodes = []
2240
-
2241
- config = config_element("ROOT", "", {
2242
- "path" => "#{@tmp_dir}/tail.txt*",
2243
- "pos_file" => "#{@tmp_dir}/tail.pos",
2244
- "tag" => "t1",
2245
- "refresh_interval" => "60s",
2246
- "read_from_head" => "true",
2247
- "format" => "none",
2248
- "rotate_wait" => "1s",
2249
- "follow_inodes" => "true"
2250
- })
2251
- d = create_driver(config, false)
2252
- d.run(timeout: 5) do
2253
- while d.events.size < 1 do
2254
- sleep 0.1
2255
- end
2256
- inodes = d.instance.instance_variable_get(:@tails).values.collect do |tw|
2257
- tw.ino
2258
- end
2259
- assert_equal([target_info.ino], inodes)
2260
-
2261
- cleanup_file("#{@tmp_dir}/tail.txt")
2262
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "test2\n"}
2263
-
2264
- while d.events.size < 2 do
2265
- sleep 0.1
2266
- end
2267
- inodes = d.instance.instance_variable_get(:@tails).values.collect do |tw|
2268
- tw.ino
2269
- end
2270
- new_target_info = create_target_info("#{@tmp_dir}/tail.txt")
2271
- assert_not_equal(target_info.ino, new_target_info.ino)
2272
- assert_equal([new_target_info.ino], inodes)
2273
- end
2274
- end
2275
- end
2276
-
2277
- sub_test_case "tail_path" do
2278
- def test_tail_path_with_singleline
2279
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2280
- f.puts "test1"
2281
- f.puts "test2"
2282
- }
2283
-
2284
- d = create_driver(SINGLE_LINE_CONFIG + config_element("", "", { "path_key" => "path" }))
2285
-
2286
- d.run(expect_emits: 1) do
2287
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
2288
- f.puts "test3"
2289
- f.puts "test4"
2290
- }
2291
- end
2292
-
2293
- events = d.events
2294
- assert_equal(true, events.length > 0)
2295
- events.each do |emit|
2296
- assert_equal("#{@tmp_dir}/tail.txt", emit[2]["path"])
2297
- end
2298
- end
2299
-
2300
- def test_tail_path_with_multiline_with_firstline
2301
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
2302
-
2303
- config = config_element("", "", {
2304
- "path_key" => "path",
2305
- "format" => "multiline",
2306
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
2307
- "format_firstline" => "/^[s]/"
2308
- })
2309
- d = create_driver(config)
2310
- d.run(expect_emits: 1) do
2311
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
2312
- f.puts "f test1"
2313
- f.puts "s test2"
2314
- f.puts "f test3"
2315
- f.puts "f test4"
2316
- f.puts "s test5"
2317
- f.puts "s test6"
2318
- f.puts "f test7"
2319
- f.puts "s test8"
2320
- }
2321
- end
2322
-
2323
- events = d.events
2324
- assert_equal(4, events.length)
2325
- events.each do |emit|
2326
- assert_equal("#{@tmp_dir}/tail.txt", emit[2]["path"])
2327
- end
2328
- end
2329
-
2330
- def test_tail_path_with_multiline_without_firstline
2331
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
2332
-
2333
- config = config_element("", "", {
2334
- "path_key" => "path",
2335
- "format" => "multiline",
2336
- "format1" => "/(?<var1>foo \\d)\\n/",
2337
- "format2" => "/(?<var2>bar \\d)\\n/",
2338
- "format3" => "/(?<var3>baz \\d)/",
2339
- })
2340
- d = create_driver(config)
2341
- d.run(expect_emits: 1) do
2342
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
2343
- f.puts "foo 1"
2344
- f.puts "bar 1"
2345
- f.puts "baz 1"
2346
- }
2347
- end
2348
-
2349
- events = d.events
2350
- assert(events.length > 0)
2351
- events.each do |emit|
2352
- assert_equal("#{@tmp_dir}/tail.txt", emit[2]["path"])
2353
- end
2354
- end
2355
-
2356
- def test_tail_path_with_multiline_with_multiple_paths
2357
- if ENV["APPVEYOR"] && Fluent.windows?
2358
- omit "This testcase is unstable on AppVeyor."
2359
- end
2360
- files = ["#{@tmp_dir}/tail1.txt", "#{@tmp_dir}/tail2.txt"]
2361
- files.each { |file| Fluent::FileWrapper.open(file, "wb") { |f| } }
2362
-
2363
- config = config_element("", "", {
2364
- "path" => "#{files[0]},#{files[1]}",
2365
- "path_key" => "path",
2366
- "tag" => "t1",
2367
- "format" => "multiline",
2368
- "format1" => "/^[s|f] (?<message>.*)/",
2369
- "format_firstline" => "/^[s]/"
2370
- })
2371
- d = create_driver(config, false)
2372
- d.run(expect_emits: 2) do
2373
- files.each do |file|
2374
- Fluent::FileWrapper.open(file, 'ab') { |f|
2375
- f.puts "f #{file} line should be ignored"
2376
- f.puts "s test1"
2377
- f.puts "f test2"
2378
- f.puts "f test3"
2379
- f.puts "s test4"
2380
- }
2381
- end
2382
- end
2383
-
2384
- events = d.events
2385
- assert_equal(4, events.length)
2386
- assert_equal(files, [events[0][2]["path"], events[1][2]["path"]].sort)
2387
- # "test4" events are here because these events are flushed at shutdown phase
2388
- assert_equal(files, [events[2][2]["path"], events[3][2]["path"]].sort)
2389
- end
2390
- end
2391
-
2392
- def test_limit_recently_modified
2393
- now = Time.new(2010, 1, 2, 3, 4, 5)
2394
- FileUtils.touch("#{@tmp_dir}/tail_unwatch.txt", mtime: (now - 3601))
2395
- FileUtils.touch("#{@tmp_dir}/tail_watch1.txt", mtime: (now - 3600))
2396
- FileUtils.touch("#{@tmp_dir}/tail_watch2.txt", mtime: now)
2397
-
2398
- config = config_element('', '', {
2399
- 'tag' => 'tail',
2400
- 'path' => "#{@tmp_dir}/*.txt",
2401
- 'format' => 'none',
2402
- 'limit_recently_modified' => '3600s'
2403
- })
2404
-
2405
- expected_files = [
2406
- create_target_info("#{@tmp_dir}/tail_watch1.txt"),
2407
- create_target_info("#{@tmp_dir}/tail_watch2.txt")
2408
- ]
2409
-
2410
- Timecop.freeze(now) do
2411
- plugin = create_driver(config, false).instance
2412
- assert_equal(expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
2413
- end
2414
- end
2415
-
2416
- def test_skip_refresh_on_startup
2417
- FileUtils.touch("#{@tmp_dir}/tail.txt")
2418
- config = config_element('', '', {
2419
- 'format' => 'none',
2420
- 'refresh_interval' => 1,
2421
- 'skip_refresh_on_startup' => true
2422
- })
2423
- d = create_driver(config)
2424
- d.run(shutdown: false) {}
2425
- assert_equal(0, d.instance.instance_variable_get(:@tails).keys.size)
2426
- # detect a file at first execution of in_tail_refresh_watchers timer
2427
- waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1 }
2428
- d.instance_shutdown
2429
- end
2430
-
2431
- def test_ENOENT_error_after_setup_watcher
2432
- path = "#{@tmp_dir}/tail.txt"
2433
- FileUtils.touch(path)
2434
- config = config_element('', '', {
2435
- 'format' => 'none',
2436
- })
2437
- d = create_driver(config)
2438
- file_deleted = false
2439
- mock.proxy(d.instance).existence_path do |hash|
2440
- unless file_deleted
2441
- cleanup_file(path)
2442
- file_deleted = true
2443
- end
2444
- hash
2445
- end.twice
2446
- assert_nothing_raised do
2447
- d.run(shutdown: false) {}
2448
- end
2449
- assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed. Continuing without tailing it.\n") },
2450
- $log.out.logs.join("\n"))
2451
- ensure
2452
- d.instance_shutdown if d && d.instance
2453
- end
2454
-
2455
- def test_EACCES_error_after_setup_watcher
2456
- omit "Cannot test with root user" if Process::UID.eid == 0
2457
- path = "#{@tmp_dir}/noaccess/tail.txt"
2458
- begin
2459
- FileUtils.mkdir_p("#{@tmp_dir}/noaccess")
2460
- FileUtils.chmod(0755, "#{@tmp_dir}/noaccess")
2461
- FileUtils.touch(path)
2462
- config = config_element('', '', {
2463
- 'tag' => "tail",
2464
- 'path' => path,
2465
- 'format' => 'none',
2466
- })
2467
- d = create_driver(config, false)
2468
- mock.proxy(d.instance).existence_path do |hash|
2469
- FileUtils.chmod(0000, "#{@tmp_dir}/noaccess")
2470
- hash
2471
- end.twice
2472
- assert_nothing_raised do
2473
- d.run(shutdown: false) {}
2474
- end
2475
- assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed. Continuing without tailing it.\n") },
2476
- $log.out.logs.join("\n"))
2477
- end
2478
- ensure
2479
- d.instance_shutdown if d && d.instance
2480
- if File.exist?("#{@tmp_dir}/noaccess")
2481
- FileUtils.chmod(0755, "#{@tmp_dir}/noaccess")
2482
- FileUtils.rm_rf("#{@tmp_dir}/noaccess")
2483
- end
2484
- end unless Fluent.windows?
2485
-
2486
- def test_EACCES
2487
- path = "#{@tmp_dir}/tail.txt"
2488
- FileUtils.touch(path)
2489
- config = config_element('', '', {
2490
- 'format' => 'none',
2491
- })
2492
- d = create_driver(config)
2493
- mock.proxy(Fluent::FileWrapper).stat(path) do |stat|
2494
- raise Errno::EACCES
2495
- end.at_least(1)
2496
- assert_nothing_raised do
2497
- d.run(shutdown: false) {}
2498
- end
2499
- assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
2500
- ensure
2501
- d.instance_shutdown if d && d.instance
2502
- end
2503
-
2504
- def test_shutdown_timeout
2505
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") do |f|
2506
- # Should be large enough to take too long time to consume
2507
- (1024 * 1024 * 5).times do
2508
- f.puts "{\"test\":\"fizzbuzz\"}"
2509
- end
2510
- end
2511
-
2512
- config =
2513
- CONFIG_READ_FROM_HEAD +
2514
- config_element('', '', {
2515
- 'format' => 'json',
2516
- 'skip_refresh_on_startup' => true,
2517
- })
2518
- shutdown_start_time = 0
2519
-
2520
- d = create_driver(config)
2521
- mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
2522
- mock.proxy(io_handler).ready_to_shutdown(anything) do
2523
- shutdown_start_time = Fluent::Clock.now
2524
- end
2525
- io_handler.shutdown_timeout = 0.5
2526
- io_handler
2527
- end
2528
-
2529
- assert_nothing_raised do
2530
- d.run(expect_emits: 1)
2531
- end
2532
-
2533
- elapsed = Fluent::Clock.now - shutdown_start_time
2534
- assert_true(elapsed > 0.5 && elapsed < 2.0,
2535
- "elapsed time: #{elapsed}")
2536
- end
2537
-
2538
- sub_test_case "throttling logs at in_tail level" do
2539
- data("file test1.log no_limit 5120 text: msg" => ["test1.log", 5120, "msg"],
2540
- "file test2.log no_limit 1024 text: test" => ["test2.log", 1024, "test"])
2541
- def test_lines_collected_with_no_throttling(data)
2542
- file, num_lines, msg = data
2543
-
2544
- pattern = "/^#{@tmp_dir}\/(?<file>.+)\.log$/"
2545
- rule = create_rule_directive({
2546
- "file" => "/test.*/",
2547
- }, -1)
2548
- group = create_group_directive(pattern, "1s", rule)
2549
- path_element = create_path_element(file)
2550
-
2551
- conf = ROOT_CONFIG + group + path_element + CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG
2552
-
2553
- Fluent::FileWrapper.open("#{@tmp_dir}/#{file}", 'wb') do |f|
2554
- num_lines.times do
2555
- f.puts "#{msg}\n"
2556
- end
2557
- end
2558
-
2559
-
2560
- d = create_driver(conf, false)
2561
- d.run(timeout: 3) do
2562
- start_time = Fluent::Clock.now
2563
-
2564
- assert_equal(num_lines, d.record_count)
2565
- assert_equal({ "message" => msg }, d.events[0][2])
2566
-
2567
- prev_count = d.record_count
2568
- sleep(0.1) while d.emit_count < 1
2569
- assert_true(Fluent::Clock.now - start_time < 2)
2570
- ## after waiting for 1 (+ jitter) secs, limit will reset
2571
- ## Plugin will start reading but it will encounter EOF Error
2572
- ## since no logs are left to be read
2573
- ## Hence, d.record_count = prev_count
2574
- tail_watcher_interval = 1.0 # hard coded value in in_tail
2575
- safety_ratio = 1.2
2576
- jitter = tail_watcher_interval * safety_ratio
2577
- sleep(1.0 + jitter)
2578
- assert_equal(0, d.record_count - prev_count)
2579
- end
2580
- end
2581
-
2582
- test "lines collected with throttling" do
2583
- file = "podname1_namespace12_container-123456.log"
2584
- limit = 1000
2585
- rate_period = 2
2586
- num_lines = 3000
2587
- msg = "a" * 8190 # Total size = 8190 bytes + 2 (\n) bytes
2588
-
2589
- rule = create_rule_directive({
2590
- "namespace"=> "/namespace.+/",
2591
- "podname"=> "/podname.+/",
2592
- }, limit)
2593
- path_element = create_path_element(file)
2594
- conf = ROOT_CONFIG + create_group_directive(tailing_group_pattern, "#{rate_period}s", rule) + path_element + SINGLE_LINE_CONFIG + CONFIG_READ_FROM_HEAD
2595
-
2596
- d = create_driver(conf, false)
2597
- file_path = "#{@tmp_dir}/#{file}"
2598
-
2599
- Fluent::FileWrapper.open(file_path, 'wb') do |f|
2600
- num_lines.times do
2601
- f.puts msg
2602
- end
2603
- end
2604
-
2605
- d.run(timeout: 15) do
2606
- sleep_interval = 0.1
2607
- tail_watcher_interval = 1.0 # hard coded value in in_tail
2608
- safety_ratio = 1.2
2609
- lower_jitter = sleep_interval * safety_ratio
2610
- upper_jitter = (tail_watcher_interval + sleep_interval) * safety_ratio
2611
- lower_interval = rate_period - lower_jitter
2612
- upper_interval = rate_period + upper_jitter
2613
-
2614
- emit_count = 0
2615
- prev_count = 0
2616
-
2617
- while emit_count < 3 do
2618
- start_time = Fluent::Clock.now
2619
- sleep(sleep_interval) while d.emit_count <= emit_count
2620
- elapsed_seconds = Fluent::Clock.now - start_time
2621
- if emit_count > 0
2622
- assert_true(elapsed_seconds > lower_interval && elapsed_seconds < upper_interval,
2623
- "elapsed_seconds #{elapsed_seconds} is out of allowed range:\n" +
2624
- " lower: #{lower_interval} [sec]\n" +
2625
- " upper: #{upper_interval} [sec]")
2626
- end
2627
- assert_equal(limit, d.record_count - prev_count)
2628
- emit_count = d.emit_count
2629
- prev_count = d.record_count
2630
- end
2631
-
2632
- ## When all the lines are read and rate_period seconds are over
2633
- ## limit will reset and since there are no more logs to be read,
2634
- ## number_lines_read will be 0
2635
- sleep upper_interval
2636
- gw = d.instance.find_group_from_metadata(file_path)
2637
- assert_equal(0, gw.current_paths[file_path].number_lines_read)
2638
- end
2639
- end
2640
- end
2641
-
2642
- sub_test_case "Update watchers for rotation with follow_inodes" do
2643
- def test_updateTW_before_refreshTW_and_detach_before_refreshTW
2644
- config = config_element(
2645
- "ROOT",
2646
- "",
2647
- {
2648
- "path" => "#{@tmp_dir}/tail.txt*",
2649
- "pos_file" => "#{@tmp_dir}/tail.pos",
2650
- "tag" => "t1",
2651
- "format" => "none",
2652
- "read_from_head" => "true",
2653
- "follow_inodes" => "true",
2654
- # In order to detach the old watcher quickly.
2655
- "rotate_wait" => "1s",
2656
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
2657
- # called by a timer.
2658
- "refresh_interval" => "1h",
2659
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
2660
- # so disable it in order to reproduce the same condition stably.
2661
- "enable_stat_watcher" => "false",
2662
- }
2663
- )
2664
- d = create_driver(config, false)
2665
-
2666
- tail_watchers = []
2667
- stub.proxy(d.instance).setup_watcher do |tw|
2668
- tail_watchers.append(tw)
2669
- tw
2670
- end
2671
-
2672
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file1 log1"}
2673
-
2674
- d.run(expect_records: 4, timeout: 10) do
2675
- # Rotate (If the timing is bad, `TailWatcher::on_notify` might be called between mv and new-file-creation)
2676
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file1 log2"}
2677
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
2678
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file2 log1"}
2679
-
2680
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` updates the TailWatcher:
2681
- # TailWatcher(path: "tail.txt", inode: inode_0) => TailWatcher(path: "tail.txt", inode: inode_1)
2682
- # The old TailWathcer is detached here since `rotate_wait` is just `1s`.
2683
- sleep 3
2684
-
2685
- # This reproduces the following situation:
2686
- # Rotation => update_watcher => refresh_watchers
2687
- # This adds a new TailWatcher: TailWatcher(path: "tail.txt1", inode: inode_0)
2688
- d.instance.refresh_watchers
2689
-
2690
- # Append to the new current log file.
2691
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file2 log2"}
2692
- end
2693
-
2694
- inode_0 = tail_watchers[0].ino
2695
- inode_1 = tail_watchers[1].ino
2696
- record_values = d.events.collect { |event| event[2]["message"] }.sort
2697
- position_entries = []
2698
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
2699
- f.readlines(chomp: true).each do |line|
2700
- values = line.split("\t")
2701
- position_entries.append([values[0], values[1], values[2].to_i(16)])
2702
- end
2703
- end
2704
-
2705
- assert_equal(
2706
- {
2707
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2"],
2708
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1"],
2709
- tail_watcher_inodes: [inode_0, inode_1, inode_0],
2710
- tail_watcher_io_handler_opened_statuses: [false, false, false],
2711
- position_entries: [
2712
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_0],
2713
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_1],
2714
- ],
2715
- },
2716
- {
2717
- record_values: record_values,
2718
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
2719
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
2720
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
2721
- position_entries: position_entries
2722
- },
2723
- )
2724
- end
2725
-
2726
- def test_updateTW_before_refreshTW_and_detach_after_refreshTW
2727
- config = config_element(
2728
- "ROOT",
2729
- "",
2730
- {
2731
- "path" => "#{@tmp_dir}/tail.txt*",
2732
- "pos_file" => "#{@tmp_dir}/tail.pos",
2733
- "tag" => "t1",
2734
- "format" => "none",
2735
- "read_from_head" => "true",
2736
- "follow_inodes" => "true",
2737
- # In order to detach the old watcher after refresh_watchers.
2738
- "rotate_wait" => "4s",
2739
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
2740
- # called by a timer.
2741
- "refresh_interval" => "1h",
2742
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
2743
- # so disable it in order to reproduce the same condition stably.
2744
- "enable_stat_watcher" => "false",
2745
- }
2746
- )
2747
- d = create_driver(config, false)
2748
-
2749
- tail_watchers = []
2750
- stub.proxy(d.instance).setup_watcher do |tw|
2751
- tail_watchers.append(tw)
2752
- tw
2753
- end
2754
-
2755
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file1 log1"}
2756
-
2757
- d.run(expect_records: 4, timeout: 10) do
2758
- # Rotate (If the timing is bad, `TailWatcher::on_notify` might be called between mv and new-file-creation)
2759
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file1 log2"}
2760
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
2761
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file2 log1"}
2762
-
2763
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` updates the TailWatcher:
2764
- # TailWatcher(path: "tail.txt", inode: inode_0) => TailWatcher(path: "tail.txt", inode: inode_1)
2765
- sleep 2
2766
-
2767
- # This reproduces the following situation:
2768
- # Rotation => update_watcher => refresh_watchers
2769
- # This adds a new TailWatcher: TailWatcher(path: "tail.txt1", inode: inode_0)
2770
- d.instance.refresh_watchers
2771
-
2772
- # The old TailWathcer is detached here since `rotate_wait` is `4s`.
2773
- sleep 3
2774
-
2775
- # Append to the new current log file.
2776
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file2 log2"}
2777
- end
2778
-
2779
- inode_0 = tail_watchers[0].ino
2780
- inode_1 = tail_watchers[1].ino
2781
- record_values = d.events.collect { |event| event[2]["message"] }.sort
2782
- position_entries = []
2783
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
2784
- f.readlines(chomp: true).each do |line|
2785
- values = line.split("\t")
2786
- position_entries.append([values[0], values[1], values[2].to_i(16)])
2787
- end
2788
- end
2789
-
2790
- assert_equal(
2791
- {
2792
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2"],
2793
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1"],
2794
- tail_watcher_inodes: [inode_0, inode_1, inode_0],
2795
- tail_watcher_io_handler_opened_statuses: [false, false, false],
2796
- position_entries: [
2797
- # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
2798
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_0],
2799
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_1],
2800
- ],
2801
- },
2802
- {
2803
- record_values: record_values,
2804
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
2805
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
2806
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
2807
- position_entries: position_entries
2808
- },
2809
- )
2810
- end
2811
-
2812
- # The scenario where in_tail wrongly detaches TailWatcher.
2813
- # This is reported in https://github.com/fluent/fluentd/issues/4190.
2814
- def test_updateTW_after_refreshTW
2815
- config = config_element(
2816
- "ROOT",
2817
- "",
2818
- {
2819
- "path" => "#{@tmp_dir}/tail.txt*",
2820
- "pos_file" => "#{@tmp_dir}/tail.pos",
2821
- "tag" => "t1",
2822
- "format" => "none",
2823
- "read_from_head" => "true",
2824
- "follow_inodes" => "true",
2825
- # In order to detach the old watcher quickly.
2826
- "rotate_wait" => "1s",
2827
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
2828
- # called by a timer.
2829
- "refresh_interval" => "1h",
2830
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
2831
- # so disable it in order to reproduce the same condition stably.
2832
- "enable_stat_watcher" => "false",
2833
- }
2834
- )
2835
- d = create_driver(config, false)
2836
-
2837
- tail_watchers = []
2838
- stub.proxy(d.instance).setup_watcher do |tw|
2839
- tail_watchers.append(tw)
2840
- tw
2841
- end
2842
-
2843
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file1 log1"}
2844
-
2845
- d.run(expect_records: 4, timeout: 10) do
2846
- # Rotate (If the timing is bad, `TailWatcher::on_notify` might be called between mv and new-file-creation)
2847
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file1 log2"}
2848
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
2849
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file2 log1"}
2850
-
2851
- # This reproduces the following situation:
2852
- # Rotation => refresh_watchers => update_watcher
2853
- # This add a new TailWatcher: TailWatcher(path: "tail.txt", inode: inode_1)
2854
- # This overwrites `@tails["tail.txt"]`
2855
- d.instance.refresh_watchers
2856
-
2857
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to update the TailWatcher:
2858
- # TailWatcher(path: "tail.txt", inode: inode_0) => TailWatcher(path: "tail.txt", inode: inode_1)
2859
- # However, it is already added in `refresh_watcher`, so `update_watcher` doesn't create the new TailWatcher.
2860
- # The old TailWathcer is detached here since `rotate_wait` is just `1s`.
2861
- sleep 3
2862
-
2863
- # This adds a new TailWatcher: TailWatcher(path: "tail.txt1", inode: inode_0)
2864
- d.instance.refresh_watchers
2865
-
2866
- # Append to the new current log file.
2867
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file2 log2"}
2868
- end
2869
-
2870
- inode_0 = tail_watchers[0].ino
2871
- inode_1 = tail_watchers[1].ino
2872
- record_values = d.events.collect { |event| event[2]["message"] }.sort
2873
- position_entries = []
2874
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
2875
- f.readlines(chomp: true).each do |line|
2876
- values = line.split("\t")
2877
- position_entries.append([values[0], values[1], values[2].to_i(16)])
2878
- end
2879
- end
2880
-
2881
- assert_equal(
2882
- {
2883
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2"],
2884
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1"],
2885
- tail_watcher_inodes: [inode_0, inode_1, inode_0],
2886
- tail_watcher_io_handler_opened_statuses: [false, false, false],
2887
- position_entries: [
2888
- # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
2889
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_0],
2890
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_1],
2891
- ],
2892
- },
2893
- {
2894
- record_values: record_values,
2895
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
2896
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
2897
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
2898
- position_entries: position_entries
2899
- },
2900
- )
2901
- end
2902
-
2903
- def test_path_resurrection
2904
- config = config_element(
2905
- "ROOT",
2906
- "",
2907
- {
2908
- "path" => "#{@tmp_dir}/tail.txt*",
2909
- "pos_file" => "#{@tmp_dir}/tail.pos",
2910
- "tag" => "t1",
2911
- "format" => "none",
2912
- "read_from_head" => "true",
2913
- "follow_inodes" => "true",
2914
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
2915
- # called by a timer.
2916
- "refresh_interval" => "1h",
2917
- # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
2918
- # Because of this problem, log duplication can occur during `rotate_wait`.
2919
- # Need to set `rotate_wait 0` for a workaround.
2920
- "rotate_wait" => "0s",
2921
- }
2922
- )
2923
- d = create_driver(config, false)
2924
-
2925
- tail_watchers = []
2926
- stub.proxy(d.instance).setup_watcher do |tw|
2927
- tail_watchers.append(tw)
2928
- tw
2929
- end
2930
-
2931
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file1 log1"}
2932
-
2933
- d.run(expect_records: 5, timeout: 10) do
2934
- # Rotate
2935
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file1 log2"}
2936
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
2937
- # TailWatcher(path: "tail.txt", inode: inode_0) detects `tail.txt` disappeared.
2938
- # Call `update_watcher` to stop and discard self.
2939
- # If not discarding, then it will be a orphan and cause leak and log duplication.
2940
- #
2941
- # This reproduces the case where the notify to TailWatcher comes before the new file for the path
2942
- # is created during rotation.
2943
- # (stat_watcher notifies faster than a new file is created)
2944
- # Overall, this is a rotation operation, but from the TailWatcher, it appears as if the file
2945
- # was resurrected once it disappeared.
2946
- sleep 2 # On Windows and macOS, StatWatcher doesn't work, so need enough interval for TimeTrigger.
2947
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file2 log1"}
2948
-
2949
- # Add new TailWatchers
2950
- # tail.txt: TailWatcher(path: "tail.txt", inode: inode_1)
2951
- # tail.txt: TailWatcher(path: "tail.txt1", inode: inode_0)
2952
- # NOTE: If not discarding the first TailWatcher on notify, this makes it a orphan because
2953
- # this overwrites the `@tails[tail.txt]` by adding TailWatcher(path: "tail.txt", inode: inode_1)
2954
- d.instance.refresh_watchers
2955
-
2956
- # This does nothing.
2957
- # NOTE: If not discarding the first TailWatcher on notify, this add
2958
- # tail.txt1: TailWatcher(path: "tail.txt1", inode: inode_0)
2959
- # because the previous refresh_watcher overwrites `@tails[tail.txt]` and the inode_0 is lost.
2960
- # This would cause log duplication.
2961
- d.instance.refresh_watchers
2962
-
2963
- # Append to the old file
2964
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt1", "ab") {|f| f.puts "file1 log3"}
2965
-
2966
- # Append to the new current log file.
2967
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file2 log2"}
2968
- end
2969
-
2970
- inode_0 = Fluent::FileWrapper.stat("#{@tmp_dir}/tail.txt1").ino
2971
- inode_1 = Fluent::FileWrapper.stat("#{@tmp_dir}/tail.txt").ino
2972
- record_values = d.events.collect { |event| event[2]["message"] }.sort
2973
- position_entries = []
2974
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
2975
- f.readlines(chomp: true).each do |line|
2976
- values = line.split("\t")
2977
- position_entries.append([values[0], values[1], values[2].to_i(16)])
2978
- end
2979
- end
2980
-
2981
- assert_equal(
2982
- {
2983
- record_values: ["file1 log1", "file1 log2", "file1 log3", "file2 log1", "file2 log2"],
2984
- tail_watcher_set: Set[
2985
- {
2986
- path: "#{@tmp_dir}/tail.txt",
2987
- inode: inode_0,
2988
- io_handler_opened_status: false,
2989
- },
2990
- {
2991
- path: "#{@tmp_dir}/tail.txt",
2992
- inode: inode_1,
2993
- io_handler_opened_status: false,
2994
- },
2995
- {
2996
- path: "#{@tmp_dir}/tail.txt1",
2997
- inode: inode_0,
2998
- io_handler_opened_status: false,
2999
- },
3000
- ],
3001
- position_entries: [
3002
- ["#{@tmp_dir}/tail.txt", "0000000000000021", inode_0],
3003
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_1],
3004
- ],
3005
- },
3006
- {
3007
- record_values: record_values,
3008
- tail_watcher_set: Set.new(tail_watchers.collect { |tw|
3009
- {
3010
- path: tw.path,
3011
- inode: tw.ino,
3012
- io_handler_opened_status: tw.instance_variable_get(:@io_handler)&.opened? || false,
3013
- }
3014
- }),
3015
- position_entries: position_entries,
3016
- },
3017
- )
3018
- end
3019
-
3020
- def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait
3021
- config = config_element(
3022
- "ROOT",
3023
- "",
3024
- {
3025
- "path" => "#{@tmp_dir}/tail.txt*",
3026
- "pos_file" => "#{@tmp_dir}/tail.pos",
3027
- "tag" => "t1",
3028
- "format" => "none",
3029
- "read_from_head" => "true",
3030
- "follow_inodes" => "true",
3031
- "rotate_wait" => "3s",
3032
- "refresh_interval" => "1h",
3033
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
3034
- # so disable it in order to reproduce the same condition stably.
3035
- "enable_stat_watcher" => "false",
3036
- }
3037
- )
3038
- d = create_driver(config, false)
3039
-
3040
- tail_watchers = []
3041
- stub.proxy(d.instance).setup_watcher do |tw|
3042
- tail_watchers.append(tw)
3043
- mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.
3044
- tw
3045
- end
3046
-
3047
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
3048
-
3049
- d.run(expect_records: 6, timeout: 15) do
3050
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
3051
-
3052
- sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3053
-
3054
- FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
3055
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
3056
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
3057
-
3058
- sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3059
-
3060
- # Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)
3061
- [1, 0].each do |i|
3062
- FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
3063
- end
3064
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
3065
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3066
-
3067
- # Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.
3068
- # (Note: Currently, there is no harm in duplicate calls)
3069
- sleep 4
3070
- end
3071
-
3072
- inode_0 = tail_watchers[0]&.ino
3073
- inode_1 = tail_watchers[1]&.ino
3074
- inode_2 = tail_watchers[2]&.ino
3075
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3076
- position_entries = []
3077
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3078
- f.readlines(chomp: true).each do |line|
3079
- values = line.split("\t")
3080
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3081
- end
3082
- end
3083
-
3084
- assert_equal(
3085
- {
3086
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
3087
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
3088
- tail_watcher_inodes: [inode_0, inode_1, inode_2],
3089
- tail_watcher_io_handler_opened_statuses: [false, false, false],
3090
- position_entries: [
3091
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_0],
3092
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_1],
3093
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
3094
- ],
3095
- },
3096
- {
3097
- record_values: record_values,
3098
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3099
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3100
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3101
- position_entries: position_entries
3102
- },
3103
- )
3104
- end
3105
- end
3106
-
3107
- sub_test_case "Update watchers for rotation without follow_inodes" do
3108
- # The scenario where in_tail wrongly unwatches the PositionEntry.
3109
- # This is reported in https://github.com/fluent/fluentd/issues/3614.
3110
- def test_refreshTW_during_rotation
3111
- config = config_element(
3112
- "ROOT",
3113
- "",
3114
- {
3115
- "path" => "#{@tmp_dir}/tail.txt0",
3116
- "pos_file" => "#{@tmp_dir}/tail.pos",
3117
- "tag" => "t1",
3118
- "format" => "none",
3119
- "read_from_head" => "true",
3120
- # In order to detach the old watcher quickly.
3121
- "rotate_wait" => "3s",
3122
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
3123
- # called by a timer.
3124
- "refresh_interval" => "1h",
3125
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
3126
- # so disable it in order to reproduce the same condition stably.
3127
- "enable_stat_watcher" => "false",
3128
- }
3129
- )
3130
- d = create_driver(config, false)
3131
-
3132
- tail_watchers = []
3133
- stub.proxy(d.instance).setup_watcher do |tw|
3134
- tail_watchers.append(tw)
3135
- tw
3136
- end
3137
-
3138
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
3139
-
3140
- d.run(expect_records: 6, timeout: 15) do
3141
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
3142
- FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
3143
-
3144
- # This reproduces the following situation:
3145
- # `refresh_watchers` is called during the rotation process and it detects the current file being lost.
3146
- # Then it stops and unwatches the TailWatcher.
3147
- d.instance.refresh_watchers
3148
-
3149
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
3150
-
3151
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to add the new TailWatcher.
3152
- # After `rotate_wait` interval, the PositionEntry is unwatched.
3153
- # HOWEVER, the new TailWatcher is still using that PositionEntry, so this breaks the PositionFile!!
3154
- # That PositionEntry is removed from `PositionFile::map`, but it is still working and remaining in the real pos file.
3155
- sleep 5
3156
-
3157
- # Append to the new current log file.
3158
- # The PositionEntry is updated although it does not exist in `PositionFile::map`.
3159
- # `PositionFile::map`: empty
3160
- # Real pos file: `.../tail.txt 0000000000000016 (inode)`
3161
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
3162
-
3163
- # Rotate again
3164
- [1, 0].each do |i|
3165
- FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
3166
- end
3167
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
3168
-
3169
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to update the TailWatcher.
3170
- sleep 3
3171
-
3172
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3173
- end
3174
-
3175
- inode_0 = tail_watchers[0]&.ino
3176
- inode_1 = tail_watchers[1]&.ino
3177
- inode_2 = tail_watchers[2]&.ino
3178
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3179
- position_entries = []
3180
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3181
- f.readlines(chomp: true).each do |line|
3182
- values = line.split("\t")
3183
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3184
- end
3185
- end
3186
-
3187
- assert_equal(
3188
- {
3189
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
3190
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
3191
- tail_watcher_inodes: [inode_0, inode_1, inode_2],
3192
- tail_watcher_io_handler_opened_statuses: [false, false, false],
3193
- position_entries: [
3194
- # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
3195
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
3196
- ],
3197
- },
3198
- {
3199
- record_values: record_values,
3200
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3201
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3202
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3203
- position_entries: position_entries
3204
- },
3205
- )
3206
- end
3207
-
3208
- def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait
3209
- config = config_element(
3210
- "ROOT",
3211
- "",
3212
- {
3213
- "path" => "#{@tmp_dir}/tail.txt0",
3214
- "pos_file" => "#{@tmp_dir}/tail.pos",
3215
- "tag" => "t1",
3216
- "format" => "none",
3217
- "read_from_head" => "true",
3218
- "rotate_wait" => "3s",
3219
- "refresh_interval" => "1h",
3220
- }
3221
- )
3222
- d = create_driver(config, false)
3223
-
3224
- tail_watchers = []
3225
- stub.proxy(d.instance).setup_watcher do |tw|
3226
- tail_watchers.append(tw)
3227
- mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.
3228
- tw
3229
- end
3230
-
3231
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
3232
-
3233
- d.run(expect_records: 6, timeout: 15) do
3234
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
3235
-
3236
- sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3237
-
3238
- FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
3239
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
3240
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
3241
-
3242
- sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3243
-
3244
- # Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)
3245
- [1, 0].each do |i|
3246
- FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
3247
- end
3248
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
3249
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3250
-
3251
- # Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.
3252
- # (Note: Currently, there is no harm in duplicate calls)
3253
- sleep 4
3254
- end
3255
-
3256
- inode_0 = tail_watchers[0]&.ino
3257
- inode_1 = tail_watchers[1]&.ino
3258
- inode_2 = tail_watchers[2]&.ino
3259
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3260
- position_entries = []
3261
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3262
- f.readlines(chomp: true).each do |line|
3263
- values = line.split("\t")
3264
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3265
- end
3266
- end
3267
-
3268
- assert_equal(
3269
- {
3270
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
3271
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
3272
- tail_watcher_inodes: [inode_0, inode_1, inode_2],
3273
- tail_watcher_io_handler_opened_statuses: [false, false, false],
3274
- position_entries: [
3275
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
3276
- ],
3277
- },
3278
- {
3279
- record_values: record_values,
3280
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3281
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3282
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3283
- position_entries: position_entries
3284
- },
3285
- )
3286
- end
3287
- end
3288
- end