fluentd 1.17.0-x86-mingw32 → 1.17.1-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
@@ -1,3429 +0,0 @@
1
- require_relative '../helper'
2
- require 'fluent/test/driver/input'
3
- require 'fluent/plugin/in_tail'
4
- require 'fluent/plugin/buffer'
5
- require 'fluent/system_config'
6
- require 'fluent/file_wrapper'
7
- require 'net/http'
8
- require 'flexmock/test_unit'
9
- require 'timecop'
10
- require 'tmpdir'
11
- require 'securerandom'
12
-
13
- class TailInputTest < Test::Unit::TestCase
14
- include FlexMock::TestCase
15
-
16
- def tmp_dir
17
- File.join(File.dirname(__FILE__), "..", "tmp", "tail#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
18
- end
19
-
20
- def setup
21
- Fluent::Test.setup
22
- @tmp_dir = tmp_dir
23
- cleanup_directory(@tmp_dir)
24
- end
25
-
26
- def teardown
27
- super
28
- cleanup_directory(@tmp_dir)
29
- Fluent::Engine.stop
30
- Timecop.return
31
- end
32
-
33
- def cleanup_directory(path)
34
- unless Dir.exist?(path)
35
- FileUtils.mkdir_p(path)
36
- return
37
- end
38
-
39
- FileUtils.remove_entry_secure(path, true)
40
- end
41
-
42
- def cleanup_file(path)
43
- FileUtils.remove_entry_secure(path, true)
44
- end
45
-
46
- def create_target_info(path)
47
- Fluent::Plugin::TailInput::TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
48
- end
49
-
50
- ROOT_CONFIG = config_element("ROOT", "", {
51
- "tag" => "t1",
52
- "rotate_wait" => "2s",
53
- "refresh_interval" => "1s"
54
- })
55
-
56
- def base_config
57
- ROOT_CONFIG + config_element("", "", { "path" => "#{@tmp_dir}/tail.txt" })
58
- end
59
-
60
- def common_config
61
- base_config + config_element("", "", { "pos_file" => "#{@tmp_dir}/tail.pos" })
62
- end
63
-
64
- def common_follow_inode_config
65
- config_element("ROOT", "", {
66
- "path" => "#{@tmp_dir}/tail.txt*",
67
- "pos_file" => "#{@tmp_dir}/tail.pos",
68
- "tag" => "t1",
69
- "refresh_interval" => "1s",
70
- "read_from_head" => "true",
71
- "format" => "none",
72
- "rotate_wait" => "1s",
73
- "follow_inodes" => "true"
74
- })
75
- end
76
-
77
- CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
78
- CONFIG_DISABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
79
- CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
80
- CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
81
- SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
82
- PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
83
- MULTILINE_CONFIG = config_element(
84
- "", "", {
85
- "format" => "multiline",
86
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
87
- "format_firstline" => "/^[s]/"
88
- })
89
- PARSE_MULTILINE_CONFIG = config_element(
90
- "", "", {},
91
- [config_element("parse", "", {
92
- "@type" => "multiline",
93
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
94
- "format_firstline" => "/^[s]/"
95
- })
96
- ])
97
-
98
- MULTILINE_CONFIG_WITH_NEWLINE = config_element(
99
- "", "", {
100
- "format" => "multiline",
101
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.[^\\n]+))?/",
102
- "format_firstline" => "/^[s]/"
103
- })
104
- PARSE_MULTILINE_CONFIG_WITH_NEWLINE = config_element(
105
- "", "", {},
106
- [config_element("parse", "", {
107
- "@type" => "multiline",
108
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.[^\\n]+))?/",
109
- "format_firstline" => "/^[s]/"
110
- })
111
- ])
112
-
113
- EX_ROTATE_WAIT = 0
114
- EX_FOLLOW_INODES = false
115
-
116
- def ex_config
117
- config_element("", "", {
118
- "tag" => "tail",
119
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
120
- "format" => "none",
121
- "pos_file" => "#{@tmp_dir}/tail.pos",
122
- "read_from_head" => true,
123
- "refresh_interval" => 30,
124
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
125
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
126
- })
127
- end
128
-
129
- def tailing_group_pattern
130
- "/#{@tmp_dir}\/(?<podname>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\/[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container>.+)-(?<docker_id>[a-z0-9]{6})\.log$/"
131
- end
132
-
133
- DEBUG_LOG_LEVEL = config_element("", "", {
134
- "@log_level" => "debug"
135
- })
136
-
137
- def create_group_directive(pattern, rate_period, *rules)
138
- config_element("", "", {}, [
139
- config_element("group", "", {
140
- "pattern" => pattern,
141
- "rate_period" => rate_period
142
- }, rules)
143
- ])
144
- end
145
-
146
- def create_rule_directive(match_named_captures, limit)
147
- params = {
148
- "limit" => limit,
149
- "match" => match_named_captures,
150
- }
151
-
152
- config_element("rule", "", params)
153
- end
154
-
155
- def create_path_element(path)
156
- config_element("source", "", { "path" => "#{@tmp_dir}/#{path}" })
157
- end
158
-
159
- def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
160
- config = use_common_conf ? common_config + conf : conf
161
- Fluent::Test::Driver::Input.new(Fluent::Plugin::TailInput).configure(config)
162
- end
163
-
164
- sub_test_case "configure" do
165
- test "plain single line" do
166
- d = create_driver
167
- assert_equal(["#{@tmp_dir}/tail.txt"], d.instance.paths)
168
- assert_equal("t1", d.instance.tag)
169
- assert_equal(2, d.instance.rotate_wait)
170
- assert_equal("#{@tmp_dir}/tail.pos", d.instance.pos_file)
171
- assert_equal(1000, d.instance.read_lines_limit)
172
- assert_equal(-1, d.instance.read_bytes_limit_per_second)
173
- assert_equal(false, d.instance.ignore_repeated_permission_error)
174
- assert_nothing_raised do
175
- d.instance.have_read_capability?
176
- end
177
- end
178
-
179
- data("empty" => config_element,
180
- "w/o @type" => config_element("", "", {}, [config_element("parse", "", {})]))
181
- test "w/o parse section" do |conf|
182
- assert_raise(Fluent::ConfigError) do
183
- create_driver(conf)
184
- end
185
- end
186
-
187
- test "multi paths with path_delimiter" do
188
- c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "|" })
189
- d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
190
- assert_equal(["tail.txt", "test2", "tmp,dev"], d.instance.paths)
191
- end
192
-
193
- test "multi paths with same path configured twice" do
194
- c = config_element("ROOT", "", { "path" => "test1.txt,test2.txt,test1.txt", "tag" => "t1", "path_delimiter" => "," })
195
- d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
196
- assert_equal(["test2.txt","test1.txt"].sort, d.instance.paths.sort)
197
- end
198
-
199
- test "multi paths with invaid path_delimiter" do
200
- c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "*" })
201
- assert_raise(Fluent::ConfigError) do
202
- create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
203
- end
204
- end
205
-
206
- test "follow_inodes w/o pos file" do
207
- assert_raise(Fluent::ConfigError) do
208
- create_driver(base_config + config_element('', '', {'follow_inodes' => 'true'}))
209
- end
210
- end
211
-
212
- sub_test_case "log throttling per file" do
213
- test "w/o watcher timer is invalid" do
214
- conf = CONFIG_DISABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
215
- assert_raise(Fluent::ConfigError) do
216
- create_driver(conf)
217
- end
218
- end
219
-
220
- test "valid" do
221
- conf = config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
222
- assert_raise(Fluent::ConfigError) do
223
- create_driver(conf)
224
- end
225
- end
226
- end
227
-
228
- test "both enable_watch_timer and enable_stat_watcher are false" do
229
- assert_raise(Fluent::ConfigError) do
230
- create_driver(CONFIG_DISABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
231
- end
232
- end
233
-
234
- sub_test_case "encoding" do
235
- test "valid" do
236
- conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" })
237
- d = create_driver(conf)
238
- assert_equal(Encoding::UTF_8, d.instance.encoding)
239
- end
240
-
241
- test "invalid" do
242
- conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "no-such-encoding" })
243
- assert_raise(Fluent::ConfigError) do
244
- create_driver(conf)
245
- end
246
- end
247
- end
248
-
249
- sub_test_case "from_encoding" do
250
- test "only specified from_encoding raise ConfigError" do
251
- conf = SINGLE_LINE_CONFIG + config_element("", "", { "from_encoding" => "utf-8" })
252
- assert_raise(Fluent::ConfigError) do
253
- create_driver(conf)
254
- end
255
- end
256
-
257
- test "valid" do
258
- conf = SINGLE_LINE_CONFIG + config_element("", "", {
259
- "from_encoding" => "utf-8",
260
- "encoding" => "utf-8"
261
- })
262
- d = create_driver(conf)
263
- assert_equal(Encoding::UTF_8, d.instance.from_encoding)
264
- end
265
-
266
- test "invalid" do
267
- conf = SINGLE_LINE_CONFIG + config_element("", "", {
268
- "from_encoding" => "no-such-encoding",
269
- "encoding" => "utf-8"
270
- })
271
- assert_raise(Fluent::ConfigError) do
272
- create_driver(conf)
273
- end
274
- end
275
- end
276
- end
277
-
278
- sub_test_case "configure group" do
279
- test "<rule> required" do
280
- conf = create_group_directive('.', '1m') + SINGLE_LINE_CONFIG
281
- assert_raise(Fluent::ConfigError) do
282
- create_driver(conf)
283
- end
284
- end
285
-
286
- test "valid configuration" do
287
- rule1 = create_rule_directive({
288
- "namespace"=> "/namespace-a/",
289
- "podname"=> "/podname-[b|c]/"
290
- }, 100)
291
- rule2 = create_rule_directive({
292
- "namespace"=> "/namespace-[d|e]/",
293
- "podname"=> "/podname-f/",
294
- }, 50)
295
- rule3 = create_rule_directive({
296
- "podname"=> "/podname-g/",
297
- }, -1)
298
- rule4 = create_rule_directive({
299
- "namespace"=> "/namespace-h/",
300
- }, 0)
301
-
302
- conf = create_group_directive(tailing_group_pattern, '1m', rule1, rule2, rule3, rule4) + SINGLE_LINE_CONFIG
303
- assert_nothing_raised do
304
- create_driver(conf)
305
- end
306
- end
307
-
308
- test "limit should be greater than DEFAULT_LIMIT (-1)" do
309
- rule1 = create_rule_directive({
310
- "namespace"=> "/namespace-a/",
311
- "podname"=> "/podname-[b|c]/",
312
- }, -100)
313
- rule2 = create_rule_directive({
314
- "namespace"=> "/namespace-[d|e]/",
315
- "podname"=> "/podname-f/",
316
- }, 50)
317
- conf = create_group_directive(tailing_group_pattern, '1m', rule1, rule2) + SINGLE_LINE_CONFIG
318
- assert_raise(RuntimeError) do
319
- create_driver(conf)
320
- end
321
- end
322
- end
323
-
324
- sub_test_case "group rules line limit resolution" do
325
- test "valid" do
326
- rule1 = create_rule_directive({
327
- "namespace"=> "/namespace-a/",
328
- "podname"=> "/podname-[b|c]/",
329
- }, 50)
330
- rule2 = create_rule_directive({
331
- "podname"=> "/podname-[b|c]/",
332
- }, 400)
333
- rule3 = create_rule_directive({
334
- "namespace"=> "/namespace-a/",
335
- }, 100)
336
-
337
- conf = create_group_directive(tailing_group_pattern, '1m', rule3, rule1, rule2) + SINGLE_LINE_CONFIG
338
- assert_nothing_raised do
339
- d = create_driver(conf)
340
- instance = d.instance
341
-
342
- metadata = {
343
- "namespace"=> "namespace-a",
344
- "podname"=> "podname-b",
345
- }
346
- assert_equal(50, instance.find_group(metadata).limit)
347
-
348
- metadata = {
349
- "namespace" => "namespace-a",
350
- "podname" => "podname-c",
351
- }
352
- assert_equal(50, instance.find_group(metadata).limit)
353
-
354
- metadata = {
355
- "namespace" => "namespace-a",
356
- "podname" => "podname-d",
357
- }
358
- assert_equal(100, instance.find_group(metadata).limit)
359
-
360
- metadata = {
361
- "namespace" => "namespace-f",
362
- "podname" => "podname-b",
363
- }
364
- assert_equal(400, instance.find_group(metadata).limit)
365
-
366
- metadata = {
367
- "podname" => "podname-c",
368
- }
369
- assert_equal(400, instance.find_group(metadata).limit)
370
-
371
- assert_equal(-1, instance.find_group({}).limit)
372
- end
373
- end
374
- end
375
-
376
- sub_test_case "files should be placed in groups" do
377
- test "invalid regex pattern places files in default group" do
378
- rule1 = create_rule_directive({}, 100) ## limits default groups
379
- conf = ROOT_CONFIG + DEBUG_LOG_LEVEL + create_group_directive(tailing_group_pattern, '1m', rule1) + create_path_element("test*.txt") + SINGLE_LINE_CONFIG
380
-
381
- d = create_driver(conf, false)
382
- Fluent::FileWrapper.open("#{@tmp_dir}/test1.txt", 'w')
383
- Fluent::FileWrapper.open("#{@tmp_dir}/test2.txt", 'w')
384
- Fluent::FileWrapper.open("#{@tmp_dir}/test3.txt", 'w')
385
-
386
- d.run do
387
- ## checking default group_watcher's paths
388
- instance = d.instance
389
- key = instance.default_group_key
390
-
391
- assert_equal(3, instance.log.logs.count{|a| a.match?("Cannot find group from metadata, Adding file in the default group\n")})
392
- assert_equal(3, instance.group_watchers[key].size)
393
- assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test1.txt'))
394
- assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test2.txt'))
395
- assert_true(instance.group_watchers[key].include? File.join(@tmp_dir, 'test3.txt'))
396
- end
397
- end
398
-
399
- test "valid regex pattern places file in their respective groups" do
400
- rule1 = create_rule_directive({
401
- "namespace"=> "/test-namespace1/",
402
- "podname"=> "/test-podname1/",
403
- }, 100)
404
- rule2 = create_rule_directive({
405
- "namespace"=> "/test-namespace1/",
406
- }, 200)
407
- rule3 = create_rule_directive({
408
- "podname"=> "/test-podname2/",
409
- }, 300)
410
- rule4 = create_rule_directive({}, 400)
411
-
412
- path_element = create_path_element("test-podname*.log")
413
-
414
- conf = ROOT_CONFIG + create_group_directive(tailing_group_pattern, '1m', rule4, rule3, rule2, rule1) + path_element + SINGLE_LINE_CONFIG
415
- d = create_driver(conf, false)
416
-
417
- file1 = File.join(@tmp_dir, "test-podname1_test-namespace1_test-container-15fabq.log")
418
- file2 = File.join(@tmp_dir, "test-podname3_test-namespace1_test-container-15fabq.log")
419
- file3 = File.join(@tmp_dir, "test-podname2_test-namespace2_test-container-15fabq.log")
420
- file4 = File.join(@tmp_dir, "test-podname4_test-namespace3_test-container-15fabq.log")
421
-
422
- d.run do
423
- Fluent::FileWrapper.open(file1, 'w')
424
- Fluent::FileWrapper.open(file2, 'w')
425
- Fluent::FileWrapper.open(file3, 'w')
426
- Fluent::FileWrapper.open(file4, 'w')
427
-
428
- instance = d.instance
429
- assert_equal(100, instance.find_group_from_metadata(file1).limit)
430
- assert_equal(200, instance.find_group_from_metadata(file2).limit)
431
- assert_equal(300, instance.find_group_from_metadata(file3).limit)
432
- assert_equal(400, instance.find_group_from_metadata(file4).limit)
433
- end
434
- end
435
- end
436
-
437
- sub_test_case "singleline" do
438
- data(flat: SINGLE_LINE_CONFIG,
439
- parse: PARSE_SINGLE_LINE_CONFIG)
440
- def test_emit(data)
441
- config = data
442
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
443
- f.puts "test1"
444
- f.puts "test2"
445
- }
446
-
447
- d = create_driver(config)
448
-
449
- d.run(expect_emits: 1) do
450
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
451
- f.puts "test3\ntest4"
452
- }
453
- end
454
-
455
- events = d.events
456
- assert_equal(true, events.length > 0)
457
- assert_equal({"message" => "test3"}, events[0][2])
458
- assert_equal({"message" => "test4"}, events[1][2])
459
- assert(events[0][1].is_a?(Fluent::EventTime))
460
- assert(events[1][1].is_a?(Fluent::EventTime))
461
- assert_equal(1, d.emit_count)
462
- end
463
-
464
- def test_emit_with_emit_unmatched_lines_true
465
- config = config_element("", "", { "format" => "/^(?<message>test.*)/", "emit_unmatched_lines" => true })
466
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
467
-
468
- d = create_driver(config)
469
- d.run(expect_emits: 1) do
470
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
471
- f.puts "test line 1"
472
- f.puts "test line 2"
473
- f.puts "bad line 1"
474
- f.puts "test line 3"
475
- }
476
- end
477
-
478
- events = d.events
479
- assert_equal(4, events.length)
480
- assert_equal({"message" => "test line 1"}, events[0][2])
481
- assert_equal({"message" => "test line 2"}, events[1][2])
482
- assert_equal({"unmatched_line" => "bad line 1"}, events[2][2])
483
- assert_equal({"message" => "test line 3"}, events[3][2])
484
- end
485
-
486
- data('flat 1' => [:flat, 1, 2],
487
- 'flat 10' => [:flat, 10, 1],
488
- 'parse 1' => [:parse, 1, 2],
489
- 'parse 10' => [:parse, 10, 1])
490
- def test_emit_with_read_lines_limit(data)
491
- config_style, limit, num_events = data
492
- case config_style
493
- when :flat
494
- config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit })
495
- when :parse
496
- config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit }) + PARSE_SINGLE_LINE_CONFIG
497
- end
498
- d = create_driver(config)
499
- msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
500
-
501
- d.run(expect_emits: num_events, timeout: 2) do
502
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
503
- f.puts msg
504
- f.puts msg
505
- }
506
- end
507
-
508
- events = d.events
509
- assert_equal(true, events.length > 0)
510
- assert_equal({"message" => msg}, events[0][2])
511
- assert_equal({"message" => msg}, events[1][2])
512
- assert num_events <= d.emit_count
513
- end
514
-
515
- sub_test_case "log throttling per file" do
516
- teardown do
517
- cleanup_file("#{@tmp_dir}/tail.txt")
518
- end
519
-
520
- sub_test_case "reads_bytes_per_second w/o throttled" do
521
- data("flat 8192 bytes, 2 events" => [:flat, 100, 8192, 2],
522
- "flat 8192 bytes, 2 events w/o stat watcher" => [:flat_without_stat, 100, 8192, 2],
523
- "flat #{8192*10} bytes, 20 events" => [:flat, 100, (8192 * 10), 20],
524
- "flat #{8192*10} bytes, 20 events w/o stat watcher" => [:flat_without_stat, 100, (8192 * 10), 20],
525
- "parse #{8192*4} bytes, 8 events" => [:parse, 100, (8192 * 4), 8],
526
- "parse #{8192*4} bytes, 8 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 4), 8],
527
- "parse #{8192*10} bytes, 20 events" => [:parse, 100, (8192 * 10), 20],
528
- "parse #{8192*10} bytes, 20 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 10), 20],
529
- "flat 8k bytes with unit, 2 events" => [:flat, 100, "8k", 2])
530
- def test_emit_with_read_bytes_limit_per_second(data)
531
- config_style, limit, limit_bytes, num_events = data
532
- case config_style
533
- when :flat
534
- config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
535
- when :parse
536
- config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
537
- when :flat_without_stat
538
- config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
539
- when :parse_without_stat
540
- config = CONFIG_READ_FROM_HEAD + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
541
- end
542
-
543
- msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
544
- start_time = Fluent::Clock.now
545
-
546
- d = create_driver(config)
547
- d.run(expect_emits: 2) do
548
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
549
- 100.times do
550
- f.puts msg
551
- end
552
- }
553
- end
554
-
555
- assert_true(Fluent::Clock.now - start_time > 1)
556
- assert_equal(num_events.times.map { {"message" => msg} },
557
- d.events.collect { |event| event[2] })
558
- end
559
-
560
- def test_read_bytes_limit_precede_read_lines_limit
561
- config = CONFIG_READ_FROM_HEAD +
562
- SINGLE_LINE_CONFIG +
563
- config_element("", "", {
564
- "read_lines_limit" => 1000,
565
- "read_bytes_limit_per_second" => 8192
566
- })
567
- msg = 'abc'
568
- start_time = Fluent::Clock.now
569
- d = create_driver(config)
570
- d.run(expect_emits: 2) do
571
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
572
- 8000.times do
573
- f.puts msg
574
- end
575
- }
576
- end
577
-
578
- assert_true(Fluent::Clock.now - start_time > 1)
579
- assert_equal(4096.times.map { {"message" => msg} },
580
- d.events.collect { |event| event[2] })
581
- end
582
- end
583
-
584
- sub_test_case "reads_bytes_per_second w/ throttled already" do
585
- data("flat 8192 bytes" => [:flat, 100, 8192],
586
- "parse 8192 bytes" => [:parse, 100, 8192])
587
- def test_emit_with_read_bytes_limit_per_second(data)
588
- config_style, limit, limit_bytes = data
589
- case config_style
590
- when :flat
591
- config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
592
- when :parse
593
- config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
594
- end
595
- d = create_driver(config)
596
- msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
597
-
598
- mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
599
- require 'fluent/config/types'
600
- limit_bytes_value = Fluent::Config.size_value(limit_bytes)
601
- io_handler.instance_variable_set(:@number_bytes_read, limit_bytes_value)
602
- if Fluent.linux?
603
- mock.proxy(io_handler).handle_notify.at_least(5)
604
- else
605
- mock.proxy(io_handler).handle_notify.twice
606
- end
607
- io_handler
608
- end
609
-
610
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") do |f|
611
- 100.times do
612
- f.puts msg
613
- end
614
- end
615
-
616
- # We should not do shutdown here due to hard timeout.
617
- d.run do
618
- start_time = Fluent::Clock.now
619
- while Fluent::Clock.now - start_time < 0.8 do
620
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") do |f|
621
- f.puts msg
622
- f.flush
623
- end
624
- sleep 0.05
625
- end
626
- end
627
-
628
- assert_equal([], d.events)
629
- end
630
- end
631
-
632
- sub_test_case "EOF with reads_bytes_per_second" do
633
- def test_longer_than_rotate_wait
634
- limit_bytes = 8192
635
- num_lines = 1024 * 3
636
- msg = "08bytes"
637
-
638
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") do |f|
639
- f.write("#{msg}\n" * num_lines)
640
- end
641
-
642
- config = CONFIG_READ_FROM_HEAD +
643
- SINGLE_LINE_CONFIG +
644
- config_element("", "", {
645
- "read_bytes_limit_per_second" => limit_bytes,
646
- "rotate_wait" => 0.1,
647
- "refresh_interval" => 0.5,
648
- })
649
-
650
- rotated = false
651
- d = create_driver(config)
652
- d.run(timeout: 10) do
653
- while d.events.size < num_lines do
654
- if d.events.size > 0 && !rotated
655
- cleanup_file("#{@tmp_dir}/tail.txt")
656
- FileUtils.touch("#{@tmp_dir}/tail.txt")
657
- rotated = true
658
- end
659
- sleep 0.3
660
- end
661
- end
662
-
663
- assert_equal(num_lines,
664
- d.events.count do |event|
665
- event[2]["message"] == msg
666
- end)
667
- end
668
-
669
- def test_shorter_than_rotate_wait
670
- limit_bytes = 8192
671
- num_lines = 1024 * 2
672
- msg = "08bytes"
673
-
674
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") do |f|
675
- f.write("#{msg}\n" * num_lines)
676
- end
677
-
678
- config = CONFIG_READ_FROM_HEAD +
679
- SINGLE_LINE_CONFIG +
680
- config_element("", "", {
681
- "read_bytes_limit_per_second" => limit_bytes,
682
- "rotate_wait" => 2,
683
- "refresh_interval" => 0.5,
684
- })
685
-
686
- start_time = Fluent::Clock.now
687
- rotated = false
688
- detached = false
689
- d = create_driver(config)
690
- mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
691
- mock.proxy(tw).detach(anything) do |v|
692
- detached = true
693
- v
694
- end
695
- tw
696
- end.twice
697
-
698
- d.run(timeout: 10) do
699
- until detached do
700
- if d.events.size > 0 && !rotated
701
- cleanup_file("#{@tmp_dir}/tail.txt")
702
- FileUtils.touch("#{@tmp_dir}/tail.txt")
703
- rotated = true
704
- end
705
- sleep 0.3
706
- end
707
- end
708
-
709
- assert_true(Fluent::Clock.now - start_time > 2)
710
- assert_equal(num_lines,
711
- d.events.count do |event|
712
- event[2]["message"] == msg
713
- end)
714
- end
715
- end
716
- end
717
-
718
- data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
719
- parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
720
- def test_emit_with_read_from_head(data)
721
- config = data
722
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
723
- f.puts "test1"
724
- f.puts "test2"
725
- }
726
-
727
- d = create_driver(config)
728
-
729
- d.run(expect_emits: 2) do
730
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
731
- f.puts "test3"
732
- f.puts "test4"
733
- }
734
- end
735
-
736
- events = d.events
737
- assert(events.length > 0)
738
- assert_equal({"message" => "test1"}, events[0][2])
739
- assert_equal({"message" => "test2"}, events[1][2])
740
- assert_equal({"message" => "test3"}, events[2][2])
741
- assert_equal({"message" => "test4"}, events[3][2])
742
- end
743
-
744
- data(flat: CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG,
745
- parse: CONFIG_DISABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG)
746
- def test_emit_without_watch_timer(data)
747
- config = data
748
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
749
- f.puts "test1"
750
- f.puts "test2"
751
- }
752
-
753
- d = create_driver(config)
754
-
755
- d.run(expect_emits: 1) do
756
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
757
- f.puts "test3"
758
- f.puts "test4"
759
- }
760
- # according to cool.io's stat_watcher.c, systems without inotify will use
761
- # an "automatic" value, typically around 5 seconds
762
- end
763
-
764
- events = d.events
765
- assert(events.length > 0)
766
- assert_equal({"message" => "test3"}, events[0][2])
767
- assert_equal({"message" => "test4"}, events[1][2])
768
- end
769
-
770
- # https://github.com/fluent/fluentd/pull/3541#discussion_r740197711
771
- def test_watch_wildcard_path_without_watch_timer
772
- omit "need inotify" unless Fluent.linux?
773
-
774
- config = config_element("ROOT", "", {
775
- "path" => "#{@tmp_dir}/tail*.txt",
776
- "tag" => "t1",
777
- })
778
- config = config + CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG
779
-
780
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
781
- f.puts "test1"
782
- f.puts "test2"
783
- }
784
-
785
- d = create_driver(config, false)
786
-
787
- d.run(expect_emits: 1, timeout: 1) do
788
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
789
- f.puts "test3"
790
- f.puts "test4"
791
- }
792
- end
793
-
794
- assert_equal(
795
- [
796
- {"message" => "test3"},
797
- {"message" => "test4"},
798
- ],
799
- d.events.collect { |event| event[2] })
800
- end
801
-
802
- data(flat: CONFIG_DISABLE_STAT_WATCHER + SINGLE_LINE_CONFIG,
803
- parse: CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
804
- def test_emit_with_disable_stat_watcher(data)
805
- config = data
806
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
807
- f.puts "test1"
808
- f.puts "test2"
809
- }
810
-
811
- d = create_driver(config)
812
-
813
- d.run(expect_emits: 1) do
814
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
815
- f.puts "test3"
816
- f.puts "test4"
817
- }
818
- end
819
-
820
- events = d.events
821
- assert(events.length > 0)
822
- assert_equal({"message" => "test3"}, events[0][2])
823
- assert_equal({"message" => "test4"}, events[1][2])
824
- end
825
-
826
- def test_always_read_from_head_on_detecting_a_new_file
827
- d = create_driver(SINGLE_LINE_CONFIG)
828
-
829
- d.run(expect_emits: 1, timeout: 3) do
830
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
831
- f.puts "test1\ntest2\n"
832
- }
833
- end
834
-
835
- assert_equal(
836
- [
837
- {"message" => "test1"},
838
- {"message" => "test2"},
839
- ],
840
- d.events.collect { |event| event[2] })
841
- end
842
- end
843
-
844
- class TestWithSystem < self
845
- include Fluent::SystemConfig::Mixin
846
-
847
- OVERRIDE_FILE_PERMISSION = 0620
848
- CONFIG_SYSTEM = %[
849
- <system>
850
- file_permission #{OVERRIDE_FILE_PERMISSION}
851
- </system>
852
- ]
853
-
854
- def setup
855
- omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
856
- super
857
- # Store default permission
858
- @default_permission = system_config.instance_variable_get(:@file_permission)
859
- end
860
-
861
- def teardown
862
- return if Fluent.windows?
863
- super
864
- # Restore default permission
865
- system_config.instance_variable_set(:@file_permission, @default_permission)
866
- end
867
-
868
- def parse_system(text)
869
- basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
870
- Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
871
- end
872
-
873
- def test_emit_with_system
874
- system_conf = parse_system(CONFIG_SYSTEM)
875
- sc = Fluent::SystemConfig.new(system_conf)
876
- Fluent::Engine.init(sc)
877
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
878
- f.puts "test1"
879
- f.puts "test2"
880
- }
881
-
882
- d = create_driver
883
-
884
- d.run(expect_emits: 1) do
885
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
886
- f.puts "test3"
887
- f.puts "test4"
888
- }
889
- end
890
-
891
- events = d.events
892
- assert_equal(true, events.length > 0)
893
- assert_equal({"message" => "test3"}, events[0][2])
894
- assert_equal({"message" => "test4"}, events[1][2])
895
- assert(events[0][1].is_a?(Fluent::EventTime))
896
- assert(events[1][1].is_a?(Fluent::EventTime))
897
- assert_equal(1, d.emit_count)
898
- pos = d.instance.instance_variable_get(:@pf_file)
899
- mode = "%o" % File.stat(pos).mode
900
- assert_equal OVERRIDE_FILE_PERMISSION, mode[-3, 3].to_i
901
- end
902
- end
903
-
904
- sub_test_case "rotate file" do
905
- def create_driver(conf = SINGLE_LINE_CONFIG)
906
- config = common_config + conf
907
- Fluent::Test::Driver::Input.new(Fluent::Plugin::TailInput).configure(config)
908
- end
909
-
910
- data(flat: SINGLE_LINE_CONFIG,
911
- parse: PARSE_SINGLE_LINE_CONFIG)
912
- def test_rotate_file(data)
913
- config = data
914
- events = sub_test_rotate_file(config, expect_emits: 2)
915
- assert_equal(3.upto(6).collect { |i| {"message" => "test#{i}"} },
916
- events.collect { |event| event[2] })
917
- end
918
-
919
- data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
920
- parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
921
- def test_rotate_file_with_read_from_head(data)
922
- config = data
923
- events = sub_test_rotate_file(config, expect_records: 6)
924
- assert_equal(1.upto(6).collect { |i| {"message" => "test#{i}"} },
925
- events.collect { |event| event[2] })
926
- end
927
-
928
- data(flat: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
929
- parse: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
930
- def test_rotate_file_with_open_on_every_update(data)
931
- config = data
932
- events = sub_test_rotate_file(config, expect_records: 6)
933
- assert_equal(1.upto(6).collect { |i| {"message" => "test#{i}"} },
934
- events.collect { |event| event[2] })
935
- end
936
-
937
- data(flat: SINGLE_LINE_CONFIG,
938
- parse: PARSE_SINGLE_LINE_CONFIG)
939
- def test_rotate_file_with_write_old(data)
940
- config = data
941
- events = sub_test_rotate_file(config, expect_emits: 3) { |rotated_file|
942
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
943
- rotated_file.puts "test7"
944
- rotated_file.puts "test8"
945
- rotated_file.flush
946
-
947
- sleep 1
948
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
949
- f.puts "test5"
950
- f.puts "test6"
951
- }
952
- }
953
- # This test sometimes fails and it shows a potential bug of in_tail
954
- # https://github.com/fluent/fluentd/issues/1434
955
- assert_equal([3, 4, 7, 8, 5, 6].collect { |i| {"message" => "test#{i}"} },
956
- events.collect { |event| event[2] })
957
- end
958
-
959
- data(flat: SINGLE_LINE_CONFIG,
960
- parse: PARSE_SINGLE_LINE_CONFIG)
961
- def test_rotate_file_with_write_old_and_no_new_file(data)
962
- config = data
963
- events = sub_test_rotate_file(config, expect_emits: 2) { |rotated_file|
964
- rotated_file.puts "test7"
965
- rotated_file.puts "test8"
966
- rotated_file.flush
967
- }
968
- assert_equal([3, 4, 7, 8].collect { |i| {"message" => "test#{i}"} },
969
- events.collect { |event| event[2] })
970
- end
971
-
972
- def sub_test_rotate_file(config = nil, expect_emits: nil, expect_records: nil, timeout: 5)
973
- file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb")
974
- file.puts "test1"
975
- file.puts "test2"
976
- file.flush
977
-
978
- d = create_driver(config)
979
- d.run(expect_emits: expect_emits, expect_records: expect_records, timeout: timeout) do
980
- sleep(0.1) while d.instance.instance_variable_get(:@startup)
981
- size = d.emit_count
982
- file.puts "test3"
983
- file.puts "test4"
984
- file.flush
985
- sleep(0.1) until d.emit_count >= size + 1
986
- size = d.emit_count
987
-
988
- if Fluent.windows?
989
- file.close
990
- FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt", force: true)
991
- file = Fluent::FileWrapper.open("#{@tmp_dir}/tail2.txt", "ab")
992
- else
993
- FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt")
994
- end
995
- if block_given?
996
- yield file
997
- else
998
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
999
- sleep 1
1000
-
1001
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1002
- f.puts "test5"
1003
- f.puts "test6"
1004
- }
1005
- end
1006
- end
1007
-
1008
- d.events
1009
- ensure
1010
- file.close if file && !file.closed?
1011
- end
1012
- end
1013
-
1014
- def test_truncate_file
1015
- config = SINGLE_LINE_CONFIG
1016
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1017
- f.puts "test1"
1018
- f.puts "test2"
1019
- f.flush
1020
- }
1021
-
1022
- d = create_driver(config)
1023
-
1024
- d.run(expect_emits: 2) do
1025
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1026
- f.puts "test3\ntest4"
1027
- f.flush
1028
- }
1029
- waiting(2) { sleep 0.1 until d.events.length == 2 }
1030
- if Fluent.windows?
1031
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f|
1032
- f.puts("test1");
1033
- }
1034
- else
1035
- File.truncate("#{@tmp_dir}/tail.txt", 6)
1036
- end
1037
- end
1038
-
1039
- expected = {
1040
- emit_count: 2,
1041
- events: [
1042
- [Fluent::EventTime, {"message" => "test3"}],
1043
- [Fluent::EventTime, {"message" => "test4"}],
1044
- [Fluent::EventTime, {"message" => "test1"}],
1045
- ]
1046
- }
1047
- actual = {
1048
- emit_count: d.emit_count,
1049
- events: d.events.collect{|event| [event[1].class, event[2]]}
1050
- }
1051
- assert_equal(expected, actual)
1052
- end
1053
-
1054
- def test_move_truncate_move_back
1055
- config = SINGLE_LINE_CONFIG
1056
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1057
- f.puts "test1"
1058
- f.puts "test2"
1059
- }
1060
-
1061
- d = create_driver(config)
1062
-
1063
- d.run(expect_emits: 1) do
1064
- if Fluent.windows?
1065
- FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt", force: true)
1066
- else
1067
- FileUtils.mv("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail2.txt")
1068
- end
1069
- sleep(1)
1070
- if Fluent.windows?
1071
- Fluent::FileWrapper.open("#{@tmp_dir}/tail2.txt", "wb") { |f|
1072
- f.puts("test1");
1073
- }
1074
- else
1075
- File.truncate("#{@tmp_dir}/tail2.txt", 6)
1076
- end
1077
- sleep(1)
1078
- if Fluent.windows?
1079
- FileUtils.mv("#{@tmp_dir}/tail2.txt", "#{@tmp_dir}/tail.txt", force: true)
1080
- else
1081
- FileUtils.mv("#{@tmp_dir}/tail2.txt", "#{@tmp_dir}/tail.txt")
1082
- end
1083
- end
1084
-
1085
- events = d.events
1086
- assert_equal(1, events.length)
1087
- assert_equal({"message" => "test1"}, events[0][2])
1088
- assert(events[0][1].is_a?(Fluent::EventTime))
1089
- assert_equal(1, d.emit_count)
1090
- end
1091
-
1092
- def test_lf
1093
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| }
1094
-
1095
- d = create_driver
1096
-
1097
- d.run(expect_emits: 1) do
1098
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1099
- f.print "test3"
1100
- }
1101
- sleep 1
1102
-
1103
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1104
- f.puts "test4"
1105
- }
1106
- end
1107
-
1108
- events = d.events
1109
- assert_equal(true, events.length > 0)
1110
- assert_equal({"message" => "test3test4"}, events[0][2])
1111
- end
1112
-
1113
- def test_whitespace
1114
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| }
1115
-
1116
- d = create_driver
1117
-
1118
- d.run(expect_emits: 1) do
1119
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
1120
- f.puts " " # 4 spaces
1121
- f.puts " 4 spaces"
1122
- f.puts "4 spaces "
1123
- f.puts " " # tab
1124
- f.puts " tab"
1125
- f.puts "tab "
1126
- }
1127
- end
1128
-
1129
- events = d.events
1130
- assert_equal(true, events.length > 0)
1131
- assert_equal({"message" => " "}, events[0][2])
1132
- assert_equal({"message" => " 4 spaces"}, events[1][2])
1133
- assert_equal({"message" => "4 spaces "}, events[2][2])
1134
- assert_equal({"message" => " "}, events[3][2])
1135
- assert_equal({"message" => " tab"}, events[4][2])
1136
- assert_equal({"message" => "tab "}, events[5][2])
1137
- end
1138
-
1139
- data(
1140
- 'flat default encoding' => [SINGLE_LINE_CONFIG, Encoding::ASCII_8BIT],
1141
- 'flat explicit encoding config' => [SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8],
1142
- 'parse default encoding' => [PARSE_SINGLE_LINE_CONFIG, Encoding::ASCII_8BIT],
1143
- 'parse explicit encoding config' => [PARSE_SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8])
1144
- def test_encoding(data)
1145
- encoding_config, encoding = data
1146
-
1147
- d = create_driver(CONFIG_READ_FROM_HEAD + encoding_config)
1148
-
1149
- d.run(expect_emits: 1) do
1150
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
1151
- f.puts "test"
1152
- }
1153
- end
1154
-
1155
- events = d.events
1156
- assert_equal(encoding, events[0][2]['message'].encoding)
1157
- end
1158
-
1159
- def test_from_encoding
1160
- conf = config_element(
1161
- "", "", {
1162
- "format" => "/(?<message>.*)/",
1163
- "read_from_head" => "true",
1164
- "from_encoding" => "cp932",
1165
- "encoding" => "utf-8"
1166
- })
1167
- d = create_driver(conf)
1168
- cp932_message = "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
1169
- utf8_message = cp932_message.encode(Encoding::UTF_8)
1170
-
1171
- d.run(expect_emits: 1) do
1172
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w:cp932") {|f|
1173
- f.puts cp932_message
1174
- }
1175
- end
1176
-
1177
- events = d.events
1178
- assert_equal(utf8_message, events[0][2]['message'])
1179
- assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
1180
- end
1181
-
1182
- def test_from_encoding_utf16
1183
- conf = config_element(
1184
- "", "", {
1185
- "format" => "/(?<message>.*)/",
1186
- "read_from_head" => "true",
1187
- "from_encoding" => "utf-16le",
1188
- "encoding" => "utf-8"
1189
- })
1190
- d = create_driver(conf)
1191
- utf16_message = "\u306F\u308D\u30FC\u308F\u30FC\u308B\u3069\n".encode(Encoding::UTF_16LE)
1192
- utf8_message = utf16_message.encode(Encoding::UTF_8).strip
1193
-
1194
- d.run(expect_emits: 1) do
1195
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w:utf-16le") { |f|
1196
- f.write utf16_message
1197
- }
1198
- end
1199
-
1200
- events = d.events
1201
- assert_equal(utf8_message, events[0][2]['message'])
1202
- assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
1203
- end
1204
-
1205
- def test_encoding_with_bad_character
1206
- conf = config_element(
1207
- "", "", {
1208
- "format" => "/(?<message>.*)/",
1209
- "read_from_head" => "true",
1210
- "from_encoding" => "ASCII-8BIT",
1211
- "encoding" => "utf-8"
1212
- })
1213
- d = create_driver(conf)
1214
-
1215
- d.run(expect_emits: 1) do
1216
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w") { |f|
1217
- f.write "te\x86st\n"
1218
- }
1219
- end
1220
-
1221
- events = d.events
1222
- assert_equal("te\uFFFDst", events[0][2]['message'])
1223
- assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
1224
- end
1225
-
1226
- sub_test_case "multiline" do
1227
- data(flat: MULTILINE_CONFIG,
1228
- parse: PARSE_MULTILINE_CONFIG)
1229
- def test_multiline(data)
1230
- config = data
1231
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1232
-
1233
- d = create_driver(config)
1234
- d.run(expect_emits: 1) do
1235
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1236
- f.puts "f test1"
1237
- f.puts "s test2"
1238
- f.puts "f test3"
1239
- f.puts "f test4"
1240
- f.puts "s test5"
1241
- f.puts "s test6"
1242
- f.puts "f test7"
1243
- f.puts "s test8"
1244
- }
1245
- end
1246
-
1247
- events = d.events
1248
- assert_equal(4, events.length)
1249
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
1250
- assert_equal({"message1" => "test5"}, events[1][2])
1251
- assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
1252
- assert_equal({"message1" => "test8"}, events[3][2])
1253
- end
1254
-
1255
- data(flat: MULTILINE_CONFIG,
1256
- parse: PARSE_MULTILINE_CONFIG)
1257
- def test_multiline_with_emit_unmatched_lines_true(data)
1258
- config = data + config_element("", "", { "emit_unmatched_lines" => true })
1259
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1260
-
1261
- d = create_driver(config)
1262
- d.run(expect_emits: 1) do
1263
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1264
- f.puts "f test1"
1265
- f.puts "s test2"
1266
- f.puts "f test3"
1267
- f.puts "f test4"
1268
- f.puts "s test5"
1269
- f.puts "s test6"
1270
- f.puts "f test7"
1271
- f.puts "s test8"
1272
- }
1273
- end
1274
-
1275
- events = d.events
1276
- assert_equal(5, events.length)
1277
- assert_equal({"unmatched_line" => "f test1"}, events[0][2])
1278
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[1][2])
1279
- assert_equal({"message1" => "test5"}, events[2][2])
1280
- assert_equal({"message1" => "test6", "message2" => "test7"}, events[3][2])
1281
- assert_equal({"message1" => "test8"}, events[4][2])
1282
- end
1283
-
1284
- data(
1285
- flat: MULTILINE_CONFIG_WITH_NEWLINE,
1286
- parse: PARSE_MULTILINE_CONFIG_WITH_NEWLINE)
1287
- def test_multiline_with_emit_unmatched_lines2(data)
1288
- config = data + config_element("", "", { "emit_unmatched_lines" => true })
1289
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1290
-
1291
- d = create_driver(config)
1292
- d.run(expect_emits: 0, timeout: 1) do
1293
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1294
- f.puts "s test0"
1295
- f.puts "f test1"
1296
- f.puts "f test2"
1297
-
1298
- f.puts "f test3"
1299
-
1300
- f.puts "s test4"
1301
- f.puts "f test5"
1302
- f.puts "f test6"
1303
- }
1304
- end
1305
-
1306
- events = d.events
1307
- assert_equal({"message1" => "test0", "message2" => "test1", "message3" => "test2"}, events[0][2])
1308
- assert_equal({ 'unmatched_line' => "f test3" }, events[1][2])
1309
- assert_equal({"message1" => "test4", "message2" => "test5", "message3" => "test6"}, events[2][2])
1310
- end
1311
-
1312
- data(flat: MULTILINE_CONFIG,
1313
- parse: PARSE_MULTILINE_CONFIG)
1314
- def test_multiline_with_flush_interval(data)
1315
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1316
-
1317
- config = data + config_element("", "", { "multiline_flush_interval" => "2s" })
1318
- d = create_driver(config)
1319
-
1320
- assert_equal(2, d.instance.multiline_flush_interval)
1321
-
1322
- d.run(expect_emits: 1) do
1323
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1324
- f.puts "f test1"
1325
- f.puts "s test2"
1326
- f.puts "f test3"
1327
- f.puts "f test4"
1328
- f.puts "s test5"
1329
- f.puts "s test6"
1330
- f.puts "f test7"
1331
- f.puts "s test8"
1332
- }
1333
- end
1334
-
1335
- events = d.events
1336
- assert_equal(4, events.length)
1337
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
1338
- assert_equal({"message1" => "test5"}, events[1][2])
1339
- assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
1340
- assert_equal({"message1" => "test8"}, events[3][2])
1341
- end
1342
-
1343
- data(
1344
- 'flat default encoding' => [MULTILINE_CONFIG, Encoding::ASCII_8BIT],
1345
- 'flat explicit encoding config' => [MULTILINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8],
1346
- 'parse default encoding' => [PARSE_MULTILINE_CONFIG, Encoding::ASCII_8BIT],
1347
- 'parse explicit encoding config' => [PARSE_MULTILINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8])
1348
- def test_multiline_encoding_of_flushed_record(data)
1349
- encoding_config, encoding = data
1350
-
1351
- config = config_element("", "", {
1352
- "multiline_flush_interval" => "2s",
1353
- "read_from_head" => "true",
1354
- })
1355
- d = create_driver(config + encoding_config)
1356
-
1357
- d.run(expect_emits: 1) do
1358
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f|
1359
- f.puts "s test"
1360
- }
1361
- end
1362
- events = d.events
1363
- assert_equal(1, events.length)
1364
- assert_equal(encoding, events[0][2]['message1'].encoding)
1365
- end
1366
-
1367
- def test_multiline_from_encoding_of_flushed_record
1368
- conf = MULTILINE_CONFIG + config_element(
1369
- "", "",
1370
- {
1371
- "multiline_flush_interval" => "1s",
1372
- "read_from_head" => "true",
1373
- "from_encoding" => "cp932",
1374
- "encoding" => "utf-8"
1375
- })
1376
- d = create_driver(conf)
1377
-
1378
- cp932_message = "s \x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
1379
- utf8_message = "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".encode(Encoding::UTF_8, Encoding::CP932)
1380
- d.run(expect_emits: 1) do
1381
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w:cp932") { |f|
1382
- f.puts cp932_message
1383
- }
1384
- end
1385
-
1386
- events = d.events
1387
- assert_equal(1, events.length)
1388
- assert_equal(utf8_message, events[0][2]['message1'])
1389
- assert_equal(Encoding::UTF_8, events[0][2]['message1'].encoding)
1390
- end
1391
-
1392
- data(flat: config_element(
1393
- "", "", {
1394
- "format" => "multiline",
1395
- "format1" => "/^s (?<message1>[^\\n]+)\\n?/",
1396
- "format2" => "/(f (?<message2>[^\\n]+)\\n?)?/",
1397
- "format3" => "/(f (?<message3>.*))?/",
1398
- "format_firstline" => "/^[s]/"
1399
- }),
1400
- parse: config_element(
1401
- "", "", {},
1402
- [config_element("parse", "", {
1403
- "@type" => "multiline",
1404
- "format1" => "/^s (?<message1>[^\\n]+)\\n?/",
1405
- "format2" => "/(f (?<message2>[^\\n]+)\\n?)?/",
1406
- "format3" => "/(f (?<message3>.*))?/",
1407
- "format_firstline" => "/^[s]/"
1408
- })
1409
- ])
1410
- )
1411
- def test_multiline_with_multiple_formats(data)
1412
- config = data
1413
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1414
-
1415
- d = create_driver(config)
1416
- d.run(expect_emits: 1) do
1417
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1418
- f.puts "f test1"
1419
- f.puts "s test2"
1420
- f.puts "f test3"
1421
- f.puts "f test4"
1422
- f.puts "s test5"
1423
- f.puts "s test6"
1424
- f.puts "f test7"
1425
- f.puts "s test8"
1426
- }
1427
- end
1428
-
1429
- events = d.events
1430
- assert(events.length > 0)
1431
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
1432
- assert_equal({"message1" => "test5"}, events[1][2])
1433
- assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
1434
- assert_equal({"message1" => "test8"}, events[3][2])
1435
- end
1436
-
1437
- data(flat: config_element(
1438
- "", "", {
1439
- "format" => "multiline",
1440
- "format1" => "/^[s|f] (?<message>.*)/",
1441
- "format_firstline" => "/^[s]/"
1442
- }),
1443
- parse: config_element(
1444
- "", "", {},
1445
- [config_element("parse", "", {
1446
- "@type" => "multiline",
1447
- "format1" => "/^[s|f] (?<message>.*)/",
1448
- "format_firstline" => "/^[s]/"
1449
- })
1450
- ])
1451
- )
1452
- def test_multilinelog_with_multiple_paths(data)
1453
- files = ["#{@tmp_dir}/tail1.txt", "#{@tmp_dir}/tail2.txt"]
1454
- files.each { |file| Fluent::FileWrapper.open(file, "wb") { |f| } }
1455
-
1456
- config = data + config_element("", "", {
1457
- "path" => "#{files[0]},#{files[1]}",
1458
- "tag" => "t1",
1459
- })
1460
- d = create_driver(config, false)
1461
- d.run(expect_emits: 2) do
1462
- files.each do |file|
1463
- Fluent::FileWrapper.open(file, 'ab') { |f|
1464
- f.puts "f #{file} line should be ignored"
1465
- f.puts "s test1"
1466
- f.puts "f test2"
1467
- f.puts "f test3"
1468
- f.puts "s test4"
1469
- }
1470
- end
1471
- end
1472
-
1473
- events = d.events
1474
- assert_equal({"message" => "test1\nf test2\nf test3"}, events[0][2])
1475
- assert_equal({"message" => "test1\nf test2\nf test3"}, events[1][2])
1476
- # "test4" events are here because these events are flushed at shutdown phase
1477
- assert_equal({"message" => "test4"}, events[2][2])
1478
- assert_equal({"message" => "test4"}, events[3][2])
1479
- end
1480
-
1481
- data(flat: config_element("", "", {
1482
- "format" => "multiline",
1483
- "format1" => "/(?<var1>foo \\d)\\n/",
1484
- "format2" => "/(?<var2>bar \\d)\\n/",
1485
- "format3" => "/(?<var3>baz \\d)/"
1486
- }),
1487
- parse: config_element(
1488
- "", "", {},
1489
- [config_element("parse", "", {
1490
- "@type" => "multiline",
1491
- "format1" => "/(?<var1>foo \\d)\\n/",
1492
- "format2" => "/(?<var2>bar \\d)\\n/",
1493
- "format3" => "/(?<var3>baz \\d)/"
1494
- })
1495
- ])
1496
- )
1497
- def test_multiline_without_firstline(data)
1498
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
1499
-
1500
- config = data
1501
- d = create_driver(config)
1502
- d.run(expect_emits: 1) do
1503
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
1504
- f.puts "foo 1"
1505
- f.puts "bar 1"
1506
- f.puts "baz 1"
1507
- f.puts "foo 2"
1508
- f.puts "bar 2"
1509
- f.puts "baz 2"
1510
- }
1511
- end
1512
-
1513
- events = d.events
1514
- assert_equal(2, events.length)
1515
- assert_equal({"var1" => "foo 1", "var2" => "bar 1", "var3" => "baz 1"}, events[0][2])
1516
- assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, events[1][2])
1517
- end
1518
- end
1519
-
1520
- sub_test_case "path" do
1521
- # * path test
1522
- # TODO: Clean up tests
1523
- def test_expand_paths
1524
- ex_paths = [
1525
- create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1526
- create_target_info('test/plugin/data/log/foo/bar.log'),
1527
- create_target_info('test/plugin/data/log/test.log')
1528
- ]
1529
- plugin = create_driver(ex_config, false).instance
1530
- flexstub(Time) do |timeclass|
1531
- timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
1532
- assert_equal(ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1533
- end
1534
-
1535
- # Test exclusion
1536
- exclude_config = ex_config + config_element("", "", { "exclude_path" => %Q(["#{ex_paths.last.path}"]) })
1537
- plugin = create_driver(exclude_config, false).instance
1538
- assert_equal(ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1539
- end
1540
-
1541
- sub_test_case "expand_paths with glob" do |data|
1542
- sub_test_case "extended_glob" do
1543
- data("curly braces" => [true, "always", "test/plugin/data/log_numeric/{0,1}*.log"],
1544
- "square brackets" => [true, "always", "test/plugin/data/log_numeric/[0-1][2-4].log"],
1545
- "asterisk" => [true, "always", "test/plugin/data/log/*.log"],
1546
- "one character matcher" => [true, "always", "test/plugin/data/log/tes?.log"],
1547
- )
1548
- def test_expand_paths_with_use_glob_p_and_almost_set_of_patterns
1549
- result, option, path = data
1550
- config = config_element("", "", {
1551
- "tag" => "tail",
1552
- "path" => path,
1553
- "format" => "none",
1554
- "pos_file" => "#{@tmp_dir}/tail.pos",
1555
- "read_from_head" => true,
1556
- "refresh_interval" => 30,
1557
- "glob_policy" => option,
1558
- "path_delimiter" => "|",
1559
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1560
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1561
- })
1562
- plugin = create_driver(config, false).instance
1563
- assert_equal(result, !!plugin.use_glob?(path))
1564
- end
1565
-
1566
- data("curly braces" => [true, false, "extended", "test/plugin/data/log_numeric/{0,1}*.log"],
1567
- "square brackets" => [false, true, "extended", "test/plugin/data/log_numeric/[0-1][2-4].log"],
1568
- "asterisk" => [false, true, "extended", "test/plugin/data/log/*.log"],
1569
- "one character matcher" => [false, true, "extended", "test/plugin/data/log/tes?.log"],
1570
- )
1571
- def test_expand_paths_with_use_glob_p
1572
- emit_exception_p, result, option, path = data
1573
- config = config_element("", "", {
1574
- "tag" => "tail",
1575
- "path" => path,
1576
- "format" => "none",
1577
- "pos_file" => "#{@tmp_dir}/tail.pos",
1578
- "read_from_head" => true,
1579
- "refresh_interval" => 30,
1580
- "glob_policy" => option,
1581
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1582
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1583
- })
1584
- if emit_exception_p
1585
- assert_raise(Fluent::ConfigError) do
1586
- plugin = create_driver(config, false).instance
1587
- end
1588
- else
1589
- plugin = create_driver(config, false).instance
1590
- assert_equal(result, !!plugin.use_glob?(path))
1591
- end
1592
- end
1593
- end
1594
-
1595
- sub_test_case "only_use_backward_compatible" do
1596
- data("square brackets" => [false, "backward_compatible", "test/plugin/data/log_numeric/[0-1][2-4].log"],
1597
- "asterisk" => [true, "backward_compatible", "test/plugin/data/log/*.log"],
1598
- "one character matcher" => [false, "backward_compatible", "test/plugin/data/log/tes?.log"],
1599
- )
1600
- def test_expand_paths_with_use_glob_p
1601
- result, option, path = data
1602
- config = config_element("", "", {
1603
- "tag" => "tail",
1604
- "path" => path,
1605
- "format" => "none",
1606
- "pos_file" => "#{@tmp_dir}/tail.pos",
1607
- "read_from_head" => true,
1608
- "refresh_interval" => 30,
1609
- "glob_policy" => option,
1610
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1611
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1612
- })
1613
- plugin = create_driver(config, false).instance
1614
- assert_equal(result, !!plugin.use_glob?(path))
1615
- end
1616
- end
1617
- end
1618
-
1619
- def ex_config_with_brackets
1620
- config_element("", "", {
1621
- "tag" => "tail",
1622
- "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
1623
- "format" => "none",
1624
- "pos_file" => "#{@tmp_dir}/tail.pos",
1625
- "read_from_head" => true,
1626
- "refresh_interval" => 30,
1627
- "glob_policy" => "extended",
1628
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1629
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1630
- })
1631
- end
1632
-
1633
- def test_config_with_always_with_default_delimiter
1634
- assert_raise(Fluent::ConfigError) do
1635
- config = config_element("", "", {
1636
- "tag" => "tail",
1637
- "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
1638
- "format" => "none",
1639
- "pos_file" => "#{@tmp_dir}/tail.pos",
1640
- "read_from_head" => true,
1641
- "refresh_interval" => 30,
1642
- "glob_policy" => "always",
1643
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1644
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1645
- })
1646
-
1647
- create_driver(config, false).instance
1648
- end
1649
- end
1650
-
1651
- def test_config_with_always_with_custom_delimiter
1652
- assert_nothing_raised do
1653
- config = config_element("", "", {
1654
- "tag" => "tail",
1655
- "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
1656
- "format" => "none",
1657
- "pos_file" => "#{@tmp_dir}/tail.pos",
1658
- "read_from_head" => true,
1659
- "refresh_interval" => 30,
1660
- "glob_policy" => "always",
1661
- "path_delimiter" => "|",
1662
- "rotate_wait" => "#{EX_ROTATE_WAIT}s",
1663
- "follow_inodes" => "#{EX_FOLLOW_INODES}",
1664
- })
1665
-
1666
- create_driver(config, false).instance
1667
- end
1668
- end
1669
-
1670
- def test_expand_paths_with_brackets
1671
- expanded_paths = [
1672
- create_target_info('test/plugin/data/log_numeric/01.log'),
1673
- create_target_info('test/plugin/data/log_numeric/02.log'),
1674
- create_target_info('test/plugin/data/log_numeric/12.log'),
1675
- create_target_info('test/plugin/data/log_numeric/14.log'),
1676
- ]
1677
-
1678
- plugin = create_driver(ex_config_with_brackets, false).instance
1679
- assert_equal(expanded_paths - [expanded_paths.first], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1680
- end
1681
-
1682
- def test_expand_paths_with_duplicate_configuration
1683
- expanded_paths = [
1684
- create_target_info('test/plugin/data/log/foo/bar.log'),
1685
- create_target_info('test/plugin/data/log/test.log')
1686
- ]
1687
- duplicate_config = ex_config.dup
1688
- duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
1689
- plugin = create_driver(ex_config, false).instance
1690
- assert_equal(expanded_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1691
- end
1692
-
1693
- def test_expand_paths_with_timezone
1694
- ex_paths = [
1695
- create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1696
- create_target_info('test/plugin/data/log/foo/bar.log'),
1697
- create_target_info('test/plugin/data/log/test.log')
1698
- ]
1699
- ['Asia/Taipei', '+08'].each do |tz_type|
1700
- taipei_config = ex_config + config_element("", "", {"path_timezone" => tz_type})
1701
- plugin = create_driver(taipei_config, false).instance
1702
-
1703
- # Test exclude
1704
- exclude_config = taipei_config + config_element("", "", { "exclude_path" => %Q(["test/plugin/**/%Y%m%d-%H%M%S.log"]) })
1705
- exclude_plugin = create_driver(exclude_config, false).instance
1706
-
1707
- with_timezone('utc') do
1708
- flexstub(Time) do |timeclass|
1709
- # env : 2010-01-01 19:04:05 (UTC), tail path : 2010-01-02 03:04:05 (Asia/Taipei)
1710
- timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 1, 19, 4, 5))
1711
-
1712
- assert_equal(ex_paths, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1713
- assert_equal(ex_paths - [ex_paths.first], exclude_plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1714
- end
1715
- end
1716
- end
1717
- end
1718
-
1719
- def test_log_file_without_extension
1720
- expected_files = [
1721
- create_target_info('test/plugin/data/log/bar'),
1722
- create_target_info('test/plugin/data/log/foo/bar.log'),
1723
- create_target_info('test/plugin/data/log/foo/bar2'),
1724
- create_target_info('test/plugin/data/log/test.log')
1725
- ]
1726
-
1727
- config = config_element("", "", {
1728
- "tag" => "tail",
1729
- "path" => "test/plugin/data/log/**/*",
1730
- "format" => "none",
1731
- "pos_file" => "#{@tmp_dir}/tail.pos"
1732
- })
1733
-
1734
- plugin = create_driver(config, false).instance
1735
- assert_equal(expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
1736
- end
1737
-
1738
- def test_unwatched_files_should_be_removed
1739
- config = config_element("", "", {
1740
- "tag" => "tail",
1741
- "path" => "#{@tmp_dir}/*.txt",
1742
- "format" => "none",
1743
- "pos_file" => "#{@tmp_dir}/tail.pos",
1744
- "read_from_head" => true,
1745
- "refresh_interval" => 1,
1746
- })
1747
- d = create_driver(config, false)
1748
- d.end_if { d.instance.instance_variable_get(:@tails).keys.size >= 1 }
1749
- d.run(expect_emits: 1, shutdown: false) do
1750
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f| f.puts "test3\n" }
1751
- end
1752
-
1753
- cleanup_file("#{@tmp_dir}/tail.txt")
1754
- waiting(20) { sleep 0.1 until Dir.glob("#{@tmp_dir}/*.txt").size == 0 } # Ensure file is deleted on Windows
1755
- waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size <= 0 }
1756
-
1757
- assert_equal(
1758
- {
1759
- files: [],
1760
- tails: []
1761
- },
1762
- {
1763
- files: Dir.glob("#{@tmp_dir}/*.txt"),
1764
- tails: d.instance.instance_variable_get(:@tails).keys
1765
- }
1766
- )
1767
- ensure
1768
- d.instance_shutdown if d && d.instance
1769
- cleanup_directory(@tmp_dir)
1770
- end
1771
-
1772
- def count_timer_object
1773
- num = 0
1774
- ObjectSpace.each_object(Fluent::PluginHelper::Timer::TimerWatcher) { |obj|
1775
- num += 1
1776
- }
1777
- num
1778
- end
1779
- end
1780
-
1781
- sub_test_case "path w/ Linux capability" do
1782
- def capability_enabled?
1783
- if Fluent.linux?
1784
- begin
1785
- require 'capng'
1786
- true
1787
- rescue LoadError
1788
- false
1789
- end
1790
- else
1791
- false
1792
- end
1793
- end
1794
-
1795
- setup do
1796
- omit "This environment is not enabled Linux capability handling feature" unless capability_enabled?
1797
-
1798
- @capng = CapNG.new(:current_process)
1799
- flexstub(Fluent::Capability) do |klass|
1800
- klass.should_receive(:new).with(:current_process).and_return(@capng)
1801
- end
1802
- end
1803
-
1804
- data("dac_read_search" => [:dac_read_search, true, 1],
1805
- "dac_override" => [:dac_override, true, 1],
1806
- "chown" => [:chown, false, 0],
1807
- )
1808
- test "with partially elevated privileges" do |data|
1809
- cap, result, readable_paths = data
1810
- @capng.update(:add, :effective, cap)
1811
-
1812
- d = create_driver(
1813
- config_element("ROOT", "", {
1814
- "path" => "/var/log/ker*.log", # Use /var/log/kern.log
1815
- "tag" => "t1",
1816
- "rotate_wait" => "2s"
1817
- }) + PARSE_SINGLE_LINE_CONFIG, false)
1818
-
1819
- assert_equal(readable_paths, d.instance.expand_paths.length)
1820
- assert_equal(result, d.instance.have_read_capability?)
1821
- end
1822
- end
1823
-
1824
- def test_pos_file_dir_creation
1825
- config = config_element("", "", {
1826
- "tag" => "tail",
1827
- "path" => "#{@tmp_dir}/*.txt",
1828
- "format" => "none",
1829
- "pos_file" => "#{@tmp_dir}/pos/tail.pos",
1830
- "read_from_head" => true,
1831
- "refresh_interval" => 1
1832
- })
1833
-
1834
- assert_path_not_exist("#{@tmp_dir}/pos")
1835
- d = create_driver(config, false)
1836
- d.run
1837
- assert_path_exist("#{@tmp_dir}/pos")
1838
- assert_equal('755', File.stat("#{@tmp_dir}/pos").mode.to_s(8)[-3, 3])
1839
- ensure
1840
- cleanup_directory(@tmp_dir)
1841
- end
1842
-
1843
- def test_pos_file_dir_creation_with_system_dir_permission
1844
- config = config_element("", "", {
1845
- "tag" => "tail",
1846
- "path" => "#{@tmp_dir}/*.txt",
1847
- "format" => "none",
1848
- "pos_file" => "#{@tmp_dir}/pos/tail.pos",
1849
- "read_from_head" => true,
1850
- "refresh_interval" => 1
1851
- })
1852
-
1853
- assert_path_not_exist("#{@tmp_dir}/pos")
1854
-
1855
- Fluent::SystemConfig.overwrite_system_config({ "dir_permission" => "744" }) do
1856
- d = create_driver(config, false)
1857
- d.run
1858
- end
1859
-
1860
- assert_path_exist("#{@tmp_dir}/pos")
1861
- if Fluent.windows?
1862
- assert_equal('755', File.stat("#{@tmp_dir}/pos").mode.to_s(8)[-3, 3])
1863
- else
1864
- assert_equal('744', File.stat("#{@tmp_dir}/pos").mode.to_s(8)[-3, 3])
1865
- end
1866
- ensure
1867
- cleanup_directory(@tmp_dir)
1868
- end
1869
-
1870
- def test_z_refresh_watchers
1871
- ex_paths = [
1872
- create_target_info('test/plugin/data/2010/01/20100102-030405.log'),
1873
- create_target_info('test/plugin/data/log/foo/bar.log'),
1874
- create_target_info('test/plugin/data/log/test.log'),
1875
- ]
1876
- plugin = create_driver(ex_config, false).instance
1877
- sio = StringIO.new
1878
- plugin.instance_eval do
1879
- @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
1880
- @loop = Coolio::Loop.new
1881
- opened_file_metrics = Fluent::Plugin::LocalMetrics.new
1882
- opened_file_metrics.configure(config_element('metrics', '', {}))
1883
- closed_file_metrics = Fluent::Plugin::LocalMetrics.new
1884
- closed_file_metrics.configure(config_element('metrics', '', {}))
1885
- rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
1886
- rotated_file_metrics.configure(config_element('metrics', '', {}))
1887
- @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
1888
- end
1889
-
1890
- Timecop.freeze(2010, 1, 2, 3, 4, 5) do
1891
- ex_paths.each do |target_info|
1892
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
1893
- end
1894
-
1895
- plugin.refresh_watchers
1896
- end
1897
-
1898
- path = 'test/plugin/data/2010/01/20100102-030405.log'
1899
- mock.proxy(plugin).detach_watcher_after_rotate_wait(plugin.instance_variable_get(:@tails)[path], Fluent::FileWrapper.stat(path).ino)
1900
-
1901
- Timecop.freeze(2010, 1, 2, 3, 4, 6) do
1902
- path = "test/plugin/data/2010/01/20100102-030406.log"
1903
- inode = Fluent::FileWrapper.stat(path).ino
1904
- target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
1905
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
1906
- plugin.refresh_watchers
1907
-
1908
- flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
1909
- watcherclass.should_receive(:new).never
1910
- plugin.refresh_watchers
1911
- end
1912
- end
1913
- end
1914
-
1915
- sub_test_case "refresh of pos file" do
1916
- test 'type of pos_file_compaction_interval is time' do
1917
- tail = {
1918
- "tag" => "tail",
1919
- "path" => "#{@tmp_dir}/*.txt",
1920
- "format" => "none",
1921
- "pos_file" => "#{@tmp_dir}/pos/tail.pos",
1922
- "refresh_interval" => 1,
1923
- "read_from_head" => true,
1924
- 'pos_file_compaction_interval' => '24h',
1925
- }
1926
- config = config_element("", "", tail)
1927
- d = create_driver(config, false)
1928
- mock(d.instance).timer_execute(:in_tail_refresh_watchers, 1.0).once
1929
- mock(d.instance).timer_execute(:in_tail_refresh_compact_pos_file, 60 * 60 * 24).once
1930
- d.run # call start
1931
- end
1932
- end
1933
-
1934
- sub_test_case "receive_lines" do
1935
- DummyWatcher = Struct.new("DummyWatcher", :tag)
1936
-
1937
- def test_tag
1938
- d = create_driver(ex_config, false)
1939
- d.run {}
1940
- plugin = d.instance
1941
- mock(plugin.router).emit_stream('tail', anything).once
1942
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1943
- end
1944
-
1945
- def test_tag_with_only_star
1946
- config = config_element("", "", {
1947
- "tag" => "*",
1948
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1949
- "format" => "none",
1950
- "read_from_head" => true
1951
- })
1952
- d = create_driver(config, false)
1953
- d.run {}
1954
- plugin = d.instance
1955
- mock(plugin.router).emit_stream('foo.bar.log', anything).once
1956
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1957
- end
1958
-
1959
- def test_tag_prefix
1960
- config = config_element("", "", {
1961
- "tag" => "pre.*",
1962
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1963
- "format" => "none",
1964
- "read_from_head" => true
1965
- })
1966
- d = create_driver(config, false)
1967
- d.run {}
1968
- plugin = d.instance
1969
- mock(plugin.router).emit_stream('pre.foo.bar.log', anything).once
1970
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1971
- end
1972
-
1973
- def test_tag_suffix
1974
- config = config_element("", "", {
1975
- "tag" => "*.post",
1976
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1977
- "format" => "none",
1978
- "read_from_head" => true
1979
- })
1980
- d = create_driver(config, false)
1981
- d.run {}
1982
- plugin = d.instance
1983
- mock(plugin.router).emit_stream('foo.bar.log.post', anything).once
1984
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1985
- end
1986
-
1987
- def test_tag_prefix_and_suffix
1988
- config = config_element("", "", {
1989
- "tag" => "pre.*.post",
1990
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1991
- "format" => "none",
1992
- "read_from_head" => true
1993
- })
1994
- d = create_driver(config, false)
1995
- d.run {}
1996
- plugin = d.instance
1997
- mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once
1998
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1999
- end
2000
-
2001
- def test_tag_prefix_and_suffix_ignore
2002
- config = config_element("", "", {
2003
- "tag" => "pre.*.post*ignore",
2004
- "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
2005
- "format" => "none",
2006
- "read_from_head" => true
2007
- })
2008
- d = create_driver(config, false)
2009
- d.run {}
2010
- plugin = d.instance
2011
- mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once
2012
- plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
2013
- end
2014
-
2015
- data(
2016
- small: ["128", 128],
2017
- KiB: ["1k", 1024]
2018
- )
2019
- test 'max_line_size' do |(label, size)|
2020
- config = config_element("", "", {
2021
- "tag" => "max_line_size",
2022
- "path" => "#{@tmp_dir}/with_long_lines.txt",
2023
- "format" => "none",
2024
- "read_from_head" => true,
2025
- "max_line_size" => label,
2026
- "log_level" => "debug"
2027
- })
2028
- Fluent::FileWrapper.open("#{@tmp_dir}/with_long_lines.txt", "w+") do |f|
2029
- f.puts "foo"
2030
- f.puts "x" * size # 'x' * size + \n > @max_line_size
2031
- f.puts "bar"
2032
- end
2033
- d = create_driver(config, false)
2034
- timestamp = Time.parse("Mon Nov 29 11:22:33 UTC 2021")
2035
- Timecop.freeze(timestamp)
2036
- d.run(expect_records: 2)
2037
- assert_equal([
2038
- [{"message" => "foo"},{"message" => "bar"}],
2039
- [
2040
- "2021-11-29 11:22:33 +0000 [warn]: received line length is longer than #{size}\n",
2041
- "2021-11-29 11:22:33 +0000 [debug]: skipped line: #{'x' * size}\n"
2042
- ]
2043
- ],
2044
- [
2045
- d.events.collect { |event| event.last },
2046
- d.logs[-2..]
2047
- ])
2048
- end
2049
- end
2050
-
2051
- # Ensure that no fatal exception is raised when a file is missing and that
2052
- # files that do exist are still tailed as expected.
2053
- def test_missing_file
2054
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2055
- f.puts "test1"
2056
- f.puts "test2"
2057
- }
2058
-
2059
- # Try two different configs - one with read_from_head and one without,
2060
- # since their interactions with the filesystem differ.
2061
- config1 = config_element("", "", {
2062
- "tag" => "t1",
2063
- "path" => "#{@tmp_dir}/non_existent_file.txt,#{@tmp_dir}/tail.txt",
2064
- "format" => "none",
2065
- "rotate_wait" => "2s",
2066
- "pos_file" => "#{@tmp_dir}/tail.pos"
2067
- })
2068
- config2 = config1 + config_element("", "", { "read_from_head" => true })
2069
- [config1, config2].each do |config|
2070
- d = create_driver(config, false)
2071
- d.run(expect_emits: 1) do
2072
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
2073
- f.puts "test3"
2074
- f.puts "test4"
2075
- }
2076
- end
2077
- # This test sometimes fails and it shows a potential bug of in_tail
2078
- # https://github.com/fluent/fluentd/issues/1434
2079
- events = d.events
2080
- assert_equal(2, events.length)
2081
- assert_equal({"message" => "test3"}, events[0][2])
2082
- assert_equal({"message" => "test4"}, events[1][2])
2083
- end
2084
- end
2085
-
2086
- sub_test_case 'inode_processing' do
2087
- def test_should_delete_file_pos_entry_for_non_existing_file_with_follow_inodes
2088
- config = common_follow_inode_config
2089
-
2090
- path = "#{@tmp_dir}/tail.txt"
2091
- ino = 1
2092
- pos = 1234
2093
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "wb") {|f|
2094
- f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
2095
- }
2096
-
2097
- d = create_driver(config, false)
2098
- d.run
2099
-
2100
- pos_file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r")
2101
- pos_file.pos = 0
2102
-
2103
- assert_raise(EOFError) do
2104
- pos_file.readline
2105
- end
2106
- end
2107
-
2108
- def test_should_write_latest_offset_after_rotate_wait
2109
- config = common_follow_inode_config
2110
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2111
- f.puts "test1"
2112
- f.puts "test2"
2113
- }
2114
-
2115
- d = create_driver(config, false)
2116
- d.run(expect_emits: 2, shutdown: false) do
2117
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2118
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
2119
- sleep 1
2120
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt" + "1", "ab") {|f| f.puts "test4\n"}
2121
- end
2122
-
2123
- pos_file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r")
2124
- pos_file.pos = 0
2125
- line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
2126
- waiting(5) {
2127
- while line_parts[2].to_i(16) != 24
2128
- sleep(0.1)
2129
- pos_file.pos = 0
2130
- line_parts = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
2131
- end
2132
- }
2133
- assert_equal(24, line_parts[2].to_i(16))
2134
- d.instance_shutdown
2135
- end
2136
-
2137
- def test_should_remove_deleted_file
2138
- config = config_element("", "", {"format" => "none"})
2139
-
2140
- path = "#{@tmp_dir}/tail.txt"
2141
- ino = 1
2142
- pos = 1234
2143
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "wb") {|f|
2144
- f.puts ("%s\t%016x\t%016x\n" % [path, pos, ino])
2145
- }
2146
-
2147
- d = create_driver(config)
2148
- d.run do
2149
- pos_file = Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r")
2150
- pos_file.pos = 0
2151
- assert_equal([], pos_file.readlines)
2152
- end
2153
- end
2154
-
2155
- def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
2156
- config = config_element("ROOT", "", {
2157
- "path" => "#{@tmp_dir}/tail.txt*",
2158
- "pos_file" => "#{@tmp_dir}/tail.pos",
2159
- "tag" => "t1",
2160
- "rotate_wait" => "1s",
2161
- "refresh_interval" => "1s",
2162
- "limit_recently_modified" => "1s",
2163
- "read_from_head" => "true",
2164
- "format" => "none",
2165
- "follow_inodes" => "true",
2166
- })
2167
-
2168
- d = create_driver(config, false)
2169
-
2170
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2171
- f.puts "test1"
2172
- f.puts "test2"
2173
- }
2174
- target_info = create_target_info("#{@tmp_dir}/tail.txt")
2175
-
2176
- d.run(expect_emits: 1, shutdown: false) do
2177
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2178
- end
2179
-
2180
-
2181
- Timecop.travel(Time.now + 10) do
2182
- waiting(5) {
2183
- # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
2184
- sleep 0.1 until d.instance.instance_variable_get(:@pf)[target_info].read_pos == 0
2185
- }
2186
- end
2187
-
2188
- assert_equal(0, d.instance.instance_variable_get(:@pf)[target_info].read_pos)
2189
-
2190
- d.instance_shutdown
2191
- end
2192
-
2193
- def test_should_read_from_head_on_file_renaming_with_star_in_pattern
2194
- config = config_element("ROOT", "", {
2195
- "path" => "#{@tmp_dir}/tail.txt*",
2196
- "pos_file" => "#{@tmp_dir}/tail.pos",
2197
- "tag" => "t1",
2198
- "rotate_wait" => "10s",
2199
- "refresh_interval" => "1s",
2200
- "limit_recently_modified" => "60s",
2201
- "read_from_head" => "true",
2202
- "format" => "none",
2203
- "follow_inodes" => "true"
2204
- })
2205
-
2206
- d = create_driver(config, false)
2207
-
2208
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2209
- f.puts "test1"
2210
- f.puts "test2"
2211
- }
2212
-
2213
- d.run(expect_emits: 2, shutdown: false) do
2214
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2215
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1")
2216
- end
2217
-
2218
- events = d.events
2219
- assert_equal(3, events.length)
2220
- d.instance_shutdown
2221
- end
2222
-
2223
- def test_should_not_read_from_head_on_rotation_when_watching_inodes
2224
- config = common_follow_inode_config
2225
-
2226
- d = create_driver(config, false)
2227
-
2228
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2229
- f.puts "test1"
2230
- f.puts "test2"
2231
- }
2232
-
2233
- d.run(expect_emits: 1, shutdown: false) do
2234
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2235
- end
2236
-
2237
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1")
2238
- Timecop.travel(Time.now + 10) do
2239
- sleep 2
2240
- events = d.events
2241
- assert_equal(3, events.length)
2242
- end
2243
-
2244
- d.instance_shutdown
2245
- end
2246
-
2247
- def test_should_mark_file_unwatched_if_same_name_file_created_with_different_inode
2248
- config = common_follow_inode_config
2249
-
2250
- d = create_driver(config, false)
2251
-
2252
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2253
- f.puts "test1"
2254
- f.puts "test2"
2255
- }
2256
- target_info = create_target_info("#{@tmp_dir}/tail.txt")
2257
-
2258
- d.run(expect_emits: 2, shutdown: false) do
2259
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2260
- cleanup_file("#{@tmp_dir}/tail.txt")
2261
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "test4\n"}
2262
- end
2263
-
2264
- new_target_info = create_target_info("#{@tmp_dir}/tail.txt")
2265
-
2266
- pos_file = d.instance.instance_variable_get(:@pf)
2267
-
2268
- waiting(10) {
2269
- # @pos will be reset as 0 when UNWATCHED_POSITION is specified.
2270
- sleep 0.1 until pos_file[target_info].read_pos == 0
2271
- }
2272
- new_position = pos_file[new_target_info].read_pos
2273
- assert_equal(6, new_position)
2274
-
2275
- d.instance_shutdown
2276
- end
2277
-
2278
- def test_should_close_watcher_after_rotate_wait
2279
- now = Time.now
2280
- config = common_follow_inode_config + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
2281
-
2282
- d = create_driver(config, false)
2283
- d.instance.instance_eval do
2284
- opened_file_metrics = Fluent::Plugin::LocalMetrics.new
2285
- opened_file_metrics.configure(config_element('metrics', '', {}))
2286
- closed_file_metrics = Fluent::Plugin::LocalMetrics.new
2287
- closed_file_metrics.configure(config_element('metrics', '', {}))
2288
- rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
2289
- rotated_file_metrics.configure(config_element('metrics', '', {}))
2290
- @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
2291
- end
2292
-
2293
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2294
- f.puts "test1"
2295
- f.puts "test2"
2296
- }
2297
- target_info = create_target_info("#{@tmp_dir}/tail.txt")
2298
- mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything, anything).once
2299
- d.run(shutdown: false)
2300
- assert d.instance.instance_variable_get(:@tails)[target_info.path]
2301
-
2302
- Timecop.travel(now + 10) do
2303
- d.instance.instance_eval do
2304
- sleep 0.1 until @tails[target_info.path] == nil
2305
- end
2306
- assert_nil d.instance.instance_variable_get(:@tails)[target_info.path]
2307
- end
2308
- d.instance_shutdown
2309
- end
2310
-
2311
- def test_should_create_new_watcher_for_new_file_with_same_name
2312
- now = Time.now
2313
- config = common_follow_inode_config + config_element('', '', {"limit_recently_modified" => "2s"})
2314
-
2315
- d = create_driver(config, false)
2316
-
2317
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2318
- f.puts "test1"
2319
- f.puts "test2"
2320
- }
2321
- path_ino = create_target_info("#{@tmp_dir}/tail.txt")
2322
-
2323
- d.run(expect_emits: 1, shutdown: false) do
2324
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2325
- end
2326
-
2327
- cleanup_file("#{@tmp_dir}/tail.txt")
2328
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2329
- f.puts "test3"
2330
- f.puts "test4"
2331
- }
2332
- new_path_ino = create_target_info("#{@tmp_dir}/tail.txt")
2333
-
2334
- Timecop.travel(now + 10) do
2335
- sleep 3
2336
- d.instance.instance_eval do
2337
- @tails[path_ino.path] == nil
2338
- @tails[new_path_ino.path] != nil
2339
- end
2340
- end
2341
-
2342
- events = d.events
2343
-
2344
- assert_equal(5, events.length)
2345
-
2346
- d.instance_shutdown
2347
- end
2348
-
2349
- def test_truncate_file_with_follow_inodes
2350
- config = common_follow_inode_config
2351
-
2352
- d = create_driver(config, false)
2353
-
2354
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2355
- f.puts "test1"
2356
- f.puts "test2"
2357
- }
2358
-
2359
- d.run(expect_emits: 3, shutdown: false) do
2360
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "test3\n"}
2361
- sleep 2
2362
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "w+b") {|f| f.puts "test4\n"}
2363
- end
2364
-
2365
- events = d.events
2366
- assert_equal(4, events.length)
2367
- assert_equal({"message" => "test1"}, events[0][2])
2368
- assert_equal({"message" => "test2"}, events[1][2])
2369
- assert_equal({"message" => "test3"}, events[2][2])
2370
- assert_equal({"message" => "test4"}, events[3][2])
2371
- d.instance_shutdown
2372
- end
2373
-
2374
- # issue #3464
2375
- def test_should_replace_target_info
2376
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2377
- f.puts "test1\n"
2378
- }
2379
- target_info = create_target_info("#{@tmp_dir}/tail.txt")
2380
- inodes = []
2381
-
2382
- config = config_element("ROOT", "", {
2383
- "path" => "#{@tmp_dir}/tail.txt*",
2384
- "pos_file" => "#{@tmp_dir}/tail.pos",
2385
- "tag" => "t1",
2386
- "refresh_interval" => "60s",
2387
- "read_from_head" => "true",
2388
- "format" => "none",
2389
- "rotate_wait" => "1s",
2390
- "follow_inodes" => "true"
2391
- })
2392
- d = create_driver(config, false)
2393
- d.run(timeout: 5) do
2394
- while d.events.size < 1 do
2395
- sleep 0.1
2396
- end
2397
- inodes = d.instance.instance_variable_get(:@tails).values.collect do |tw|
2398
- tw.ino
2399
- end
2400
- assert_equal([target_info.ino], inodes)
2401
-
2402
- cleanup_file("#{@tmp_dir}/tail.txt")
2403
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "test2\n"}
2404
-
2405
- while d.events.size < 2 do
2406
- sleep 0.1
2407
- end
2408
- inodes = d.instance.instance_variable_get(:@tails).values.collect do |tw|
2409
- tw.ino
2410
- end
2411
- new_target_info = create_target_info("#{@tmp_dir}/tail.txt")
2412
- assert_not_equal(target_info.ino, new_target_info.ino)
2413
- assert_equal([new_target_info.ino], inodes)
2414
- end
2415
- end
2416
- end
2417
-
2418
- sub_test_case "tail_path" do
2419
- def test_tail_path_with_singleline
2420
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f|
2421
- f.puts "test1"
2422
- f.puts "test2"
2423
- }
2424
-
2425
- d = create_driver(SINGLE_LINE_CONFIG + config_element("", "", { "path_key" => "path" }))
2426
-
2427
- d.run(expect_emits: 1) do
2428
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f|
2429
- f.puts "test3"
2430
- f.puts "test4"
2431
- }
2432
- end
2433
-
2434
- events = d.events
2435
- assert_equal(true, events.length > 0)
2436
- events.each do |emit|
2437
- assert_equal("#{@tmp_dir}/tail.txt", emit[2]["path"])
2438
- end
2439
- end
2440
-
2441
- def test_tail_path_with_multiline_with_firstline
2442
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
2443
-
2444
- config = config_element("", "", {
2445
- "path_key" => "path",
2446
- "format" => "multiline",
2447
- "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
2448
- "format_firstline" => "/^[s]/"
2449
- })
2450
- d = create_driver(config)
2451
- d.run(expect_emits: 1) do
2452
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
2453
- f.puts "f test1"
2454
- f.puts "s test2"
2455
- f.puts "f test3"
2456
- f.puts "f test4"
2457
- f.puts "s test5"
2458
- f.puts "s test6"
2459
- f.puts "f test7"
2460
- f.puts "s test8"
2461
- }
2462
- end
2463
-
2464
- events = d.events
2465
- assert_equal(4, events.length)
2466
- events.each do |emit|
2467
- assert_equal("#{@tmp_dir}/tail.txt", emit[2]["path"])
2468
- end
2469
- end
2470
-
2471
- def test_tail_path_with_multiline_without_firstline
2472
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") { |f| }
2473
-
2474
- config = config_element("", "", {
2475
- "path_key" => "path",
2476
- "format" => "multiline",
2477
- "format1" => "/(?<var1>foo \\d)\\n/",
2478
- "format2" => "/(?<var2>bar \\d)\\n/",
2479
- "format3" => "/(?<var3>baz \\d)/",
2480
- })
2481
- d = create_driver(config)
2482
- d.run(expect_emits: 1) do
2483
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") { |f|
2484
- f.puts "foo 1"
2485
- f.puts "bar 1"
2486
- f.puts "baz 1"
2487
- }
2488
- end
2489
-
2490
- events = d.events
2491
- assert(events.length > 0)
2492
- events.each do |emit|
2493
- assert_equal("#{@tmp_dir}/tail.txt", emit[2]["path"])
2494
- end
2495
- end
2496
-
2497
- def test_tail_path_with_multiline_with_multiple_paths
2498
- if ENV["APPVEYOR"] && Fluent.windows?
2499
- omit "This testcase is unstable on AppVeyor."
2500
- end
2501
- files = ["#{@tmp_dir}/tail1.txt", "#{@tmp_dir}/tail2.txt"]
2502
- files.each { |file| Fluent::FileWrapper.open(file, "wb") { |f| } }
2503
-
2504
- config = config_element("", "", {
2505
- "path" => "#{files[0]},#{files[1]}",
2506
- "path_key" => "path",
2507
- "tag" => "t1",
2508
- "format" => "multiline",
2509
- "format1" => "/^[s|f] (?<message>.*)/",
2510
- "format_firstline" => "/^[s]/"
2511
- })
2512
- d = create_driver(config, false)
2513
- d.run(expect_emits: 2) do
2514
- files.each do |file|
2515
- Fluent::FileWrapper.open(file, 'ab') { |f|
2516
- f.puts "f #{file} line should be ignored"
2517
- f.puts "s test1"
2518
- f.puts "f test2"
2519
- f.puts "f test3"
2520
- f.puts "s test4"
2521
- }
2522
- end
2523
- end
2524
-
2525
- events = d.events
2526
- assert_equal(4, events.length)
2527
- assert_equal(files, [events[0][2]["path"], events[1][2]["path"]].sort)
2528
- # "test4" events are here because these events are flushed at shutdown phase
2529
- assert_equal(files, [events[2][2]["path"], events[3][2]["path"]].sort)
2530
- end
2531
- end
2532
-
2533
- def test_limit_recently_modified
2534
- now = Time.new(2010, 1, 2, 3, 4, 5)
2535
- FileUtils.touch("#{@tmp_dir}/tail_unwatch.txt", mtime: (now - 3601))
2536
- FileUtils.touch("#{@tmp_dir}/tail_watch1.txt", mtime: (now - 3600))
2537
- FileUtils.touch("#{@tmp_dir}/tail_watch2.txt", mtime: now)
2538
-
2539
- config = config_element('', '', {
2540
- 'tag' => 'tail',
2541
- 'path' => "#{@tmp_dir}/*.txt",
2542
- 'format' => 'none',
2543
- 'limit_recently_modified' => '3600s'
2544
- })
2545
-
2546
- expected_files = [
2547
- create_target_info("#{@tmp_dir}/tail_watch1.txt"),
2548
- create_target_info("#{@tmp_dir}/tail_watch2.txt")
2549
- ]
2550
-
2551
- Timecop.freeze(now) do
2552
- plugin = create_driver(config, false).instance
2553
- assert_equal(expected_files, plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
2554
- end
2555
- end
2556
-
2557
- def test_skip_refresh_on_startup
2558
- FileUtils.touch("#{@tmp_dir}/tail.txt")
2559
- config = config_element('', '', {
2560
- 'format' => 'none',
2561
- 'refresh_interval' => 1,
2562
- 'skip_refresh_on_startup' => true
2563
- })
2564
- d = create_driver(config)
2565
- d.run(shutdown: false) {}
2566
- assert_equal(0, d.instance.instance_variable_get(:@tails).keys.size)
2567
- # detect a file at first execution of in_tail_refresh_watchers timer
2568
- waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1 }
2569
- d.instance_shutdown
2570
- end
2571
-
2572
- def test_ENOENT_error_after_setup_watcher
2573
- path = "#{@tmp_dir}/tail.txt"
2574
- FileUtils.touch(path)
2575
- config = config_element('', '', {
2576
- 'format' => 'none',
2577
- })
2578
- d = create_driver(config)
2579
- file_deleted = false
2580
- mock.proxy(d.instance).existence_path do |hash|
2581
- unless file_deleted
2582
- cleanup_file(path)
2583
- file_deleted = true
2584
- end
2585
- hash
2586
- end.twice
2587
- assert_nothing_raised do
2588
- d.run(shutdown: false) {}
2589
- end
2590
- assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed. Continuing without tailing it.\n") },
2591
- $log.out.logs.join("\n"))
2592
- ensure
2593
- d.instance_shutdown if d && d.instance
2594
- end
2595
-
2596
- def test_EACCES_error_after_setup_watcher
2597
- omit "Cannot test with root user" if Process::UID.eid == 0
2598
- path = "#{@tmp_dir}/noaccess/tail.txt"
2599
- begin
2600
- FileUtils.mkdir_p("#{@tmp_dir}/noaccess")
2601
- FileUtils.chmod(0755, "#{@tmp_dir}/noaccess")
2602
- FileUtils.touch(path)
2603
- config = config_element('', '', {
2604
- 'tag' => "tail",
2605
- 'path' => path,
2606
- 'format' => 'none',
2607
- })
2608
- d = create_driver(config, false)
2609
- mock.proxy(d.instance).existence_path do |hash|
2610
- FileUtils.chmod(0000, "#{@tmp_dir}/noaccess")
2611
- hash
2612
- end.twice
2613
- assert_nothing_raised do
2614
- d.run(shutdown: false) {}
2615
- end
2616
- assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed. Continuing without tailing it.\n") },
2617
- $log.out.logs.join("\n"))
2618
- end
2619
- ensure
2620
- d.instance_shutdown if d && d.instance
2621
- if File.exist?("#{@tmp_dir}/noaccess")
2622
- FileUtils.chmod(0755, "#{@tmp_dir}/noaccess")
2623
- FileUtils.rm_rf("#{@tmp_dir}/noaccess")
2624
- end
2625
- end unless Fluent.windows?
2626
-
2627
- def test_EACCES
2628
- path = "#{@tmp_dir}/tail.txt"
2629
- FileUtils.touch(path)
2630
- config = config_element('', '', {
2631
- 'format' => 'none',
2632
- })
2633
- d = create_driver(config)
2634
- mock.proxy(Fluent::FileWrapper).stat(path) do |stat|
2635
- raise Errno::EACCES
2636
- end.at_least(1)
2637
- assert_nothing_raised do
2638
- d.run(shutdown: false) {}
2639
- end
2640
- assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
2641
- ensure
2642
- d.instance_shutdown if d && d.instance
2643
- end
2644
-
2645
- def test_shutdown_timeout
2646
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") do |f|
2647
- # Should be large enough to take too long time to consume
2648
- (1024 * 1024 * 5).times do
2649
- f.puts "{\"test\":\"fizzbuzz\"}"
2650
- end
2651
- end
2652
-
2653
- config =
2654
- CONFIG_READ_FROM_HEAD +
2655
- config_element('', '', {
2656
- 'format' => 'json',
2657
- 'skip_refresh_on_startup' => true,
2658
- })
2659
- shutdown_start_time = 0
2660
-
2661
- d = create_driver(config)
2662
- mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
2663
- mock.proxy(io_handler).ready_to_shutdown(anything) do
2664
- shutdown_start_time = Fluent::Clock.now
2665
- end
2666
- io_handler.shutdown_timeout = 0.5
2667
- io_handler
2668
- end
2669
-
2670
- assert_nothing_raised do
2671
- d.run(expect_emits: 1)
2672
- end
2673
-
2674
- elapsed = Fluent::Clock.now - shutdown_start_time
2675
- assert_true(elapsed > 0.5 && elapsed < 2.0,
2676
- "elapsed time: #{elapsed}")
2677
- end
2678
-
2679
- sub_test_case "throttling logs at in_tail level" do
2680
- data("file test1.log no_limit 5120 text: msg" => ["test1.log", 5120, "msg"],
2681
- "file test2.log no_limit 1024 text: test" => ["test2.log", 1024, "test"])
2682
- def test_lines_collected_with_no_throttling(data)
2683
- file, num_lines, msg = data
2684
-
2685
- pattern = "/^#{@tmp_dir}\/(?<file>.+)\.log$/"
2686
- rule = create_rule_directive({
2687
- "file" => "/test.*/",
2688
- }, -1)
2689
- group = create_group_directive(pattern, "1s", rule)
2690
- path_element = create_path_element(file)
2691
-
2692
- conf = ROOT_CONFIG + group + path_element + CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG
2693
-
2694
- Fluent::FileWrapper.open("#{@tmp_dir}/#{file}", 'wb') do |f|
2695
- num_lines.times do
2696
- f.puts "#{msg}\n"
2697
- end
2698
- end
2699
-
2700
-
2701
- d = create_driver(conf, false)
2702
- d.run(timeout: 3) do
2703
- start_time = Fluent::Clock.now
2704
-
2705
- assert_equal(num_lines, d.record_count)
2706
- assert_equal({ "message" => msg }, d.events[0][2])
2707
-
2708
- prev_count = d.record_count
2709
- sleep(0.1) while d.emit_count < 1
2710
- assert_true(Fluent::Clock.now - start_time < 2)
2711
- ## after waiting for 1 (+ jitter) secs, limit will reset
2712
- ## Plugin will start reading but it will encounter EOF Error
2713
- ## since no logs are left to be read
2714
- ## Hence, d.record_count = prev_count
2715
- tail_watcher_interval = 1.0 # hard coded value in in_tail
2716
- safety_ratio = 1.2
2717
- jitter = tail_watcher_interval * safety_ratio
2718
- sleep(1.0 + jitter)
2719
- assert_equal(0, d.record_count - prev_count)
2720
- end
2721
- end
2722
-
2723
- test "lines collected with throttling" do
2724
- file = "podname1_namespace12_container-123456.log"
2725
- limit = 1000
2726
- rate_period = 2
2727
- num_lines = 3000
2728
- msg = "a" * 8190 # Total size = 8190 bytes + 2 (\n) bytes
2729
-
2730
- rule = create_rule_directive({
2731
- "namespace"=> "/namespace.+/",
2732
- "podname"=> "/podname.+/",
2733
- }, limit)
2734
- path_element = create_path_element(file)
2735
- conf = ROOT_CONFIG + create_group_directive(tailing_group_pattern, "#{rate_period}s", rule) + path_element + SINGLE_LINE_CONFIG + CONFIG_READ_FROM_HEAD
2736
-
2737
- d = create_driver(conf, false)
2738
- file_path = "#{@tmp_dir}/#{file}"
2739
-
2740
- Fluent::FileWrapper.open(file_path, 'wb') do |f|
2741
- num_lines.times do
2742
- f.puts msg
2743
- end
2744
- end
2745
-
2746
- d.run(timeout: 15) do
2747
- sleep_interval = 0.1
2748
- tail_watcher_interval = 1.0 # hard coded value in in_tail
2749
- safety_ratio = 1.2
2750
- lower_jitter = sleep_interval * safety_ratio
2751
- upper_jitter = (tail_watcher_interval + sleep_interval) * safety_ratio
2752
- lower_interval = rate_period - lower_jitter
2753
- upper_interval = rate_period + upper_jitter
2754
-
2755
- emit_count = 0
2756
- prev_count = 0
2757
-
2758
- while emit_count < 3 do
2759
- start_time = Fluent::Clock.now
2760
- sleep(sleep_interval) while d.emit_count <= emit_count
2761
- elapsed_seconds = Fluent::Clock.now - start_time
2762
- if emit_count > 0
2763
- assert_true(elapsed_seconds > lower_interval && elapsed_seconds < upper_interval,
2764
- "elapsed_seconds #{elapsed_seconds} is out of allowed range:\n" +
2765
- " lower: #{lower_interval} [sec]\n" +
2766
- " upper: #{upper_interval} [sec]")
2767
- end
2768
- assert_equal(limit, d.record_count - prev_count)
2769
- emit_count = d.emit_count
2770
- prev_count = d.record_count
2771
- end
2772
-
2773
- ## When all the lines are read and rate_period seconds are over
2774
- ## limit will reset and since there are no more logs to be read,
2775
- ## number_lines_read will be 0
2776
- sleep upper_interval
2777
- gw = d.instance.find_group_from_metadata(file_path)
2778
- assert_equal(0, gw.current_paths[file_path].number_lines_read)
2779
- end
2780
- end
2781
- end
2782
-
2783
- sub_test_case "Update watchers for rotation with follow_inodes" do
2784
- def test_updateTW_before_refreshTW_and_detach_before_refreshTW
2785
- config = config_element(
2786
- "ROOT",
2787
- "",
2788
- {
2789
- "path" => "#{@tmp_dir}/tail.txt*",
2790
- "pos_file" => "#{@tmp_dir}/tail.pos",
2791
- "tag" => "t1",
2792
- "format" => "none",
2793
- "read_from_head" => "true",
2794
- "follow_inodes" => "true",
2795
- # In order to detach the old watcher quickly.
2796
- "rotate_wait" => "1s",
2797
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
2798
- # called by a timer.
2799
- "refresh_interval" => "1h",
2800
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
2801
- # so disable it in order to reproduce the same condition stably.
2802
- "enable_stat_watcher" => "false",
2803
- }
2804
- )
2805
- d = create_driver(config, false)
2806
-
2807
- tail_watchers = []
2808
- stub.proxy(d.instance).setup_watcher do |tw|
2809
- tail_watchers.append(tw)
2810
- tw
2811
- end
2812
-
2813
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file1 log1"}
2814
-
2815
- d.run(expect_records: 4, timeout: 10) do
2816
- # Rotate (If the timing is bad, `TailWatcher::on_notify` might be called between mv and new-file-creation)
2817
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file1 log2"}
2818
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
2819
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file2 log1"}
2820
-
2821
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` updates the TailWatcher:
2822
- # TailWatcher(path: "tail.txt", inode: inode_0) => TailWatcher(path: "tail.txt", inode: inode_1)
2823
- # The old TailWathcer is detached here since `rotate_wait` is just `1s`.
2824
- sleep 3
2825
-
2826
- # This reproduces the following situation:
2827
- # Rotation => update_watcher => refresh_watchers
2828
- # This adds a new TailWatcher: TailWatcher(path: "tail.txt1", inode: inode_0)
2829
- d.instance.refresh_watchers
2830
-
2831
- # Append to the new current log file.
2832
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file2 log2"}
2833
- end
2834
-
2835
- inode_0 = tail_watchers[0].ino
2836
- inode_1 = tail_watchers[1].ino
2837
- record_values = d.events.collect { |event| event[2]["message"] }.sort
2838
- position_entries = []
2839
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
2840
- f.readlines(chomp: true).each do |line|
2841
- values = line.split("\t")
2842
- position_entries.append([values[0], values[1], values[2].to_i(16)])
2843
- end
2844
- end
2845
-
2846
- assert_equal(
2847
- {
2848
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2"],
2849
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1"],
2850
- tail_watcher_inodes: [inode_0, inode_1, inode_0],
2851
- tail_watcher_io_handler_opened_statuses: [false, false, false],
2852
- position_entries: [
2853
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_0],
2854
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_1],
2855
- ],
2856
- },
2857
- {
2858
- record_values: record_values,
2859
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
2860
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
2861
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
2862
- position_entries: position_entries
2863
- },
2864
- )
2865
- end
2866
-
2867
- def test_updateTW_before_refreshTW_and_detach_after_refreshTW
2868
- config = config_element(
2869
- "ROOT",
2870
- "",
2871
- {
2872
- "path" => "#{@tmp_dir}/tail.txt*",
2873
- "pos_file" => "#{@tmp_dir}/tail.pos",
2874
- "tag" => "t1",
2875
- "format" => "none",
2876
- "read_from_head" => "true",
2877
- "follow_inodes" => "true",
2878
- # In order to detach the old watcher after refresh_watchers.
2879
- "rotate_wait" => "4s",
2880
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
2881
- # called by a timer.
2882
- "refresh_interval" => "1h",
2883
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
2884
- # so disable it in order to reproduce the same condition stably.
2885
- "enable_stat_watcher" => "false",
2886
- }
2887
- )
2888
- d = create_driver(config, false)
2889
-
2890
- tail_watchers = []
2891
- stub.proxy(d.instance).setup_watcher do |tw|
2892
- tail_watchers.append(tw)
2893
- tw
2894
- end
2895
-
2896
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file1 log1"}
2897
-
2898
- d.run(expect_records: 4, timeout: 10) do
2899
- # Rotate (If the timing is bad, `TailWatcher::on_notify` might be called between mv and new-file-creation)
2900
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file1 log2"}
2901
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
2902
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file2 log1"}
2903
-
2904
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` updates the TailWatcher:
2905
- # TailWatcher(path: "tail.txt", inode: inode_0) => TailWatcher(path: "tail.txt", inode: inode_1)
2906
- sleep 2
2907
-
2908
- # This reproduces the following situation:
2909
- # Rotation => update_watcher => refresh_watchers
2910
- # This adds a new TailWatcher: TailWatcher(path: "tail.txt1", inode: inode_0)
2911
- d.instance.refresh_watchers
2912
-
2913
- # The old TailWathcer is detached here since `rotate_wait` is `4s`.
2914
- sleep 3
2915
-
2916
- # Append to the new current log file.
2917
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file2 log2"}
2918
- end
2919
-
2920
- inode_0 = tail_watchers[0].ino
2921
- inode_1 = tail_watchers[1].ino
2922
- record_values = d.events.collect { |event| event[2]["message"] }.sort
2923
- position_entries = []
2924
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
2925
- f.readlines(chomp: true).each do |line|
2926
- values = line.split("\t")
2927
- position_entries.append([values[0], values[1], values[2].to_i(16)])
2928
- end
2929
- end
2930
-
2931
- assert_equal(
2932
- {
2933
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2"],
2934
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1"],
2935
- tail_watcher_inodes: [inode_0, inode_1, inode_0],
2936
- tail_watcher_io_handler_opened_statuses: [false, false, false],
2937
- position_entries: [
2938
- # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
2939
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_0],
2940
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_1],
2941
- ],
2942
- },
2943
- {
2944
- record_values: record_values,
2945
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
2946
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
2947
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
2948
- position_entries: position_entries
2949
- },
2950
- )
2951
- end
2952
-
2953
- # The scenario where in_tail wrongly detaches TailWatcher.
2954
- # This is reported in https://github.com/fluent/fluentd/issues/4190.
2955
- def test_updateTW_after_refreshTW
2956
- config = config_element(
2957
- "ROOT",
2958
- "",
2959
- {
2960
- "path" => "#{@tmp_dir}/tail.txt*",
2961
- "pos_file" => "#{@tmp_dir}/tail.pos",
2962
- "tag" => "t1",
2963
- "format" => "none",
2964
- "read_from_head" => "true",
2965
- "follow_inodes" => "true",
2966
- # In order to detach the old watcher quickly.
2967
- "rotate_wait" => "1s",
2968
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
2969
- # called by a timer.
2970
- "refresh_interval" => "1h",
2971
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
2972
- # so disable it in order to reproduce the same condition stably.
2973
- "enable_stat_watcher" => "false",
2974
- }
2975
- )
2976
- d = create_driver(config, false)
2977
-
2978
- tail_watchers = []
2979
- stub.proxy(d.instance).setup_watcher do |tw|
2980
- tail_watchers.append(tw)
2981
- tw
2982
- end
2983
-
2984
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file1 log1"}
2985
-
2986
- d.run(expect_records: 4, timeout: 10) do
2987
- # Rotate (If the timing is bad, `TailWatcher::on_notify` might be called between mv and new-file-creation)
2988
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file1 log2"}
2989
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
2990
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file2 log1"}
2991
-
2992
- # This reproduces the following situation:
2993
- # Rotation => refresh_watchers => update_watcher
2994
- # This add a new TailWatcher: TailWatcher(path: "tail.txt", inode: inode_1)
2995
- # This overwrites `@tails["tail.txt"]`
2996
- d.instance.refresh_watchers
2997
-
2998
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to update the TailWatcher:
2999
- # TailWatcher(path: "tail.txt", inode: inode_0) => TailWatcher(path: "tail.txt", inode: inode_1)
3000
- # However, it is already added in `refresh_watcher`, so `update_watcher` doesn't create the new TailWatcher.
3001
- # The old TailWathcer is detached here since `rotate_wait` is just `1s`.
3002
- sleep 3
3003
-
3004
- # This adds a new TailWatcher: TailWatcher(path: "tail.txt1", inode: inode_0)
3005
- d.instance.refresh_watchers
3006
-
3007
- # Append to the new current log file.
3008
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file2 log2"}
3009
- end
3010
-
3011
- inode_0 = tail_watchers[0].ino
3012
- inode_1 = tail_watchers[1].ino
3013
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3014
- position_entries = []
3015
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3016
- f.readlines(chomp: true).each do |line|
3017
- values = line.split("\t")
3018
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3019
- end
3020
- end
3021
-
3022
- assert_equal(
3023
- {
3024
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2"],
3025
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt1"],
3026
- tail_watcher_inodes: [inode_0, inode_1, inode_0],
3027
- tail_watcher_io_handler_opened_statuses: [false, false, false],
3028
- position_entries: [
3029
- # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
3030
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_0],
3031
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_1],
3032
- ],
3033
- },
3034
- {
3035
- record_values: record_values,
3036
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3037
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3038
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3039
- position_entries: position_entries
3040
- },
3041
- )
3042
- end
3043
-
3044
- def test_path_resurrection
3045
- config = config_element(
3046
- "ROOT",
3047
- "",
3048
- {
3049
- "path" => "#{@tmp_dir}/tail.txt*",
3050
- "pos_file" => "#{@tmp_dir}/tail.pos",
3051
- "tag" => "t1",
3052
- "format" => "none",
3053
- "read_from_head" => "true",
3054
- "follow_inodes" => "true",
3055
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
3056
- # called by a timer.
3057
- "refresh_interval" => "1h",
3058
- # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
3059
- # Because of this problem, log duplication can occur during `rotate_wait`.
3060
- # Need to set `rotate_wait 0` for a workaround.
3061
- "rotate_wait" => "0s",
3062
- }
3063
- )
3064
- d = create_driver(config, false)
3065
-
3066
- tail_watchers = []
3067
- stub.proxy(d.instance).setup_watcher do |tw|
3068
- tail_watchers.append(tw)
3069
- tw
3070
- end
3071
-
3072
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file1 log1"}
3073
-
3074
- d.run(expect_records: 5, timeout: 10) do
3075
- # Rotate
3076
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file1 log2"}
3077
- FileUtils.move("#{@tmp_dir}/tail.txt", "#{@tmp_dir}/tail.txt" + "1")
3078
- # TailWatcher(path: "tail.txt", inode: inode_0) detects `tail.txt` disappeared.
3079
- # Call `update_watcher` to stop and discard self.
3080
- # If not discarding, then it will be a orphan and cause leak and log duplication.
3081
- #
3082
- # This reproduces the case where the notify to TailWatcher comes before the new file for the path
3083
- # is created during rotation.
3084
- # (stat_watcher notifies faster than a new file is created)
3085
- # Overall, this is a rotation operation, but from the TailWatcher, it appears as if the file
3086
- # was resurrected once it disappeared.
3087
- sleep 2 # On Windows and macOS, StatWatcher doesn't work, so need enough interval for TimeTrigger.
3088
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "file2 log1"}
3089
-
3090
- # Add new TailWatchers
3091
- # tail.txt: TailWatcher(path: "tail.txt", inode: inode_1)
3092
- # tail.txt: TailWatcher(path: "tail.txt1", inode: inode_0)
3093
- # NOTE: If not discarding the first TailWatcher on notify, this makes it a orphan because
3094
- # this overwrites the `@tails[tail.txt]` by adding TailWatcher(path: "tail.txt", inode: inode_1)
3095
- d.instance.refresh_watchers
3096
-
3097
- # This does nothing.
3098
- # NOTE: If not discarding the first TailWatcher on notify, this add
3099
- # tail.txt1: TailWatcher(path: "tail.txt1", inode: inode_0)
3100
- # because the previous refresh_watcher overwrites `@tails[tail.txt]` and the inode_0 is lost.
3101
- # This would cause log duplication.
3102
- d.instance.refresh_watchers
3103
-
3104
- # Append to the old file
3105
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt1", "ab") {|f| f.puts "file1 log3"}
3106
-
3107
- # Append to the new current log file.
3108
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "ab") {|f| f.puts "file2 log2"}
3109
- end
3110
-
3111
- inode_0 = Fluent::FileWrapper.stat("#{@tmp_dir}/tail.txt1").ino
3112
- inode_1 = Fluent::FileWrapper.stat("#{@tmp_dir}/tail.txt").ino
3113
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3114
- position_entries = []
3115
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3116
- f.readlines(chomp: true).each do |line|
3117
- values = line.split("\t")
3118
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3119
- end
3120
- end
3121
-
3122
- assert_equal(
3123
- {
3124
- record_values: ["file1 log1", "file1 log2", "file1 log3", "file2 log1", "file2 log2"],
3125
- tail_watcher_set: Set[
3126
- {
3127
- path: "#{@tmp_dir}/tail.txt",
3128
- inode: inode_0,
3129
- io_handler_opened_status: false,
3130
- },
3131
- {
3132
- path: "#{@tmp_dir}/tail.txt",
3133
- inode: inode_1,
3134
- io_handler_opened_status: false,
3135
- },
3136
- {
3137
- path: "#{@tmp_dir}/tail.txt1",
3138
- inode: inode_0,
3139
- io_handler_opened_status: false,
3140
- },
3141
- ],
3142
- position_entries: [
3143
- ["#{@tmp_dir}/tail.txt", "0000000000000021", inode_0],
3144
- ["#{@tmp_dir}/tail.txt", "0000000000000016", inode_1],
3145
- ],
3146
- },
3147
- {
3148
- record_values: record_values,
3149
- tail_watcher_set: Set.new(tail_watchers.collect { |tw|
3150
- {
3151
- path: tw.path,
3152
- inode: tw.ino,
3153
- io_handler_opened_status: tw.instance_variable_get(:@io_handler)&.opened? || false,
3154
- }
3155
- }),
3156
- position_entries: position_entries,
3157
- },
3158
- )
3159
- end
3160
-
3161
- def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait
3162
- config = config_element(
3163
- "ROOT",
3164
- "",
3165
- {
3166
- "path" => "#{@tmp_dir}/tail.txt*",
3167
- "pos_file" => "#{@tmp_dir}/tail.pos",
3168
- "tag" => "t1",
3169
- "format" => "none",
3170
- "read_from_head" => "true",
3171
- "follow_inodes" => "true",
3172
- "rotate_wait" => "3s",
3173
- "refresh_interval" => "1h",
3174
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
3175
- # so disable it in order to reproduce the same condition stably.
3176
- "enable_stat_watcher" => "false",
3177
- }
3178
- )
3179
- d = create_driver(config, false)
3180
-
3181
- tail_watchers = []
3182
- stub.proxy(d.instance).setup_watcher do |tw|
3183
- tail_watchers.append(tw)
3184
- mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.
3185
- tw
3186
- end
3187
-
3188
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
3189
-
3190
- d.run(expect_records: 6, timeout: 15) do
3191
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
3192
-
3193
- sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3194
-
3195
- FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
3196
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
3197
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
3198
-
3199
- sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3200
-
3201
- # Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)
3202
- [1, 0].each do |i|
3203
- FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
3204
- end
3205
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
3206
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3207
-
3208
- # Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.
3209
- # (Note: Currently, there is no harm in duplicate calls)
3210
- sleep 4
3211
- end
3212
-
3213
- inode_0 = tail_watchers[0]&.ino
3214
- inode_1 = tail_watchers[1]&.ino
3215
- inode_2 = tail_watchers[2]&.ino
3216
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3217
- position_entries = []
3218
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3219
- f.readlines(chomp: true).each do |line|
3220
- values = line.split("\t")
3221
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3222
- end
3223
- end
3224
-
3225
- assert_equal(
3226
- {
3227
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
3228
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
3229
- tail_watcher_inodes: [inode_0, inode_1, inode_2],
3230
- tail_watcher_io_handler_opened_statuses: [false, false, false],
3231
- position_entries: [
3232
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_0],
3233
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_1],
3234
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
3235
- ],
3236
- },
3237
- {
3238
- record_values: record_values,
3239
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3240
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3241
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3242
- position_entries: position_entries
3243
- },
3244
- )
3245
- end
3246
- end
3247
-
3248
- sub_test_case "Update watchers for rotation without follow_inodes" do
3249
- # The scenario where in_tail wrongly unwatches the PositionEntry.
3250
- # This is reported in https://github.com/fluent/fluentd/issues/3614.
3251
- def test_refreshTW_during_rotation
3252
- config = config_element(
3253
- "ROOT",
3254
- "",
3255
- {
3256
- "path" => "#{@tmp_dir}/tail.txt0",
3257
- "pos_file" => "#{@tmp_dir}/tail.pos",
3258
- "tag" => "t1",
3259
- "format" => "none",
3260
- "read_from_head" => "true",
3261
- # In order to detach the old watcher quickly.
3262
- "rotate_wait" => "3s",
3263
- # In order to reproduce the same condition stably, ensure that `refresh_watchers` is not
3264
- # called by a timer.
3265
- "refresh_interval" => "1h",
3266
- # stat_watcher often calls `TailWatcher::on_notify` faster than creating a new log file,
3267
- # so disable it in order to reproduce the same condition stably.
3268
- "enable_stat_watcher" => "false",
3269
- }
3270
- )
3271
- d = create_driver(config, false)
3272
-
3273
- tail_watchers = []
3274
- stub.proxy(d.instance).setup_watcher do |tw|
3275
- tail_watchers.append(tw)
3276
- tw
3277
- end
3278
-
3279
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
3280
-
3281
- d.run(expect_records: 6, timeout: 15) do
3282
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
3283
- FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
3284
-
3285
- # This reproduces the following situation:
3286
- # `refresh_watchers` is called during the rotation process and it detects the current file being lost.
3287
- # Then it stops and unwatches the TailWatcher.
3288
- d.instance.refresh_watchers
3289
-
3290
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
3291
-
3292
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to add the new TailWatcher.
3293
- # After `rotate_wait` interval, the PositionEntry is unwatched.
3294
- # HOWEVER, the new TailWatcher is still using that PositionEntry, so this breaks the PositionFile!!
3295
- # That PositionEntry is removed from `PositionFile::map`, but it is still working and remaining in the real pos file.
3296
- sleep 5
3297
-
3298
- # Append to the new current log file.
3299
- # The PositionEntry is updated although it does not exist in `PositionFile::map`.
3300
- # `PositionFile::map`: empty
3301
- # Real pos file: `.../tail.txt 0000000000000016 (inode)`
3302
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
3303
-
3304
- # Rotate again
3305
- [1, 0].each do |i|
3306
- FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
3307
- end
3308
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
3309
-
3310
- # `watch_timer` calls `TailWatcher::on_notify`, and then `update_watcher` trys to update the TailWatcher.
3311
- sleep 3
3312
-
3313
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3314
- end
3315
-
3316
- inode_0 = tail_watchers[0]&.ino
3317
- inode_1 = tail_watchers[1]&.ino
3318
- inode_2 = tail_watchers[2]&.ino
3319
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3320
- position_entries = []
3321
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3322
- f.readlines(chomp: true).each do |line|
3323
- values = line.split("\t")
3324
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3325
- end
3326
- end
3327
-
3328
- assert_equal(
3329
- {
3330
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
3331
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
3332
- tail_watcher_inodes: [inode_0, inode_1, inode_2],
3333
- tail_watcher_io_handler_opened_statuses: [false, false, false],
3334
- position_entries: [
3335
- # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
3336
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
3337
- ],
3338
- },
3339
- {
3340
- record_values: record_values,
3341
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3342
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3343
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3344
- position_entries: position_entries
3345
- },
3346
- )
3347
- end
3348
-
3349
- def test_next_rotation_occurs_very_fast_while_old_TW_still_waiting_rotate_wait
3350
- config = config_element(
3351
- "ROOT",
3352
- "",
3353
- {
3354
- "path" => "#{@tmp_dir}/tail.txt0",
3355
- "pos_file" => "#{@tmp_dir}/tail.pos",
3356
- "tag" => "t1",
3357
- "format" => "none",
3358
- "read_from_head" => "true",
3359
- "rotate_wait" => "3s",
3360
- "refresh_interval" => "1h",
3361
- }
3362
- )
3363
- d = create_driver(config, false)
3364
-
3365
- tail_watchers = []
3366
- stub.proxy(d.instance).setup_watcher do |tw|
3367
- tail_watchers.append(tw)
3368
- mock.proxy(tw).close.once # Note: Currently, there is no harm in duplicate calls.
3369
- tw
3370
- end
3371
-
3372
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file1 log1"}
3373
-
3374
- d.run(expect_records: 6, timeout: 15) do
3375
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file1 log2"}
3376
-
3377
- sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3378
-
3379
- FileUtils.move("#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt" + "1")
3380
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file2 log1"}
3381
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file2 log2"}
3382
-
3383
- sleep 1.5 # Need to be larger than 1s (the interval of watch_timer)
3384
-
3385
- # Rotate again (Old TailWatcher waiting rotate_wait also calls update_watcher)
3386
- [1, 0].each do |i|
3387
- FileUtils.move("#{@tmp_dir}/tail.txt#{i}", "#{@tmp_dir}/tail.txt#{i + 1}")
3388
- end
3389
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "wb") {|f| f.puts "file3 log1"}
3390
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
3391
-
3392
- # Wait rotate_wait to confirm that TailWatcher.close is not called in duplicate.
3393
- # (Note: Currently, there is no harm in duplicate calls)
3394
- sleep 4
3395
- end
3396
-
3397
- inode_0 = tail_watchers[0]&.ino
3398
- inode_1 = tail_watchers[1]&.ino
3399
- inode_2 = tail_watchers[2]&.ino
3400
- record_values = d.events.collect { |event| event[2]["message"] }.sort
3401
- position_entries = []
3402
- Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
3403
- f.readlines(chomp: true).each do |line|
3404
- values = line.split("\t")
3405
- position_entries.append([values[0], values[1], values[2].to_i(16)])
3406
- end
3407
- end
3408
-
3409
- assert_equal(
3410
- {
3411
- record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
3412
- tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
3413
- tail_watcher_inodes: [inode_0, inode_1, inode_2],
3414
- tail_watcher_io_handler_opened_statuses: [false, false, false],
3415
- position_entries: [
3416
- ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
3417
- ],
3418
- },
3419
- {
3420
- record_values: record_values,
3421
- tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
3422
- tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
3423
- tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
3424
- position_entries: position_entries
3425
- },
3426
- )
3427
- end
3428
- end
3429
- end