fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,204 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'cool.io'
18
+ require 'yajl'
19
+
20
+ require 'fluent/plugin/input'
21
+ require 'fluent/config/error'
22
+ require 'fluent/plugin/parser'
23
+
24
+ module Fluent::Plugin
25
+ class SyslogInput < Input
26
+ Fluent::Plugin.register_input('syslog', self)
27
+
28
+ helpers :parser, :event_loop
29
+
30
+ SYSLOG_REGEXP = /^\<([0-9]+)\>(.*)/
31
+
32
+ FACILITY_MAP = {
33
+ 0 => 'kern',
34
+ 1 => 'user',
35
+ 2 => 'mail',
36
+ 3 => 'daemon',
37
+ 4 => 'auth',
38
+ 5 => 'syslog',
39
+ 6 => 'lpr',
40
+ 7 => 'news',
41
+ 8 => 'uucp',
42
+ 9 => 'cron',
43
+ 10 => 'authpriv',
44
+ 11 => 'ftp',
45
+ 12 => 'ntp',
46
+ 13 => 'audit',
47
+ 14 => 'alert',
48
+ 15 => 'at',
49
+ 16 => 'local0',
50
+ 17 => 'local1',
51
+ 18 => 'local2',
52
+ 19 => 'local3',
53
+ 20 => 'local4',
54
+ 21 => 'local5',
55
+ 22 => 'local6',
56
+ 23 => 'local7'
57
+ }
58
+
59
+ PRIORITY_MAP = {
60
+ 0 => 'emerg',
61
+ 1 => 'alert',
62
+ 2 => 'crit',
63
+ 3 => 'err',
64
+ 4 => 'warn',
65
+ 5 => 'notice',
66
+ 6 => 'info',
67
+ 7 => 'debug'
68
+ }
69
+
70
+ def initialize
71
+ super
72
+ require 'fluent/plugin/socket_util'
73
+ end
74
+
75
+ desc 'The port to listen to.'
76
+ config_param :port, :integer, default: 5140
77
+ desc 'The bind address to listen to.'
78
+ config_param :bind, :string, default: '0.0.0.0'
79
+ desc 'The prefix of the tag. The tag itself is generated by the tag prefix, facility level, and priority.'
80
+ config_param :tag, :string
81
+ desc 'The transport protocol used to receive logs.(udp, tcp)'
82
+ config_param :protocol_type, default: :udp do |val|
83
+ case val.downcase
84
+ when 'tcp'
85
+ :tcp
86
+ when 'udp'
87
+ :udp
88
+ else
89
+ raise ConfigError, "syslog input protocol type should be 'tcp' or 'udp'"
90
+ end
91
+ end
92
+ desc 'If true, add source host to event record.'
93
+ config_param :include_source_host, :bool, default: false
94
+ desc 'Specify key of source host when include_source_host is true.'
95
+ config_param :source_host_key, :string, default: 'source_host'.freeze
96
+ config_param :blocking_timeout, :time, default: 0.5
97
+ config_param :message_length_limit, :size, default: 2048
98
+
99
+ def configure(conf)
100
+ super
101
+
102
+ @use_default = false
103
+
104
+ if conf.has_key?('format')
105
+ @parser = parser_create(usage: 'syslog_input', type: conf['format'], conf: conf)
106
+ else
107
+ conf['with_priority'] = true
108
+ @parser = parser_create(usage: 'syslog_input', type: 'syslog', conf: conf)
109
+ @use_default = true
110
+ end
111
+ @_event_loop_run_timeout = @blocking_timeout
112
+ end
113
+
114
+ def start
115
+ super
116
+
117
+ callback = if @use_default
118
+ method(:receive_data)
119
+ else
120
+ method(:receive_data_parser)
121
+ end
122
+
123
+ @handler = listen(callback)
124
+ event_loop_attach(@handler)
125
+ end
126
+
127
+ def shutdown
128
+ @handler.close
129
+
130
+ super
131
+ end
132
+
133
+ private
134
+
135
+ def receive_data_parser(data, addr)
136
+ m = SYSLOG_REGEXP.match(data)
137
+ unless m
138
+ log.warn "invalid syslog message: #{data.dump}"
139
+ return
140
+ end
141
+ pri = m[1].to_i
142
+ text = m[2]
143
+
144
+ @parser.parse(text) { |time, record|
145
+ unless time && record
146
+ log.warn "pattern not match: #{text.inspect}"
147
+ return
148
+ end
149
+
150
+ record[@source_host_key] = addr[2] if @include_source_host
151
+ emit(pri, time, record)
152
+ }
153
+ rescue => e
154
+ log.error data.dump, error: e.to_s
155
+ log.error_backtrace
156
+ end
157
+
158
+ def receive_data(data, addr)
159
+ @parser.parse(data) { |time, record|
160
+ unless time && record
161
+ log.warn "invalid syslog message", data: data
162
+ return
163
+ end
164
+
165
+ pri = record.delete('pri')
166
+ record[@source_host_key] = addr[2] if @include_source_host
167
+ emit(pri, time, record)
168
+ }
169
+ rescue => e
170
+ log.error data.dump, error: e.to_s
171
+ log.error_backtrace
172
+ end
173
+
174
+ private
175
+
176
+ def listen(callback)
177
+ log.info "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
178
+ socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
179
+ if Fluent.windows?
180
+ socket_manager_path = socket_manager_path.to_i
181
+ end
182
+ client = ServerEngine::SocketManager::Client.new(socket_manager_path)
183
+ if @protocol_type == :udp
184
+ @usock = client.listen_udp(@bind, @port)
185
+ Fluent::SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback)
186
+ else
187
+ # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
188
+ lsock = client.listen_tcp(@bind, @port)
189
+ Coolio::TCPServer.new(lsock, nil, Fluent::SocketUtil::TcpHandler, log, "\n", callback)
190
+ end
191
+ end
192
+
193
+ def emit(pri, time, record)
194
+ facility = FACILITY_MAP[pri >> 3]
195
+ priority = PRIORITY_MAP[pri & 0b111]
196
+
197
+ tag = "#{@tag}.#{facility}.#{priority}"
198
+
199
+ router.emit(tag, time, record)
200
+ rescue => e
201
+ log.error "syslog failed to emit", error: e, tag: tag, record: Yajl.dump(record)
202
+ end
203
+ end
204
+ end
@@ -0,0 +1,838 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'cool.io'
18
+
19
+ require 'fluent/input'
20
+ require 'fluent/config/error'
21
+ require 'fluent/event'
22
+ require 'fluent/system_config'
23
+ require 'fluent/plugin/buffer'
24
+
25
+ if Fluent.windows?
26
+ require_relative 'file_wrapper'
27
+ else
28
+ Fluent::FileWrapper = File
29
+ end
30
+
31
+ module Fluent
32
+ class TailInput < Input
33
+ include SystemConfig::Mixin
34
+
35
+ Plugin.register_input('tail', self)
36
+
37
+ FILE_PERMISSION = 0644
38
+
39
+ def initialize
40
+ super
41
+ @paths = []
42
+ @tails = {}
43
+ @pf_file = nil
44
+ @pf = nil
45
+ end
46
+
47
+ desc 'The paths to read. Multiple paths can be specified, separated by comma.'
48
+ config_param :path, :string
49
+ desc 'The tag of the event.'
50
+ config_param :tag, :string
51
+ desc 'The paths to exclude the files from watcher list.'
52
+ config_param :exclude_path, :array, default: []
53
+ desc 'Specify interval to keep reference to old file when rotate a file.'
54
+ config_param :rotate_wait, :time, default: 5
55
+ desc 'Fluentd will record the position it last read into this file.'
56
+ config_param :pos_file, :string, default: nil
57
+ desc 'Start to read the logs from the head of file, not bottom.'
58
+ config_param :read_from_head, :bool, default: false
59
+ # When the program deletes log file and re-creates log file with same filename after passed refresh_interval,
60
+ # in_tail may raise a pos_file related error. This is a known issue but there is no such program on production.
61
+ # If we find such program / application, we will fix the problem.
62
+ desc 'The interval of refreshing the list of watch file.'
63
+ config_param :refresh_interval, :time, default: 60
64
+ desc 'The number of reading lines at each IO.'
65
+ config_param :read_lines_limit, :integer, default: 1000
66
+ desc 'The interval of flushing the buffer for multiline format'
67
+ config_param :multiline_flush_interval, :time, default: nil
68
+ desc 'Enable the additional watch timer.'
69
+ config_param :enable_watch_timer, :bool, default: true
70
+ desc 'The encoding after conversion of the input.'
71
+ config_param :encoding, :string, default: nil
72
+ desc 'The encoding of the input.'
73
+ config_param :from_encoding, :string, default: nil
74
+ desc 'Add the log path being tailed to records. Specify the field name to be used.'
75
+ config_param :path_key, :string, default: nil
76
+
77
+ attr_reader :paths
78
+
79
+ def configure(conf)
80
+ super
81
+
82
+ @paths = @path.split(',').map {|path| path.strip }
83
+ if @paths.empty?
84
+ raise ConfigError, "tail: 'path' parameter is required on tail input"
85
+ end
86
+
87
+ unless @pos_file
88
+ $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
89
+ $log.warn "this parameter is highly recommended to save the position to resume tailing."
90
+ end
91
+
92
+ configure_parser(conf)
93
+ configure_tag
94
+ configure_encoding
95
+
96
+ @multiline_mode = conf['format'] =~ /multiline/
97
+ @receive_handler = if @multiline_mode
98
+ method(:parse_multilines)
99
+ else
100
+ method(:parse_singleline)
101
+ end
102
+ @file_perm = system_config.file_permission || FILE_PERMISSION
103
+ end
104
+
105
+ def configure_parser(conf)
106
+ @parser = Plugin.new_parser(conf['format'])
107
+ @parser.configure(conf)
108
+ end
109
+
110
+ def configure_tag
111
+ if @tag.index('*')
112
+ @tag_prefix, @tag_suffix = @tag.split('*')
113
+ @tag_suffix ||= ''
114
+ else
115
+ @tag_prefix = nil
116
+ @tag_suffix = nil
117
+ end
118
+ end
119
+
120
+ def configure_encoding
121
+ unless @encoding
122
+ if @from_encoding
123
+ raise ConfigError, "tail: 'from_encoding' parameter must be specified with 'encoding' parameter."
124
+ end
125
+ end
126
+
127
+ @encoding = parse_encoding_param(@encoding) if @encoding
128
+ @from_encoding = parse_encoding_param(@from_encoding) if @from_encoding
129
+ end
130
+
131
+ def parse_encoding_param(encoding_name)
132
+ begin
133
+ Encoding.find(encoding_name) if encoding_name
134
+ rescue ArgumentError => e
135
+ raise ConfigError, e.message
136
+ end
137
+ end
138
+
139
+ def start
140
+ super
141
+
142
+ if @pos_file
143
+ @pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY, @file_perm)
144
+ @pf_file.sync = true
145
+ @pf = PositionFile.parse(@pf_file)
146
+ end
147
+
148
+ @loop = Coolio::Loop.new
149
+ refresh_watchers
150
+
151
+ @refresh_trigger = TailWatcher::TimerWatcher.new(@refresh_interval, true, log, &method(:refresh_watchers))
152
+ @refresh_trigger.attach(@loop)
153
+ @thread = Thread.new(&method(:run))
154
+ end
155
+
156
+ def shutdown
157
+ @refresh_trigger.detach if @refresh_trigger && @refresh_trigger.attached?
158
+
159
+ stop_watchers(@tails.keys, true)
160
+ @loop.stop rescue nil # when all watchers are detached, `stop` raises RuntimeError. We can ignore this exception.
161
+ @thread.join
162
+ @pf_file.close if @pf_file
163
+
164
+ super
165
+ end
166
+
167
+ def expand_paths
168
+ date = Time.now
169
+ paths = []
170
+
171
+ excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
172
+ @paths.each { |path|
173
+ path = date.strftime(path)
174
+ if path.include?('*')
175
+ paths += Dir.glob(path).select { |p|
176
+ if File.readable?(p)
177
+ true
178
+ else
179
+ log.warn "#{p} unreadable. It is excluded and would be examined next time."
180
+ false
181
+ end
182
+ }
183
+ else
184
+ # When file is not created yet, Dir.glob returns an empty array. So just add when path is static.
185
+ paths << path
186
+ end
187
+ }
188
+ paths - excluded
189
+ end
190
+
191
+ # in_tail with '*' path doesn't check rotation file equality at refresh phase.
192
+ # So you should not use '*' path when your logs will be rotated by another tool.
193
+ # It will cause log duplication after updated watch files.
194
+ # In such case, you should separate log directory and specify two paths in path parameter.
195
+ # e.g. path /path/to/dir/*,/path/to/rotated_logs/target_file
196
+ def refresh_watchers
197
+ target_paths = expand_paths
198
+ existence_paths = @tails.keys
199
+
200
+ unwatched = existence_paths - target_paths
201
+ added = target_paths - existence_paths
202
+
203
+ stop_watchers(unwatched, false, true) unless unwatched.empty?
204
+ start_watchers(added) unless added.empty?
205
+ end
206
+
207
+ def setup_watcher(path, pe)
208
+ line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
209
+ tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, &method(:receive_lines))
210
+ tw.attach(@loop)
211
+ tw
212
+ end
213
+
214
+ def start_watchers(paths)
215
+ paths.each { |path|
216
+ pe = nil
217
+ if @pf
218
+ pe = @pf[path]
219
+ if @read_from_head && pe.read_inode.zero?
220
+ begin
221
+ pe.update(FileWrapper.stat(path).ino, 0)
222
+ rescue Errno::ENOENT
223
+ $log.warn "#{path} not found. Continuing without tailing it."
224
+ end
225
+ end
226
+ end
227
+
228
+ @tails[path] = setup_watcher(path, pe)
229
+ }
230
+ end
231
+
232
+ def stop_watchers(paths, immediate = false, unwatched = false)
233
+ paths.each { |path|
234
+ tw = @tails.delete(path)
235
+ if tw
236
+ tw.unwatched = unwatched
237
+ if immediate
238
+ close_watcher(tw, false)
239
+ else
240
+ close_watcher_after_rotate_wait(tw)
241
+ end
242
+ end
243
+ }
244
+ end
245
+
246
+ # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
247
+ def update_watcher(path, pe)
248
+ rotated_tw = @tails[path]
249
+ @tails[path] = setup_watcher(path, pe)
250
+ close_watcher_after_rotate_wait(rotated_tw) if rotated_tw
251
+ end
252
+
253
+ # TailWatcher#close is called by another thread at shutdown phase.
254
+ # It causes 'can't modify string; temporarily locked' error in IOHandler
255
+ # so adding close_io argument to avoid this problem.
256
+ # At shutdown, IOHandler's io will be released automatically after detached the event loop
257
+ def close_watcher(tw, close_io = true)
258
+ tw.close(close_io)
259
+ flush_buffer(tw)
260
+ if tw.unwatched && @pf
261
+ @pf[tw.path].update_pos(PositionFile::UNWATCHED_POSITION)
262
+ end
263
+ end
264
+
265
+ def close_watcher_after_rotate_wait(tw)
266
+ closer = TailWatcher::Closer.new(@rotate_wait, tw, log, &method(:close_watcher))
267
+ closer.attach(@loop)
268
+ end
269
+
270
+ def flush_buffer(tw)
271
+ if lb = tw.line_buffer
272
+ lb.chomp!
273
+ if @encoding
274
+ if @from_encoding
275
+ lb.encode!(@encoding, @from_encoding)
276
+ else
277
+ lb.force_encoding(@encoding)
278
+ end
279
+ end
280
+ @parser.parse(lb) { |time, record|
281
+ if time && record
282
+ tag = if @tag_prefix || @tag_suffix
283
+ @tag_prefix + tw.tag + @tag_suffix
284
+ else
285
+ @tag
286
+ end
287
+ record[@path_key] ||= tw.path unless @path_key.nil?
288
+ router.emit(tag, time, record)
289
+ else
290
+ log.warn "got incomplete line at shutdown from #{tw.path}: #{lb.inspect}"
291
+ end
292
+ }
293
+ end
294
+ end
295
+
296
+ def run
297
+ @loop.run
298
+ rescue
299
+ log.error "unexpected error", error: $!.to_s
300
+ log.error_backtrace
301
+ end
302
+
303
+ # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
304
+ def receive_lines(lines, tail_watcher)
305
+ es = @receive_handler.call(lines, tail_watcher)
306
+ unless es.empty?
307
+ tag = if @tag_prefix || @tag_suffix
308
+ @tag_prefix + tail_watcher.tag + @tag_suffix
309
+ else
310
+ @tag
311
+ end
312
+ begin
313
+ router.emit_stream(tag, es)
314
+ rescue Fluent::Plugin::Buffer::BufferOverflowError
315
+ return false
316
+ rescue
317
+ # ignore non BufferQueueLimitError errors because in_tail can't recover. Engine shows logs and backtraces.
318
+ return true
319
+ end
320
+ end
321
+
322
+ return true
323
+ end
324
+
325
+ def convert_line_to_event(line, es, tail_watcher)
326
+ begin
327
+ line.chomp! # remove \n
328
+ if @encoding
329
+ if @from_encoding
330
+ line.encode!(@encoding, @from_encoding)
331
+ else
332
+ line.force_encoding(@encoding)
333
+ end
334
+ end
335
+ @parser.parse(line) { |time, record|
336
+ if time && record
337
+ record[@path_key] ||= tail_watcher.path unless @path_key.nil?
338
+ es.add(time, record)
339
+ else
340
+ log.warn "pattern not match: #{line.inspect}"
341
+ end
342
+ }
343
+ rescue => e
344
+ log.warn line.dump, error: e.to_s
345
+ log.debug_backtrace(e.backtrace)
346
+ end
347
+ end
348
+
349
+ def parse_singleline(lines, tail_watcher)
350
+ es = MultiEventStream.new
351
+ lines.each { |line|
352
+ convert_line_to_event(line, es, tail_watcher)
353
+ }
354
+ es
355
+ end
356
+
357
+ def parse_multilines(lines, tail_watcher)
358
+ lb = tail_watcher.line_buffer
359
+ es = MultiEventStream.new
360
+ if @parser.has_firstline?
361
+ tail_watcher.line_buffer_timer_flusher.reset_timer if tail_watcher.line_buffer_timer_flusher
362
+ lines.each { |line|
363
+ if @parser.firstline?(line)
364
+ if lb
365
+ convert_line_to_event(lb, es, tail_watcher)
366
+ end
367
+ lb = line
368
+ else
369
+ if lb.nil?
370
+ log.warn "got incomplete line before first line from #{tail_watcher.path}: #{line.inspect}"
371
+ else
372
+ lb << line
373
+ end
374
+ end
375
+ }
376
+ else
377
+ lb ||= ''
378
+ lines.each do |line|
379
+ lb << line
380
+ @parser.parse(lb) { |time, record|
381
+ if time && record
382
+ convert_line_to_event(lb, es, tail_watcher)
383
+ lb = ''
384
+ end
385
+ }
386
+ end
387
+ end
388
+ tail_watcher.line_buffer = lb
389
+ es
390
+ end
391
+
392
+ class TailWatcher
393
+ def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, read_lines_limit, update_watcher, line_buffer_timer_flusher, &receive_lines)
394
+ @path = path
395
+ @rotate_wait = rotate_wait
396
+ @pe = pe || MemoryPositionEntry.new
397
+ @read_from_head = read_from_head
398
+ @enable_watch_timer = enable_watch_timer
399
+ @read_lines_limit = read_lines_limit
400
+ @receive_lines = receive_lines
401
+ @update_watcher = update_watcher
402
+
403
+ @timer_trigger = TimerWatcher.new(1, true, log, &method(:on_notify)) if @enable_watch_timer
404
+
405
+ @stat_trigger = StatWatcher.new(path, log, &method(:on_notify))
406
+
407
+ @rotate_handler = RotateHandler.new(path, log, &method(:on_rotate))
408
+ @io_handler = nil
409
+ @log = log
410
+
411
+ @line_buffer_timer_flusher = line_buffer_timer_flusher
412
+ end
413
+
414
+ attr_reader :path
415
+ attr_accessor :line_buffer, :line_buffer_timer_flusher
416
+ attr_accessor :unwatched # This is used for removing position entry from PositionFile
417
+
418
+ def tag
419
+ @parsed_tag ||= @path.tr('/', '.').gsub(/\.+/, '.').gsub(/^\./, '')
420
+ end
421
+
422
+ def wrap_receive_lines(lines)
423
+ @receive_lines.call(lines, self)
424
+ end
425
+
426
+ def attach(loop)
427
+ @timer_trigger.attach(loop) if @enable_watch_timer
428
+ @stat_trigger.attach(loop)
429
+ on_notify
430
+ end
431
+
432
+ def detach
433
+ @timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
434
+ @stat_trigger.detach if @stat_trigger.attached?
435
+ end
436
+
437
+ def close(close_io = true)
438
+ if close_io && @io_handler
439
+ @io_handler.on_notify
440
+ @io_handler.close
441
+ end
442
+ detach
443
+ end
444
+
445
+ def on_notify
446
+ @rotate_handler.on_notify if @rotate_handler
447
+ @line_buffer_timer_flusher.on_notify(self) if @line_buffer_timer_flusher
448
+ return unless @io_handler
449
+ @io_handler.on_notify
450
+ end
451
+
452
+ def on_rotate(io)
453
+ if @io_handler == nil
454
+ if io
455
+ # first time
456
+ stat = io.stat
457
+ fsize = stat.size
458
+ inode = stat.ino
459
+
460
+ last_inode = @pe.read_inode
461
+ if inode == last_inode
462
+ # rotated file has the same inode number with the last file.
463
+ # assuming following situation:
464
+ # a) file was once renamed and backed, or
465
+ # b) symlink or hardlink to the same file is recreated
466
+ # in either case, seek to the saved position
467
+ pos = @pe.read_pos
468
+ elsif last_inode != 0
469
+ # this is FilePositionEntry and fluentd once started.
470
+ # read data from the head of the rotated file.
471
+ # logs never duplicate because this file is a rotated new file.
472
+ pos = 0
473
+ @pe.update(inode, pos)
474
+ else
475
+ # this is MemoryPositionEntry or this is the first time fluentd started.
476
+ # seek to the end of the any files.
477
+ # logs may duplicate without this seek because it's not sure the file is
478
+ # existent file or rotated new file.
479
+ pos = @read_from_head ? 0 : fsize
480
+ @pe.update(inode, pos)
481
+ end
482
+ io.seek(pos)
483
+
484
+ @io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
485
+ else
486
+ @io_handler = NullIOHandler.new
487
+ end
488
+ else
489
+ log_msg = "detected rotation of #{@path}"
490
+ log_msg << "; waiting #{@rotate_wait} seconds" if @io_handler.io # wait rotate_time if previous file is exist
491
+ @log.info log_msg
492
+
493
+ if io
494
+ stat = io.stat
495
+ inode = stat.ino
496
+ if inode == @pe.read_inode # truncated
497
+ @pe.update_pos(stat.size)
498
+ io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
499
+ @io_handler.close
500
+ @io_handler = io_handler
501
+ elsif @io_handler.io.nil? # There is no previous file. Reuse TailWatcher
502
+ @pe.update(inode, io.pos)
503
+ io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
504
+ @io_handler = io_handler
505
+ else # file is rotated and new file found
506
+ @update_watcher.call(@path, swap_state(@pe))
507
+ end
508
+ else # file is rotated and new file not found
509
+ # Clear RotateHandler to avoid duplicated file watch in same path.
510
+ @rotate_handler = nil
511
+ @update_watcher.call(@path, swap_state(@pe))
512
+ end
513
+ end
514
+ end
515
+
516
+ def swap_state(pe)
517
+ # Use MemoryPositionEntry for rotated file temporary
518
+ mpe = MemoryPositionEntry.new
519
+ mpe.update(pe.read_inode, pe.read_pos)
520
+ @pe = mpe
521
+ @io_handler.pe = mpe # Don't re-create IOHandler because IOHandler has an internal buffer.
522
+
523
+ pe # This pe will be updated in on_rotate after TailWatcher is initialized
524
+ end
525
+
526
+ class TimerWatcher < Coolio::TimerWatcher
527
+ def initialize(interval, repeat, log, &callback)
528
+ @callback = callback
529
+ @log = log
530
+ super(interval, repeat)
531
+ end
532
+
533
+ def on_timer
534
+ @callback.call
535
+ rescue
536
+ # TODO log?
537
+ @log.error $!.to_s
538
+ @log.error_backtrace
539
+ end
540
+ end
541
+
542
+ class StatWatcher < Coolio::StatWatcher
543
+ def initialize(path, log, &callback)
544
+ @callback = callback
545
+ @log = log
546
+ super(path)
547
+ end
548
+
549
+ def on_change(prev, cur)
550
+ @callback.call
551
+ rescue
552
+ # TODO log?
553
+ @log.error $!.to_s
554
+ @log.error_backtrace
555
+ end
556
+ end
557
+
558
+ class Closer < Coolio::TimerWatcher
559
+ def initialize(interval, tw, log, &callback)
560
+ @callback = callback
561
+ @tw = tw
562
+ @log = log
563
+ super(interval, false)
564
+ end
565
+
566
+ def on_timer
567
+ @callback.call(@tw)
568
+ rescue => e
569
+ @log.error e.to_s
570
+ @log.error_backtrace(e.backtrace)
571
+ ensure
572
+ detach
573
+ end
574
+ end
575
+
576
+ class IOHandler
577
+ def initialize(io, pe, log, read_lines_limit, first = true, &receive_lines)
578
+ @log = log
579
+ @log.info "following tail of #{io.path}" if first
580
+ @io = io
581
+ @pe = pe
582
+ @read_lines_limit = read_lines_limit
583
+ @receive_lines = receive_lines
584
+ @buffer = ''.force_encoding('ASCII-8BIT')
585
+ @iobuf = ''.force_encoding('ASCII-8BIT')
586
+ @lines = []
587
+ end
588
+
589
+ attr_reader :io
590
+ attr_accessor :pe
591
+
592
+ def on_notify
593
+ begin
594
+ read_more = false
595
+
596
+ if @lines.empty?
597
+ begin
598
+ while true
599
+ if @buffer.empty?
600
+ @io.readpartial(2048, @buffer)
601
+ else
602
+ @buffer << @io.readpartial(2048, @iobuf)
603
+ end
604
+ while line = @buffer.slice!(/.*?\n/m)
605
+ @lines << line
606
+ end
607
+ if @lines.size >= @read_lines_limit
608
+ # not to use too much memory in case the file is very large
609
+ read_more = true
610
+ break
611
+ end
612
+ end
613
+ rescue EOFError
614
+ end
615
+ end
616
+
617
+ unless @lines.empty?
618
+ if @receive_lines.call(@lines)
619
+ @pe.update_pos(@io.pos - @buffer.bytesize)
620
+ @lines.clear
621
+ else
622
+ read_more = false
623
+ end
624
+ end
625
+ end while read_more
626
+
627
+ rescue
628
+ @log.error $!.to_s
629
+ @log.error_backtrace
630
+ close
631
+ end
632
+
633
+ def close
634
+ @io.close unless @io.closed?
635
+ end
636
+ end
637
+
638
+ class NullIOHandler
639
+ def initialize
640
+ end
641
+
642
+ def io
643
+ end
644
+
645
+ def on_notify
646
+ end
647
+
648
+ def close
649
+ end
650
+ end
651
+
652
+ class RotateHandler
653
+ def initialize(path, log, &on_rotate)
654
+ @path = path
655
+ @inode = nil
656
+ @fsize = -1 # first
657
+ @on_rotate = on_rotate
658
+ @log = log
659
+ end
660
+
661
+ def on_notify
662
+ begin
663
+ stat = FileWrapper.stat(@path)
664
+ inode = stat.ino
665
+ fsize = stat.size
666
+ rescue Errno::ENOENT
667
+ # moved or deleted
668
+ inode = nil
669
+ fsize = 0
670
+ end
671
+
672
+ begin
673
+ if @inode != inode || fsize < @fsize
674
+ # rotated or truncated
675
+ begin
676
+ io = FileWrapper.open(@path)
677
+ rescue Errno::ENOENT
678
+ end
679
+ @on_rotate.call(io)
680
+ end
681
+ @inode = inode
682
+ @fsize = fsize
683
+ end
684
+
685
+ rescue
686
+ @log.error $!.to_s
687
+ @log.error_backtrace
688
+ end
689
+ end
690
+
691
+
692
+ class LineBufferTimerFlusher
693
+ def initialize(log, flush_interval, &flush_method)
694
+ @log = log
695
+ @flush_interval = flush_interval
696
+ @flush_method = flush_method
697
+ @start = nil
698
+ end
699
+
700
+ def on_notify(tw)
701
+ if @start && @flush_interval
702
+ if Time.now - @start >= @flush_interval
703
+ @flush_method.call(tw)
704
+ tw.line_buffer = nil
705
+ @start = nil
706
+ end
707
+ end
708
+ end
709
+
710
+ def reset_timer
711
+ @start = Time.now
712
+ end
713
+ end
714
+ end
715
+
716
+
717
+ class PositionFile
718
+ UNWATCHED_POSITION = 0xffffffffffffffff
719
+
720
+ def initialize(file, map, last_pos)
721
+ @file = file
722
+ @map = map
723
+ @last_pos = last_pos
724
+ end
725
+
726
+ def [](path)
727
+ if m = @map[path]
728
+ return m
729
+ end
730
+
731
+ @file.pos = @last_pos
732
+ @file.write path
733
+ @file.write "\t"
734
+ seek = @file.pos
735
+ @file.write "0000000000000000\t0000000000000000\n"
736
+ @last_pos = @file.pos
737
+
738
+ @map[path] = FilePositionEntry.new(@file, seek)
739
+ end
740
+
741
+ def self.parse(file)
742
+ compact(file)
743
+
744
+ map = {}
745
+ file.pos = 0
746
+ file.each_line {|line|
747
+ m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
748
+ next unless m
749
+ path = m[1]
750
+ seek = file.pos - line.bytesize + path.bytesize + 1
751
+ map[path] = FilePositionEntry.new(file, seek)
752
+ }
753
+ new(file, map, file.pos)
754
+ end
755
+
756
+ # Clean up unwatched file entries
757
+ def self.compact(file)
758
+ file.pos = 0
759
+ existent_entries = file.each_line.map { |line|
760
+ m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
761
+ next unless m
762
+ path = m[1]
763
+ pos = m[2].to_i(16)
764
+ ino = m[3].to_i(16)
765
+ # 32bit inode converted to 64bit at this phase
766
+ pos == UNWATCHED_POSITION ? nil : ("%s\t%016x\t%016x\n" % [path, pos, ino])
767
+ }.compact
768
+
769
+ file.pos = 0
770
+ file.truncate(0)
771
+ file.write(existent_entries.join)
772
+ end
773
+ end
774
+
775
+ # pos inode
776
+ # ffffffffffffffff\tffffffffffffffff\n
777
+ class FilePositionEntry
778
+ POS_SIZE = 16
779
+ INO_OFFSET = 17
780
+ INO_SIZE = 16
781
+ LN_OFFSET = 33
782
+ SIZE = 34
783
+
784
+ def initialize(file, seek)
785
+ @file = file
786
+ @seek = seek
787
+ end
788
+
789
+ def update(ino, pos)
790
+ @file.pos = @seek
791
+ @file.write "%016x\t%016x" % [pos, ino]
792
+ end
793
+
794
+ def update_pos(pos)
795
+ @file.pos = @seek
796
+ @file.write "%016x" % pos
797
+ end
798
+
799
+ def read_inode
800
+ @file.pos = @seek + INO_OFFSET
801
+ raw = @file.read(16)
802
+ raw ? raw.to_i(16) : 0
803
+ end
804
+
805
+ def read_pos
806
+ @file.pos = @seek
807
+ raw = @file.read(16)
808
+ raw ? raw.to_i(16) : 0
809
+ end
810
+ end
811
+
812
+ class MemoryPositionEntry
813
+ def initialize
814
+ @pos = 0
815
+ @inode = 0
816
+ end
817
+
818
+ def update(ino, pos)
819
+ @inode = ino
820
+ @pos = pos
821
+ end
822
+
823
+ def update_pos(pos)
824
+ @pos = pos
825
+ end
826
+
827
+ def read_pos
828
+ @pos
829
+ end
830
+
831
+ def read_inode
832
+ @inode
833
+ end
834
+ end
835
+ end
836
+
837
+ NewTailInput = TailInput # for backward compatibility
838
+ end