fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,401 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'json'
18
+ require 'webrick'
19
+ require 'cgi'
20
+
21
+ require 'cool.io'
22
+
23
+ require 'fluent/plugin/input'
24
+ require 'fluent/plugin/output'
25
+ require 'fluent/plugin/multi_output'
26
+ require 'fluent/plugin/filter'
27
+
28
+ module Fluent::Plugin
29
+ class MonitorAgentInput < Input
30
+ Fluent::Plugin.register_input('monitor_agent', self)
31
+
32
+ helpers :timer, :thread
33
+
34
+ config_param :bind, :string, default: '0.0.0.0'
35
+ config_param :port, :integer, default: 24220
36
+ config_param :tag, :string, default: nil
37
+ config_param :emit_interval, :time, default: 60
38
+
39
+ class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
40
+ def initialize(server, agent)
41
+ @agent = agent
42
+ end
43
+
44
+ def do_GET(req, res)
45
+ begin
46
+ code, header, body = process(req, res)
47
+ rescue
48
+ code, header, body = render_json_error(500, {
49
+ 'message '=> 'Internal Server Error',
50
+ 'error' => "#{$!}",
51
+ 'backgrace'=> $!.backtrace,
52
+ })
53
+ end
54
+
55
+ # set response code, header and body
56
+ res.status = code
57
+ header.each_pair {|k,v|
58
+ res[k] = v
59
+ }
60
+ res.body = body
61
+ end
62
+
63
+ def build_object(req, res)
64
+ unless req.path_info == ""
65
+ return render_json_error(404, "Not found")
66
+ end
67
+
68
+ # parse ?=query string
69
+ if req.query_string
70
+ begin
71
+ qs = CGI.parse(req.query_string)
72
+ rescue
73
+ return render_json_error(400, "Invalid query string")
74
+ end
75
+ else
76
+ qs = Hash.new {|h,k| [] }
77
+ end
78
+
79
+ # if ?debug=1 is set, set :with_debug_info for get_monitor_info
80
+ # and :pretty_json for render_json_error
81
+ opts = {}
82
+ if s = qs['debug'] and s[0]
83
+ opts[:with_debug_info] = true
84
+ opts[:pretty_json] = true
85
+ end
86
+
87
+ if tag = get_search_parameter(qs, 'tag'.freeze)
88
+ # ?tag= to search an output plugin by match pattern
89
+ if obj = @agent.plugin_info_by_tag(tag, opts)
90
+ list = [obj]
91
+ else
92
+ list = []
93
+ end
94
+
95
+ elsif plugin_id = get_search_parameter(qs, '@id'.freeze)
96
+ # ?@id= to search a plugin by 'id <plugin_id>' config param
97
+ if obj = @agent.plugin_info_by_id(plugin_id, opts)
98
+ list = [obj]
99
+ else
100
+ list = []
101
+ end
102
+
103
+ elsif plugin_id = get_search_parameter(qs, 'id'.freeze)
104
+ # Without @ version of ?@id= for backward compatibility
105
+ if obj = @agent.plugin_info_by_id(plugin_id, opts)
106
+ list = [obj]
107
+ else
108
+ list = []
109
+ end
110
+
111
+ elsif plugin_type = get_search_parameter(qs, '@type'.freeze)
112
+ # ?@type= to search plugins by 'type <type>' config param
113
+ list = @agent.plugins_info_by_type(plugin_type, opts)
114
+
115
+ elsif plugin_type = get_search_parameter(qs, 'type'.freeze)
116
+ # Without @ version of ?@type= for backward compatibility
117
+ list = @agent.plugins_info_by_type(plugin_type, opts)
118
+
119
+ else
120
+ # otherwise show all plugins
121
+ list = @agent.plugins_info_all(opts)
122
+ end
123
+
124
+ return list, opts
125
+ end
126
+
127
+ def get_search_parameter(qs, param_name)
128
+ return nil unless qs.has_key?(param_name)
129
+ qs[param_name].first
130
+ end
131
+
132
+ def render_json(obj, opts={})
133
+ render_json_error(200, obj, opts)
134
+ end
135
+
136
+ def render_json_error(code, obj, opts={})
137
+ if opts[:pretty_json]
138
+ js = JSON.pretty_generate(obj)
139
+ else
140
+ js = obj.to_json
141
+ end
142
+ [code, {'Content-Type'=>'application/json'}, js]
143
+ end
144
+ end
145
+
146
+ class LTSVMonitorServlet < MonitorServlet
147
+ def process(req, res)
148
+ list, _opts = build_object(req, res)
149
+ return unless list
150
+
151
+ normalized = JSON.parse(list.to_json)
152
+
153
+ text = ''
154
+
155
+ normalized.map {|hash|
156
+ row = []
157
+ hash.each_pair {|k,v|
158
+ unless v.is_a?(Hash) || v.is_a?(Array)
159
+ row << "#{k}:#{v}"
160
+ end
161
+ }
162
+ text << row.join("\t") << "\n"
163
+ }
164
+
165
+ [200, {'Content-Type'=>'text/plain'}, text]
166
+ end
167
+ end
168
+
169
+ class JSONMonitorServlet < MonitorServlet
170
+ def process(req, res)
171
+ list, opts = build_object(req, res)
172
+ return unless list
173
+
174
+ render_json({
175
+ 'plugins' => list
176
+ }, opts)
177
+ end
178
+ end
179
+
180
+ class ConfigMonitorServlet < MonitorServlet
181
+ def build_object(req, res)
182
+ {
183
+ 'pid' => Process.pid,
184
+ 'ppid' => Process.ppid
185
+ }.merge(@agent.fluentd_opts)
186
+ end
187
+ end
188
+
189
+ class LTSVConfigMonitorServlet < ConfigMonitorServlet
190
+ def process(req, res)
191
+ result = build_object(req, res)
192
+
193
+ row = []
194
+ JSON.parse(result.to_json).each_pair { |k, v|
195
+ row << "#{k}:#{v}"
196
+ }
197
+ text = row.join("\t")
198
+
199
+ [200, {'Content-Type'=>'text/plain'}, text]
200
+ end
201
+ end
202
+
203
+ class JSONConfigMonitorServlet < ConfigMonitorServlet
204
+ def process(req, res)
205
+ result = build_object(req, res)
206
+ render_json(result)
207
+ end
208
+ end
209
+
210
+ def start
211
+ super
212
+
213
+ log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins"
214
+ @srv = WEBrick::HTTPServer.new({
215
+ BindAddress: @bind,
216
+ Port: @port,
217
+ Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
218
+ AccessLog: [],
219
+ })
220
+ @srv.mount('/api/plugins', LTSVMonitorServlet, self)
221
+ @srv.mount('/api/plugins.json', JSONMonitorServlet, self)
222
+ @srv.mount('/api/config', LTSVConfigMonitorServlet, self)
223
+ @srv.mount('/api/config.json', JSONConfigMonitorServlet, self)
224
+ thread_create :in_monitor_agent_servlet do
225
+ @srv.start
226
+ end
227
+ if @tag
228
+ log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
229
+
230
+ opts = {with_config: false}
231
+ timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
232
+ es = Fluent::MultiEventStream.new
233
+ now = Fluent::Engine.now
234
+ plugins_info_all(opts).each { |record|
235
+ es.add(now, record)
236
+ }
237
+ router.emit_stream(@tag, es)
238
+ }
239
+ end
240
+ end
241
+
242
+ def shutdown
243
+ if @srv
244
+ @srv.shutdown
245
+ @srv = nil
246
+ end
247
+
248
+ super
249
+ end
250
+
251
+ MONITOR_INFO = {
252
+ 'output_plugin' => ->(){ is_a?(::Fluent::Plugin::Output) },
253
+ 'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer); @buffer.queue.size },
254
+ 'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer); @buffer.stage_size },
255
+ 'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
256
+ }
257
+
258
+ def all_plugins
259
+ array = []
260
+
261
+ # get all input plugins
262
+ array.concat Fluent::Engine.root_agent.inputs
263
+
264
+ # get all output plugins
265
+ Fluent::Engine.root_agent.outputs.each { |o|
266
+ MonitorAgentInput.collect_children(o, array)
267
+ }
268
+ # get all filter plugins
269
+ Fluent::Engine.root_agent.filters.each { |f|
270
+ MonitorAgentInput.collect_children(f, array)
271
+ }
272
+ Fluent::Engine.root_agent.labels.each { |name, l|
273
+ # TODO: Add label name to outputs / filters for identifing plugins
274
+ l.outputs.each { |o| MonitorAgentInput.collect_children(o, array) }
275
+ l.filters.each { |f| MonitorAgentInput.collect_children(f, array) }
276
+ }
277
+
278
+ array
279
+ end
280
+
281
+ # get nexted plugins (such as <store> of the copy plugin)
282
+ # from the plugin `pe` recursively
283
+ def self.collect_children(pe, array=[])
284
+ array << pe
285
+ if pe.is_a?(Fluent::Plugin::MultiOutput) || pe.is_a?(Fluent::MultiOutput) && pe.respond_to?(:outputs)
286
+ pe.outputs.each {|nop|
287
+ collect_children(nop, array)
288
+ }
289
+ end
290
+ array
291
+ end
292
+
293
+ # try to match the tag and get the info from the matched output plugin
294
+ # TODO: Support output in label
295
+ def plugin_info_by_tag(tag, opts={})
296
+ matches = Fluent::Engine.root_agent.event_router.instance_variable_get(:@match_rules)
297
+ matches.each { |rule|
298
+ if rule.match?(tag)
299
+ if rule.collector.is_a?(Fluent::Plugin::Output) || rule.collector.is_a?(Fluent::Output)
300
+ return get_monitor_info(rule.collector, opts)
301
+ end
302
+ end
303
+ }
304
+ nil
305
+ end
306
+
307
+ # search a plugin by plugin_id
308
+ def plugin_info_by_id(plugin_id, opts={})
309
+ found = all_plugins.find {|pe|
310
+ pe.respond_to?(:plugin_id) && pe.plugin_id.to_s == plugin_id
311
+ }
312
+ if found
313
+ get_monitor_info(found, opts)
314
+ else
315
+ nil
316
+ end
317
+ end
318
+
319
+ # This method returns an array because
320
+ # multiple plugins could have the same type
321
+ def plugins_info_by_type(type, opts={})
322
+ array = all_plugins.select {|pe|
323
+ (pe.config['@type'] == type) rescue nil
324
+ }
325
+ array.map {|pe|
326
+ get_monitor_info(pe, opts)
327
+ }
328
+ end
329
+
330
+ def plugins_info_all(opts={})
331
+ all_plugins.map {|pe|
332
+ get_monitor_info(pe, opts)
333
+ }
334
+ end
335
+
336
+ IGNORE_ATTRIBUTES = %i(@config_root_section @config @masked_config)
337
+
338
+ # get monitor info from the plugin `pe` and return a hash object
339
+ def get_monitor_info(pe, opts={})
340
+ obj = {}
341
+
342
+ # Common plugin information
343
+ obj['plugin_id'] = pe.plugin_id
344
+ obj['plugin_category'] = plugin_category(pe)
345
+ obj['type'] = pe.config['@type']
346
+ obj['config'] = pe.config if !opts.has_key?(:with_config) || opts[:with_config]
347
+
348
+ # run MONITOR_INFO in plugins' instance context and store the info to obj
349
+ MONITOR_INFO.each_pair {|key,code|
350
+ begin
351
+ catch(:skip) do
352
+ obj[key] = pe.instance_exec(&code)
353
+ end
354
+ rescue => e
355
+ log.warn "unexpected error in monitoring plugins", key: key, plugin: pe.class, error: e
356
+ end
357
+ }
358
+
359
+ # include all instance variables if :with_debug_info is set
360
+ if opts[:with_debug_info]
361
+ iv = {}
362
+ pe.instance_eval do
363
+ instance_variables.each {|sym|
364
+ next if IGNORE_ATTRIBUTES.include?(sym)
365
+ key = sym.to_s[1..-1] # removes first '@'
366
+ iv[key] = instance_variable_get(sym)
367
+ }
368
+ end
369
+ obj['instance_variables'] = iv
370
+ end
371
+
372
+ obj
373
+ end
374
+
375
+ def plugin_category(pe)
376
+ case pe
377
+ when Fluent::Plugin::Input
378
+ 'input'.freeze
379
+ when Fluent::Plugin::Output, Fluent::Plugin::BareOutput
380
+ 'output'.freeze
381
+ when Fluent::Plugin::Filter
382
+ 'filter'.freeze
383
+ else
384
+ 'unknown'.freeze
385
+ end
386
+ end
387
+
388
+ def fluentd_opts
389
+ @fluentd_opts ||= get_fluentd_opts
390
+ end
391
+
392
+ def get_fluentd_opts
393
+ opts = {}
394
+ ObjectSpace.each_object(Fluent::Supervisor) { |obj|
395
+ opts.merge!(obj.options)
396
+ break
397
+ }
398
+ opts
399
+ end
400
+ end
401
+ end
@@ -0,0 +1,90 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'cool.io'
18
+ require 'yajl'
19
+
20
+ require 'fluent/plugin/input'
21
+
22
+ module Fluent::Plugin
23
+ class ObjectSpaceInput < Fluent::Plugin::Input
24
+ Fluent::Plugin.register_input('object_space', self)
25
+
26
+ helpers :timer
27
+
28
+ def initialize
29
+ super
30
+ end
31
+
32
+ config_param :emit_interval, :time, default: 60
33
+ config_param :tag, :string
34
+ config_param :top, :integer, default: 15
35
+
36
+ def start
37
+ super
38
+
39
+ timer_execute(:object_space_input, @emit_interval, &method(:on_timer))
40
+ end
41
+
42
+ class Counter
43
+ def initialize(klass, init_count)
44
+ @klass = klass
45
+ @count = init_count
46
+ end
47
+
48
+ def incr!
49
+ @count += 1
50
+ end
51
+
52
+ def name
53
+ @klass.name
54
+ end
55
+
56
+ attr_reader :count
57
+ end
58
+
59
+ def on_timer
60
+ now = Fluent::EventTime.now
61
+
62
+ array = []
63
+ map = {}
64
+
65
+ ObjectSpace.each_object {|obj|
66
+ klass = obj.class rescue Object
67
+ if c = map[klass]
68
+ c.incr!
69
+ else
70
+ c = Counter.new(klass, 1)
71
+ array << c
72
+ map[klass] = c
73
+ end
74
+ }
75
+
76
+ array.sort_by! {|c| -c.count }
77
+
78
+ record = {}
79
+ array.each_with_index {|c,i|
80
+ break if i >= @top
81
+ record[c.name] = c.count
82
+ }
83
+
84
+ router.emit(@tag, now, record)
85
+ rescue => e
86
+ log.error "object space failed to emit", error: e, tag: @tag, record: Yajl.dump(record)
87
+ log.error_backtrace
88
+ end
89
+ end
90
+ end