fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,190 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'strscan'
18
+ require 'uri'
19
+
20
+ require 'fluent/config/error'
21
+ require 'fluent/config/basic_parser'
22
+ require 'fluent/config/literal_parser'
23
+ require 'fluent/config/element'
24
+
25
+ module Fluent
26
+ module Config
27
+ class V1Parser < LiteralParser
28
+ ELEMENT_NAME = /[a-zA-Z0-9_]+/
29
+
30
+ def self.parse(data, fname, basepath = Dir.pwd, eval_context = nil)
31
+ ss = StringScanner.new(data)
32
+ ps = V1Parser.new(ss, basepath, fname, eval_context)
33
+ ps.parse!
34
+ end
35
+
36
+ def initialize(strscan, include_basepath, fname, eval_context)
37
+ super(strscan, eval_context)
38
+ @include_basepath = include_basepath
39
+ @fname = fname
40
+ end
41
+
42
+ def parse!
43
+ attrs, elems = parse_element(true, nil)
44
+ root = Element.new('ROOT', '', attrs, elems)
45
+ root.v1_config = true
46
+
47
+ spacing
48
+ unless eof?
49
+ parse_error! "expected EOF"
50
+ end
51
+
52
+ return root
53
+ end
54
+
55
+ ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
56
+ RESERVED_PARAMS = %W(@type @id @label @log_level)
57
+
58
+ def parse_element(root_element, elem_name, attrs = {}, elems = [])
59
+ while true
60
+ spacing
61
+ if eof?
62
+ if root_element
63
+ break
64
+ end
65
+ parse_error! "expected end tag '</#{elem_name}>' but got end of file"
66
+ end
67
+
68
+ if skip(/\<\//)
69
+ e_name = scan(ELEMENT_NAME)
70
+ spacing
71
+ unless skip(/\>/)
72
+ parse_error! "expected character in tag name"
73
+ end
74
+ unless line_end
75
+ parse_error! "expected end of line after end tag"
76
+ end
77
+ if e_name != elem_name
78
+ parse_error! "unmatched end tag"
79
+ end
80
+ break
81
+
82
+ elsif skip(/\</)
83
+ e_name = scan(ELEMENT_NAME)
84
+ spacing
85
+ e_arg = scan_nonquoted_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
86
+ spacing
87
+ unless skip(/\>/)
88
+ parse_error! "expected '>'"
89
+ end
90
+ unless line_end
91
+ parse_error! "expected end of line after tag"
92
+ end
93
+ e_arg ||= ''
94
+ # call parse_element recursively
95
+ e_attrs, e_elems = parse_element(false, e_name)
96
+ new_e = Element.new(e_name, e_arg, e_attrs, e_elems)
97
+ new_e.v1_config = true
98
+ elems << new_e
99
+
100
+ elsif root_element && skip(/(\@include|include)#{SPACING}/)
101
+ if !prev_match.start_with?('@')
102
+ $log.warn "'include' is deprecated. Use '@include' instead"
103
+ end
104
+ parse_include(attrs, elems)
105
+
106
+ else
107
+ k = scan_string(SPACING)
108
+ spacing_without_comment
109
+ if prev_match.include?("\n") # support 'tag_mapped' like "without value" configuration
110
+ attrs[k] = ""
111
+ else
112
+ if k == '@include'
113
+ parse_include(attrs, elems)
114
+ elsif RESERVED_PARAMS.include?(k)
115
+ v = parse_literal
116
+ unless line_end
117
+ parse_error! "expected end of line"
118
+ end
119
+ attrs[k] = v
120
+ else
121
+ if k.start_with?('@')
122
+ if root_element || ELEM_SYMBOLS.include?(elem_name)
123
+ parse_error! "'@' is the system reserved prefix. Don't use '@' prefix parameter in the configuration: #{k}"
124
+ else
125
+ # TODO: This is for backward compatibility. It will throw an error in the future.
126
+ $log.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}"
127
+ end
128
+ end
129
+
130
+ v = parse_literal
131
+ unless line_end
132
+ parse_error! "expected end of line"
133
+ end
134
+ attrs[k] = v
135
+ end
136
+ end
137
+ end
138
+ end
139
+
140
+ return attrs, elems
141
+ end
142
+
143
+ def parse_include(attrs, elems)
144
+ uri = scan_string(LINE_END)
145
+ eval_include(attrs, elems, uri)
146
+ line_end
147
+ end
148
+
149
+ def eval_include(attrs, elems, uri)
150
+ u = URI.parse(uri)
151
+ if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri # file path
152
+ # When the Windows absolute path then u.scheme.length == 1
153
+ # e.g. C:
154
+ path = u.path
155
+ if path[0] != ?/
156
+ pattern = File.expand_path("#{@include_basepath}/#{path}")
157
+ else
158
+ pattern = path
159
+ end
160
+ Dir.glob(pattern).sort.each { |entry|
161
+ basepath = File.dirname(entry)
162
+ fname = File.basename(entry)
163
+ data = File.read(entry)
164
+ data.force_encoding('UTF-8')
165
+ ss = StringScanner.new(data)
166
+ V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
167
+ }
168
+ else
169
+ require 'open-uri'
170
+ basepath = '/'
171
+ fname = path
172
+ data = open(uri) { |f| f.read }
173
+ data.force_encoding('UTF-8')
174
+ ss = StringScanner.new(data)
175
+ V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
176
+ end
177
+
178
+ rescue SystemCallError => e
179
+ cpe = ConfigParseError.new("include error #{uri}")
180
+ cpe.set_backtrace(e.backtrace)
181
+ raise cpe
182
+ end
183
+
184
+ # override
185
+ def error_sample
186
+ "#{@fname} #{super}"
187
+ end
188
+ end
189
+ end
190
+ end
@@ -0,0 +1,176 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/config/configure_proxy'
18
+ require 'fluent/config/section'
19
+ require 'fluent/config/error'
20
+ require 'fluent/registry'
21
+ require 'fluent/plugin'
22
+ require 'fluent/config/types'
23
+
24
+ module Fluent
25
+ module Configurable
26
+ def self.included(mod)
27
+ mod.extend(ClassMethods)
28
+ end
29
+
30
+ def initialize
31
+ super
32
+ # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
33
+ proxy = self.class.merged_configure_proxy
34
+ proxy.params.keys.each do |name|
35
+ next if name.to_s.start_with?('@')
36
+ if proxy.defaults.has_key?(name)
37
+ instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
38
+ end
39
+ end
40
+ proxy.sections.keys.each do |name|
41
+ next if name.to_s.start_with?('@')
42
+ subproxy = proxy.sections[name]
43
+ if subproxy.multi?
44
+ instance_variable_set("@#{subproxy.variable_name}".to_sym, [])
45
+ else
46
+ instance_variable_set("@#{subproxy.variable_name}".to_sym, nil)
47
+ end
48
+ end
49
+ end
50
+
51
+ def configure(conf)
52
+ @config = conf
53
+
54
+ logger = self.respond_to?(:log) ? log : (defined?($log) ? $log : nil)
55
+ proxy = self.class.merged_configure_proxy
56
+ conf.corresponding_proxies << proxy
57
+
58
+ if self.respond_to?(:owner) && self.owner
59
+ owner_proxy = owner.class.merged_configure_proxy
60
+ if proxy.configured_in_section
61
+ owner_proxy = owner_proxy.sections[proxy.configured_in_section]
62
+ end
63
+ proxy.overwrite_defaults(owner_proxy) if owner_proxy
64
+ end
65
+
66
+ # In the nested section, can't get plugin class through proxies so get plugin class here
67
+ plugin_class = Fluent::Plugin.lookup_type_from_class(proxy.name.to_s)
68
+ root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
69
+ @config_root_section = root
70
+
71
+ root.instance_eval{ @params.keys }.each do |param_name|
72
+ next if param_name.to_s.start_with?('@')
73
+ varname = "@#{param_name}".to_sym
74
+ if (! root[param_name].nil?) || (instance_variable_defined?(varname) && instance_variable_get(varname).nil?)
75
+ instance_variable_set(varname, root[param_name])
76
+ end
77
+ end
78
+
79
+ self
80
+ end
81
+
82
+ def config
83
+ @masked_config ||= @config.to_masked_element
84
+ end
85
+
86
+ CONFIG_TYPE_REGISTRY = Registry.new(:config_type, 'fluent/plugin/type_')
87
+
88
+ def self.register_type(type, callable = nil, &block)
89
+ callable ||= block
90
+ CONFIG_TYPE_REGISTRY.register(type, callable)
91
+ end
92
+
93
+ def self.lookup_type(type)
94
+ CONFIG_TYPE_REGISTRY.lookup(type)
95
+ end
96
+
97
+ {
98
+ string: Config::STRING_TYPE,
99
+ enum: Config::ENUM_TYPE,
100
+ integer: Config::INTEGER_TYPE,
101
+ float: Config::FLOAT_TYPE,
102
+ size: Config::SIZE_TYPE,
103
+ bool: Config::BOOL_TYPE,
104
+ time: Config::TIME_TYPE,
105
+ hash: Config::HASH_TYPE,
106
+ array: Config::ARRAY_TYPE,
107
+ }.each do |name, type|
108
+ register_type(name, type)
109
+ end
110
+
111
+ module ClassMethods
112
+ def configure_proxy_map
113
+ map = {}
114
+ self.define_singleton_method(:configure_proxy_map){ map }
115
+ map
116
+ end
117
+
118
+ def configure_proxy(mod_name)
119
+ map = configure_proxy_map
120
+ unless map[mod_name]
121
+ type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) }
122
+ proxy = Fluent::Config::ConfigureProxy.new(mod_name, root: true, required: true, multi: false, type_lookup: type_lookup)
123
+ map[mod_name] = proxy
124
+ end
125
+ map[mod_name]
126
+ end
127
+
128
+ def configured_in(section_name)
129
+ configure_proxy(self.name).configured_in(section_name)
130
+ end
131
+
132
+ def config_param(name, type = nil, **kwargs, &block)
133
+ configure_proxy(self.name).config_param(name, type, **kwargs, &block)
134
+ # reserved names '@foo' are invalid as attr_accessor name
135
+ attr_accessor(name) unless kwargs[:skip_accessor] || Fluent::Config::Element::RESERVED_PARAMETERS.include?(name.to_s)
136
+ end
137
+
138
+ def config_set_default(name, defval)
139
+ configure_proxy(self.name).config_set_default(name, defval)
140
+ end
141
+
142
+ def config_set_desc(name, desc)
143
+ configure_proxy(self.name).config_set_desc(name, desc)
144
+ end
145
+
146
+ def config_section(name, **kwargs, &block)
147
+ section_already_exists = !!merged_configure_proxy.sections[name]
148
+ configure_proxy(self.name).config_section(name, **kwargs, &block)
149
+ variable_name = configure_proxy(self.name).sections[name].variable_name
150
+ if !section_already_exists && !self.respond_to?(variable_name)
151
+ attr_accessor variable_name
152
+ end
153
+ end
154
+
155
+ def desc(description)
156
+ configure_proxy(self.name).desc(description)
157
+ end
158
+
159
+ def merged_configure_proxy
160
+ configurables = ancestors.reverse.select{ |a| a.respond_to?(:configure_proxy) }
161
+
162
+ # 'a.object_id.to_s' is to support anonymous class
163
+ # which created in tests to overwrite original behavior temporally
164
+ #
165
+ # p Module.new.name #=> nil
166
+ # p Class.new.name #=> nil
167
+ # p AnyGreatClass.dup.name #=> nil
168
+ configurables.map{ |a| a.configure_proxy(a.name || a.object_id.to_s) }.reduce(:merge)
169
+ end
170
+
171
+ def dump(level = 0)
172
+ configure_proxy_map[self.to_s].dump(level)
173
+ end
174
+ end
175
+ end
176
+ end
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env ruby
2
+ # -*- coding: utf-8 -*-
3
+
4
+ here = File.dirname(__FILE__)
5
+ $LOAD_PATH << File.expand_path(File.join(here, '..'))
6
+
7
+ require 'serverengine'
8
+ require 'fluent/supervisor'
9
+
10
+ server_module = Fluent.const_get(ARGV[0])
11
+ worker_module = Fluent.const_get(ARGV[1])
12
+ # it doesn't call ARGV in block because when reloading config, params will be initialized and then it can't use previous config.
13
+ config_path = ARGV[2]
14
+ params = JSON.parse(ARGV[3])
15
+ ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.load_config(config_path, params) }
@@ -0,0 +1,220 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'socket'
18
+
19
+ require 'cool.io'
20
+
21
+ require 'fluent/config'
22
+ require 'fluent/event'
23
+ require 'fluent/event_router'
24
+ require 'fluent/msgpack_factory'
25
+ require 'fluent/root_agent'
26
+ require 'fluent/time'
27
+ require 'fluent/system_config'
28
+ require 'fluent/plugin'
29
+
30
+ module Fluent
31
+ class EngineClass
32
+ include Fluent::MessagePackFactory::Mixin
33
+
34
+ def initialize
35
+ @root_agent = nil
36
+ @event_router = nil
37
+ @default_loop = nil
38
+ @engine_stopped = false
39
+
40
+ @log_emit_thread = nil
41
+ @log_event_loop_stop = false
42
+ @log_event_queue = []
43
+
44
+ @suppress_config_dump = false
45
+
46
+ @system_config = SystemConfig.new
47
+ end
48
+
49
+ MAINLOOP_SLEEP_INTERVAL = 0.3
50
+
51
+ MATCH_CACHE_SIZE = 1024
52
+ LOG_EMIT_INTERVAL = 0.1
53
+
54
+ attr_reader :root_agent
55
+ attr_reader :matches, :sources
56
+ attr_reader :system_config
57
+
58
+ def init(system_config)
59
+ @system_config = system_config
60
+
61
+ BasicSocket.do_not_reverse_lookup = true
62
+
63
+ suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
64
+ @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
65
+ @without_source = system_config.without_source unless system_config.without_source.nil?
66
+
67
+ @root_agent = RootAgent.new(log: log, system_config: @system_config)
68
+
69
+ MessagePackFactory.init
70
+
71
+ self
72
+ end
73
+
74
+ def log
75
+ $log
76
+ end
77
+
78
+ def suppress_interval(interval_time)
79
+ @suppress_emit_error_log_interval = interval_time
80
+ @next_emit_error_log_time = Time.now.to_i
81
+ end
82
+
83
+ def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
84
+ if fname =~ /\.rb$/
85
+ require 'fluent/config/dsl'
86
+ Config::DSL::Parser.parse(io, File.join(basepath, fname))
87
+ else
88
+ Config.parse(io, fname, basepath, v1_config)
89
+ end
90
+ end
91
+
92
+ def run_configure(conf)
93
+ configure(conf)
94
+ conf.check_not_fetched { |key, e|
95
+ parent_name, plugin_name = e.unused_in
96
+ if parent_name
97
+ message = if plugin_name
98
+ "section <#{e.name}> is not used in <#{parent_name}> of #{plugin_name} plugin"
99
+ else
100
+ "section <#{e.name}> is not used in <#{parent_name}>"
101
+ end
102
+ $log.warn message
103
+ next
104
+ end
105
+ unless e.name == 'system'
106
+ unless @without_source && e.name == 'source'
107
+ $log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."
108
+ end
109
+ end
110
+ }
111
+ end
112
+
113
+ def configure(conf)
114
+ # plugins / configuration dumps
115
+ Gem::Specification.find_all.select{|x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/}.each do |spec|
116
+ $log.info "gem '#{spec.name}' version '#{spec.version}'"
117
+ end
118
+
119
+ @root_agent.configure(conf)
120
+ @event_router = @root_agent.event_router
121
+
122
+ unless @suppress_config_dump
123
+ $log.info "using configuration file: #{conf.to_s.rstrip}"
124
+ end
125
+ end
126
+
127
+ def add_plugin_dir(dir)
128
+ Plugin.add_plugin_dir(dir)
129
+ end
130
+
131
+ def emit(tag, time, record)
132
+ raise "BUG: use router.emit instead of Engine.emit"
133
+ end
134
+
135
+ def emit_array(tag, array)
136
+ raise "BUG: use router.emit_array instead of Engine.emit_array"
137
+ end
138
+
139
+ def emit_stream(tag, es)
140
+ raise "BUG: use router.emit_stream instead of Engine.emit_stream"
141
+ end
142
+
143
+ def flush!
144
+ @root_agent.flush!
145
+ end
146
+
147
+ def now
148
+ # TODO thread update
149
+ Fluent::EventTime.now
150
+ end
151
+
152
+ def log_event_loop
153
+ $log.disable_events(Thread.current)
154
+
155
+ while sleep(LOG_EMIT_INTERVAL)
156
+ break if @log_event_loop_stop
157
+ next if @log_event_queue.empty?
158
+
159
+ # NOTE: thead-safe of slice! depends on GVL
160
+ events = @log_event_queue.slice!(0..-1)
161
+ next if events.empty?
162
+
163
+ events.each {|tag,time,record|
164
+ begin
165
+ @event_router.emit(tag, time, record)
166
+ rescue => e
167
+ $log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
168
+ end
169
+ }
170
+ end
171
+ end
172
+
173
+ def run
174
+ begin
175
+ start
176
+
177
+ if @event_router.match?($log.tag)
178
+ $log.enable_event
179
+ @log_emit_thread = Thread.new(&method(:log_event_loop))
180
+ end
181
+
182
+ sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
183
+
184
+ rescue Exception => e
185
+ $log.error "unexpected error", error: e
186
+ $log.error_backtrace
187
+ raise
188
+ end
189
+
190
+ $log.info "shutting down fluentd"
191
+ shutdown
192
+ if @log_emit_thread
193
+ @log_event_loop_stop = true
194
+ @log_emit_thread.join
195
+ end
196
+ end
197
+
198
+ def stop
199
+ @engine_stopped = true
200
+ nil
201
+ end
202
+
203
+ def push_log_event(tag, time, record)
204
+ return if @log_emit_thread.nil?
205
+ @log_event_queue.push([tag, time, record])
206
+ end
207
+
208
+ private
209
+
210
+ def start
211
+ @root_agent.start
212
+ end
213
+
214
+ def shutdown
215
+ @root_agent.shutdown
216
+ end
217
+ end
218
+
219
+ Engine = EngineClass.new
220
+ end