fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,23 @@
1
+ <source>
2
+ @type dummy
3
+ tag test
4
+ </source>
5
+
6
+ <source>
7
+ @type monitor_agent
8
+ emit_interval 5
9
+ </source>
10
+
11
+ <match test>
12
+ @type forward
13
+ buffer_path /tmp/fluentd.forward
14
+ buffer_type file
15
+ flush_interval 5
16
+ send_timeout 60
17
+ heartbeat_type tcp
18
+ heartbeat_interval 1
19
+ <server>
20
+ host 127.0.0.1
21
+ port 24224
22
+ </server>
23
+ </match>
@@ -0,0 +1,78 @@
1
+ # An example config to use filter plugins.
2
+ # THIS FEATURE IS SUPPORTED FOR v0.12 AND ABOVE.
3
+
4
+ # in_forward to generate events to be tested.
5
+ # You can send an arbitrary event with an arbitrary tag.
6
+ # For example, the following command creates the event
7
+ # {"message":"hello world"} with tag = foo
8
+ #
9
+ # $ echo '{"message":"hello world"}' | fluent-cat foo
10
+
11
+ <source>
12
+ @type forward
13
+ port 24224
14
+ </source>
15
+
16
+ # For all events with the tag "foo", filter it out
17
+ # UNLESS the value of the "message" field matches /keep this/
18
+ #
19
+ # - {"message":"keep this please"} is kept.
20
+ # - {"message":"Do not keep"} is filtered out.
21
+ # - {"messag22":"keep this please"} is filtered out.
22
+
23
+ <filter foo>
24
+ @type grep
25
+ regexp1 message keep this
26
+ </filter>
27
+
28
+ # Matches the events that was kept by the above filter
29
+ <match foo>
30
+ @type stdout
31
+ </match>
32
+
33
+ # For all events with the tag "bar", add the machine's hostname with
34
+ # the key "hostname" BEFORE forwarding to another instance of Fluentd
35
+ # at 123.4.2.4:24224.
36
+
37
+ <filter bar>
38
+ @type record_transformer
39
+ <record>
40
+ hostname ${hostname}
41
+ </record>
42
+ </filter>
43
+
44
+ # By the time it is getting matched here, the event has
45
+ # the "hostname" field.
46
+ <match bar>
47
+ @type forward
48
+ <server>
49
+ host 123.4.2.4
50
+ port 24225
51
+ </server>
52
+ </match>
53
+
54
+ # Composing two filters. For all events with the tag foo.bar,
55
+ #
56
+ # 1. The first filter filters out all events that has the field "hello"
57
+ # 2. Then, for those events, we downcase the value of the "name" field.
58
+ #
59
+ # - {"name":"SADA", "hello":100} gets filtered out
60
+ # - {"name":"SADA"} becomes {"name":"sada"}
61
+ # - {"last_name":"FURUHASHI"} throws an error because it has no field called "name"
62
+
63
+ <filter foo.bar>
64
+ @type grep
65
+ exclude1 hello .
66
+ </filter>
67
+
68
+ <filter foo.bar>
69
+ @type record_transformer
70
+ enable_ruby true
71
+ <record>
72
+ name ${name.downcase}
73
+ </record>
74
+ </filter>
75
+
76
+ <match foo.bar>
77
+ @type stdout
78
+ </match>
@@ -0,0 +1,36 @@
1
+ <section1>
2
+ key1 'text' # text
3
+ key2 '\'' # ' (1 char)
4
+ key3 '\\' # \ (1 char)
5
+ key4 '\t' # \t (2 char)
6
+ key5 '\[' # \[ (2 char)
7
+ key6 '\\[' # \[ (2 char)
8
+ key7 '#t' # #t (2 char)
9
+ key8 '\#{test}' # \#{test} (8 char)
10
+ key9 '#{test}' # #{test} (7 char)
11
+ key10 '\[(?<time>[^\]]*)\] (?<message>.*)' # \[(?<time>[^\]]*\] (?<message>.*)
12
+ </section1>
13
+ <section2>
14
+ key1 "text" # text
15
+ key2 "\"" # " (1 char)
16
+ key3 "\\" # \ (1 char)
17
+ key4 "\t" # TAB (1 char)
18
+ key5 "\[" # [ (1 char)
19
+ key6 "\\[" # \[ (2 char)
20
+ key7 "#t" # #t (2 char)
21
+ key8 "\#{test}" # #{test} (7 char)
22
+ key9 "#{test}" # replaced by eval('test')
23
+ key10 "\\[(?<time>[^\\]]*)\\] (?<message>.*)" # \[(?<time>[^\]]*\] (?<message>.*)
24
+ </section2>
25
+ <section3>
26
+ key1 text # text
27
+ key2 \ # \ (1 char)
28
+ key3 \\ # \\ (2 char)
29
+ key4 \t # \t (2 char)
30
+ key5 \[ # \[ (2 char)
31
+ key6 \\[ # \\[ (3 char)
32
+ key7 #t # #t (2 char)
33
+ key8 \#{test} # \#{test} (8 char)
34
+ key9 #{test} # #{test} (7 char)
35
+ key10 \[(?<time>[^\]]*)\] (?<message>.*) # \[(?<time>[^\]]*\] (?<message>.*)
36
+ </section3>
@@ -0,0 +1,139 @@
1
+ # In v1 configuration, type and id are @ prefix parameters.
2
+ # @type and @id are recommended. type and id are still available for backward compatibility
3
+
4
+ ## built-in TCP input
5
+ ## $ echo <json> | fluent-cat <tag>
6
+ <source>
7
+ @type forward
8
+ @id forward_input
9
+ </source>
10
+
11
+ ## built-in UNIX socket input
12
+ #<source>
13
+ # @type unix
14
+ #</source>
15
+
16
+ # HTTP input
17
+ # http://localhost:8888/<tag>?json=<json>
18
+ <source>
19
+ @type http
20
+ @id http_input
21
+
22
+ port 8888
23
+ </source>
24
+
25
+ ## File input
26
+ ## read apache logs with tag=apache.access
27
+ #<source>
28
+ # @type tail
29
+ # format apache
30
+ # path /var/log/httpd-access.log
31
+ # tag apache.access
32
+ #</source>
33
+
34
+ ## Mutating event filter
35
+ ## Add hostname and tag fields to apache.access tag events
36
+ #<filter apache.access>
37
+ # @type record_transformer
38
+ # <record>
39
+ # hostname ${hostname}
40
+ # tag ${tag}
41
+ # </record>
42
+ #</filter>
43
+
44
+ ## Selecting event filter
45
+ ## Remove unnecessary events from apache prefixed tag events
46
+ #<filter apache.**>
47
+ # @type grep
48
+ # include1 method GET # pass only GET in 'method' field
49
+ # exclude1 message debug # remove debug event
50
+ #</filter>
51
+
52
+ # Listen HTTP for monitoring
53
+ # http://localhost:24220/api/plugins
54
+ # http://localhost:24220/api/plugins?type=TYPE
55
+ # http://localhost:24220/api/plugins?tag=MYTAG
56
+ <source>
57
+ @type monitor_agent
58
+ @id monitor_agent_input
59
+
60
+ port 24220
61
+ </source>
62
+
63
+ # Listen DRb for debug
64
+ <source>
65
+ @type debug_agent
66
+ @id debug_agent_input
67
+
68
+ bind 127.0.0.1
69
+ port 24230
70
+ </source>
71
+
72
+ ## match tag=apache.access and write to file
73
+ #<match apache.access>
74
+ # @type file
75
+ # path /var/log/fluent/access
76
+ #</match>
77
+
78
+ ## match tag=debug.** and dump to console
79
+ <match debug.**>
80
+ @type stdout
81
+ @id stdout_output
82
+ </match>
83
+
84
+ # match tag=system.** and forward to another fluent server
85
+ <match system.**>
86
+ @type forward
87
+ @id forward_output
88
+
89
+ <server>
90
+ host 192.168.0.11
91
+ </server>
92
+ <secondary>
93
+ <server>
94
+ host 192.168.0.12
95
+ </server>
96
+ </secondary>
97
+ </match>
98
+
99
+ ## match tag=myapp.** and forward and write to file
100
+ #<match myapp.**>
101
+ # @type copy
102
+ # <store>
103
+ # @type forward
104
+ # buffer_type file
105
+ # buffer_path /var/log/fluent/myapp-forward
106
+ # retry_limit 50
107
+ # flush_interval 10s
108
+ # <server>
109
+ # host 192.168.0.13
110
+ # </server>
111
+ # </store>
112
+ # <store>
113
+ # @type file
114
+ # path /var/log/fluent/myapp
115
+ # </store>
116
+ #</match>
117
+
118
+ ## match fluent's internal events
119
+ #<match fluent.**>
120
+ # @type null
121
+ #</match>
122
+
123
+ ## match not matched logs and write to file
124
+ #<match **>
125
+ # @type file
126
+ # path /var/log/fluent/else
127
+ # compress gz
128
+ #</match>
129
+
130
+ ## Label: For handling complex event routing
131
+ #<label @STAGING>
132
+ # <match system.**>
133
+ # @type forward
134
+ # @id staging_forward_output
135
+ # <server>
136
+ # host 192.168.0.101
137
+ # </server>
138
+ # </match>
139
+ #</label>
@@ -0,0 +1,51 @@
1
+ require File.expand_path('../lib/fluent/version', __FILE__)
2
+
3
+ Gem::Specification.new do |gem|
4
+ gem.name = "fluentd"
5
+ gem.version = Fluent::VERSION # see lib/fluent/version.rb
6
+
7
+ gem.authors = ["Sadayuki Furuhashi"]
8
+ gem.email = ["frsyuki@gmail.com"]
9
+ gem.description = %q{Fluentd is an open source data collector designed to scale and simplify log management. It can collect, process and ship many kinds of data in near real-time.}
10
+ gem.summary = %q{Fluentd event collector}
11
+ gem.homepage = "http://fluentd.org/"
12
+
13
+ gem.files = `git ls-files`.split($\)
14
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
15
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
16
+ gem.require_paths = ["lib"]
17
+ gem.has_rdoc = false
18
+ gem.license = "Apache-2.0"
19
+
20
+ gem.required_ruby_version = '>= 2.1'
21
+
22
+ gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
23
+ gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
24
+ gem.add_runtime_dependency("cool.io", ["~> 1.4.5"])
25
+ gem.add_runtime_dependency("serverengine", ["~> 2.0"])
26
+ gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
27
+ gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
28
+ gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
29
+ gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
30
+ gem.add_runtime_dependency("strptime", ["~> 0.1.7"])
31
+
32
+ # build gem for a certain platform. see also Rakefile
33
+ fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
34
+ gem.platform = fake_platform unless fake_platform.empty?
35
+ if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
36
+ gem.add_runtime_dependency("win32-service", ["~> 0.8.3"])
37
+ gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
38
+ gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
39
+ gem.add_runtime_dependency("windows-pr", ["~> 1.2.5"])
40
+ end
41
+
42
+ gem.add_development_dependency("rake", ["~> 11.0"])
43
+ gem.add_development_dependency("flexmock", ["~> 2.0"])
44
+ gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
45
+ gem.add_development_dependency("simplecov", ["~> 0.7"])
46
+ gem.add_development_dependency("rr", ["~> 1.0"])
47
+ gem.add_development_dependency("timecop", ["~> 0.3"])
48
+ gem.add_development_dependency("test-unit", ["~> 3.2"])
49
+ gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
50
+ gem.add_development_dependency("oj", ["~> 2.14"])
51
+ end
@@ -0,0 +1,194 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/configurable'
18
+ require 'fluent/plugin'
19
+ require 'fluent/output'
20
+
21
+ module Fluent
22
+ #
23
+ # Agent is a resource unit who manages emittable plugins
24
+ #
25
+ # Next step: `fluentd/root_agent.rb`
26
+ # Next step: `fluentd/label.rb`
27
+ #
28
+ class Agent
29
+ include Configurable
30
+
31
+ def initialize(log:)
32
+ super()
33
+
34
+ @context = nil
35
+ @outputs = []
36
+ @filters = []
37
+
38
+ @lifecycle_control_list = nil
39
+ # lifecycle_control_list is the list of plugins in this agent, and ordered
40
+ # from plugins which DOES emit, then DOESN'T emit
41
+ # (input -> output w/ router -> filter -> output w/o router)
42
+ # for start: use this order DESC
43
+ # (because plugins which appears later in configurations will receive events from plugins which appears ealier)
44
+ # for stop/before_shutdown/shutdown/after_shutdown/close/terminate: use this order ASC
45
+ @lifecycle_cache = nil
46
+
47
+ @log = log
48
+ @event_router = EventRouter.new(NoMatchMatch.new(log), self)
49
+ @error_collector = nil
50
+ end
51
+
52
+ attr_reader :log
53
+ attr_reader :outputs
54
+ attr_reader :filters
55
+ attr_reader :context
56
+ attr_reader :event_router
57
+ attr_reader :error_collector
58
+
59
+ def configure(conf)
60
+ super
61
+
62
+ # initialize <match> and <filter> elements
63
+ conf.elements('filter', 'match').each { |e|
64
+ pattern = e.arg.empty? ? '**' : e.arg
65
+ type = e['@type']
66
+ if e.name == 'filter'
67
+ add_filter(type, pattern, e)
68
+ else
69
+ add_match(type, pattern, e)
70
+ end
71
+ }
72
+ end
73
+
74
+ def lifecycle_control_list
75
+ return @lifecycle_control_list if @lifecycle_control_list
76
+
77
+ lifecycle_control_list = {
78
+ input: [],
79
+ output_with_router: [],
80
+ filter: [],
81
+ output: [],
82
+ }
83
+ if self.respond_to?(:inputs)
84
+ inputs.each do |i|
85
+ lifecycle_control_list[:input] << i
86
+ end
87
+ end
88
+ recursive_output_traverse = ->(o) {
89
+ if o.has_router?
90
+ lifecycle_control_list[:output_with_router] << o
91
+ else
92
+ lifecycle_control_list[:output] << o
93
+ end
94
+
95
+ if o.respond_to?(:outputs)
96
+ o.outputs.each do |store|
97
+ recursive_output_traverse.call(store)
98
+ end
99
+ end
100
+ }
101
+ outputs.each do |o|
102
+ recursive_output_traverse.call(o)
103
+ end
104
+ filters.each do |f|
105
+ lifecycle_control_list[:filter] << f
106
+ end
107
+
108
+ @lifecycle_control_list = lifecycle_control_list
109
+ end
110
+
111
+ def lifecycle(desc: false)
112
+ kind_list = if desc
113
+ [:output, :filter, :output_with_router]
114
+ else
115
+ [:output_with_router, :filter, :output]
116
+ end
117
+ kind_list.each do |kind|
118
+ list = if desc
119
+ lifecycle_control_list[kind].reverse
120
+ else
121
+ lifecycle_control_list[kind]
122
+ end
123
+ display_kind = (kind == :output_with_router ? :output : kind)
124
+ list.each do |instance|
125
+ yield instance, display_kind
126
+ end
127
+ end
128
+ end
129
+
130
+ def add_match(type, pattern, conf)
131
+ log.info "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
132
+
133
+ output = Plugin.new_output(type)
134
+ output.router = @event_router if output.respond_to?(:router=)
135
+ output.configure(conf)
136
+ @outputs << output
137
+ if output.respond_to?(:outputs) && (output.is_a?(Fluent::Plugin::MultiOutput) || output.is_a?(Fluent::MultiOutput))
138
+ @outputs.push(*output.outputs)
139
+ end
140
+ @event_router.add_rule(pattern, output)
141
+
142
+ output
143
+ end
144
+
145
+ def add_filter(type, pattern, conf)
146
+ log.info "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
147
+
148
+ filter = Plugin.new_filter(type)
149
+ filter.router = @event_router
150
+ filter.configure(conf)
151
+ @filters << filter
152
+ @event_router.add_rule(pattern, filter)
153
+
154
+ filter
155
+ end
156
+
157
+ # For handling invalid record
158
+ def emit_error_event(tag, time, record, error)
159
+ end
160
+
161
+ def handle_emits_error(tag, es, error)
162
+ end
163
+
164
+ class NoMatchMatch
165
+ def initialize(log)
166
+ @log = log
167
+ @count = 0
168
+ end
169
+
170
+ def emit_events(tag, es)
171
+ # TODO use time instead of num of records
172
+ c = (@count += 1)
173
+ if c < 512
174
+ if Math.log(c) / Math.log(2) % 1.0 == 0
175
+ @log.warn "no patterns matched", tag: tag
176
+ return
177
+ end
178
+ else
179
+ if c % 512 == 0
180
+ @log.warn "no patterns matched", tag: tag
181
+ return
182
+ end
183
+ end
184
+ @log.on_trace { @log.trace "no patterns matched", tag: tag }
185
+ end
186
+
187
+ def start
188
+ end
189
+
190
+ def shutdown
191
+ end
192
+ end
193
+ end
194
+ end