fluentd 0.14.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,27 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ module Fluent
18
+ DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
19
+ DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
20
+ DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
21
+ IS_WINDOWS = /mswin|mingw/ === RUBY_PLATFORM
22
+ private_constant :IS_WINDOWS
23
+
24
+ def self.windows?
25
+ IS_WINDOWS
26
+ end
27
+ end
@@ -0,0 +1,287 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/msgpack_factory'
18
+
19
+ module Fluent
20
+ class EventStream
21
+ include Enumerable
22
+ include MessagePackFactory::Mixin
23
+
24
+ # dup does deep copy for event stream
25
+ def dup
26
+ raise NotImplementedError, "DO NOT USE THIS CLASS directly."
27
+ end
28
+
29
+ def size
30
+ raise NotImplementedError, "DO NOT USE THIS CLASS directly."
31
+ end
32
+ alias :length :size
33
+
34
+ def empty?
35
+ size == 0
36
+ end
37
+
38
+ # for tests
39
+ def ==(other)
40
+ other.is_a?(EventStream) && self.to_msgpack_stream == other.to_msgpack_stream
41
+ end
42
+
43
+ def repeatable?
44
+ false
45
+ end
46
+
47
+ def slice(index, num)
48
+ raise NotImplementedError, "DO NOT USE THIS CLASS directly."
49
+ end
50
+
51
+ def each(&block)
52
+ raise NotImplementedError, "DO NOT USE THIS CLASS directly."
53
+ end
54
+
55
+ def to_msgpack_stream(time_int: false)
56
+ return to_msgpack_stream_forced_integer if time_int
57
+ out = msgpack_packer
58
+ each {|time,record|
59
+ out.write([time,record])
60
+ }
61
+ out.to_s
62
+ end
63
+
64
+ def to_msgpack_stream_forced_integer
65
+ out = msgpack_packer
66
+ each {|time,record|
67
+ out.write([time.to_i,record])
68
+ }
69
+ out.to_s
70
+ end
71
+ end
72
+
73
+ class OneEventStream < EventStream
74
+ def initialize(time, record)
75
+ @time = time
76
+ @record = record
77
+ end
78
+
79
+ def dup
80
+ OneEventStream.new(@time, @record.dup)
81
+ end
82
+
83
+ def size
84
+ 1
85
+ end
86
+
87
+ def repeatable?
88
+ true
89
+ end
90
+
91
+ def slice(index, num)
92
+ if index > 0 || num == 0
93
+ ArrayEventStream.new([])
94
+ else
95
+ self.dup
96
+ end
97
+ end
98
+
99
+ def each(&block)
100
+ block.call(@time, @record)
101
+ nil
102
+ end
103
+ end
104
+
105
+ # EventStream from entries: Array of [time, record]
106
+ #
107
+ # Use this class for many events data with a tag
108
+ # and its representation is [ [time, record], [time, record], .. ]
109
+ class ArrayEventStream < EventStream
110
+ def initialize(entries)
111
+ @entries = entries
112
+ end
113
+
114
+ def dup
115
+ entries = @entries.map{ |time, record| [time, record.dup] }
116
+ ArrayEventStream.new(entries)
117
+ end
118
+
119
+ def size
120
+ @entries.size
121
+ end
122
+
123
+ def repeatable?
124
+ true
125
+ end
126
+
127
+ def empty?
128
+ @entries.empty?
129
+ end
130
+
131
+ def slice(index, num)
132
+ ArrayEventStream.new(@entries.slice(index, num))
133
+ end
134
+
135
+ def each(&block)
136
+ @entries.each(&block)
137
+ nil
138
+ end
139
+ end
140
+
141
+ # EventStream from entries: numbers of pairs of time and record.
142
+ #
143
+ # This class can handle many events more efficiently than ArrayEventStream
144
+ # because this class generate less objects than ArrayEventStream.
145
+ #
146
+ # Use this class as below, in loop of data-enumeration:
147
+ # 1. initialize blank stream:
148
+ # streams[tag] ||= MultiEventStream.new
149
+ # 2. add events
150
+ # stream[tag].add(time, record)
151
+ class MultiEventStream < EventStream
152
+ def initialize(time_array = [], record_array = [])
153
+ @time_array = time_array
154
+ @record_array = record_array
155
+ end
156
+
157
+ def dup
158
+ MultiEventStream.new(@time_array.dup, @record_array.map(&:dup))
159
+ end
160
+
161
+ def size
162
+ @time_array.size
163
+ end
164
+
165
+ def add(time, record)
166
+ @time_array << time
167
+ @record_array << record
168
+ end
169
+
170
+ def repeatable?
171
+ true
172
+ end
173
+
174
+ def empty?
175
+ @time_array.empty?
176
+ end
177
+
178
+ def slice(index, num)
179
+ MultiEventStream.new(@time_array.slice(index, num), @record_array.slice(index, num))
180
+ end
181
+
182
+ def each(&block)
183
+ time_array = @time_array
184
+ record_array = @record_array
185
+ for i in 0..time_array.length-1
186
+ block.call(time_array[i], record_array[i])
187
+ end
188
+ nil
189
+ end
190
+ end
191
+
192
+ class MessagePackEventStream < EventStream
193
+ # https://github.com/msgpack/msgpack-ruby/issues/119
194
+
195
+ # Keep cached_unpacker argument for existing plugins
196
+ def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
197
+ @data = data
198
+ @size = size
199
+ @unpacked_times = unpacked_times
200
+ @unpacked_records = unpacked_records
201
+ end
202
+
203
+ def empty?
204
+ @data.empty?
205
+ end
206
+
207
+ def dup
208
+ if @unpacked_times
209
+ MessagePackEventStream.new(@data.dup, nil, @size, unpacked_times: @unpacked_times, unpacked_records: @unpacked_records.map(&:dup))
210
+ else
211
+ MessagePackEventStream.new(@data.dup, nil, @size)
212
+ end
213
+ end
214
+
215
+ def size
216
+ # @size is unbelievable always when @size == 0
217
+ # If the number of events is really zero, unpacking events takes very short time.
218
+ ensure_unpacked! if @size == 0
219
+ @size
220
+ end
221
+
222
+ def repeatable?
223
+ true
224
+ end
225
+
226
+ def ensure_unpacked!
227
+ return if @unpacked_times && @unpacked_records
228
+ @unpacked_times = []
229
+ @unpacked_records = []
230
+ msgpack_unpacker.feed_each(@data) do |time, record|
231
+ @unpacked_times << time
232
+ @unpacked_records << record
233
+ end
234
+ # @size should be updated always right after unpack.
235
+ # The real size of unpacked objects are correct, rather than given size.
236
+ @size = @unpacked_times.size
237
+ end
238
+
239
+ # This method returns MultiEventStream, because there are no reason
240
+ # to surve binary serialized by msgpack.
241
+ def slice(index, num)
242
+ ensure_unpacked!
243
+ MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
244
+ end
245
+
246
+ def each(&block)
247
+ if @unpacked_times
248
+ @unpacked_times.each_with_index do |time, i|
249
+ block.call(time, @unpacked_records[i])
250
+ end
251
+ else
252
+ @unpacked_times = []
253
+ @unpacked_records = []
254
+ msgpack_unpacker.feed_each(@data) do |time, record|
255
+ @unpacked_times << time
256
+ @unpacked_records << record
257
+ block.call(time, record)
258
+ end
259
+ @size = @unpacked_times.size
260
+ end
261
+ nil
262
+ end
263
+
264
+ def to_msgpack_stream(time_int: false)
265
+ # time_int is always ignored because @data is always packed binary in this class
266
+ @data
267
+ end
268
+ end
269
+
270
+ module ChunkMessagePackEventStreamer
271
+ include MessagePackFactory::Mixin
272
+ # chunk.extend(ChunkEventStreamer)
273
+ # => chunk.each{|time, record| ... }
274
+ def each(&block)
275
+ open do |io|
276
+ msgpack_unpacker(io).each(&block)
277
+ end
278
+ nil
279
+ end
280
+ alias :msgpack_each :each
281
+
282
+ def to_msgpack_stream(time_int: false)
283
+ # time_int is always ignored because data is already packed and written in chunk
284
+ read
285
+ end
286
+ end
287
+ end
@@ -0,0 +1,259 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/match'
18
+ require 'fluent/event'
19
+ require 'fluent/filter'
20
+
21
+ module Fluent
22
+ #
23
+ # EventRouter is responsible to route events to a collector.
24
+ #
25
+ # It has a list of MatchPattern and Collector pairs:
26
+ #
27
+ # +----------------+ +-----------------+
28
+ # | MatchPattern | | Collector |
29
+ # +----------------+ +-----------------+
30
+ # | access.** ---------> type forward |
31
+ # | logs.** ---------> type copy |
32
+ # | archive.** ---------> type s3 |
33
+ # +----------------+ +-----------------+
34
+ #
35
+ # EventRouter does:
36
+ #
37
+ # 1) receive an event at `#emit` methods
38
+ # 2) match the event's tag with the MatchPatterns
39
+ # 3) forward the event to the corresponding Collector
40
+ #
41
+ # Collector is either of Output, Filter or other EventRouter.
42
+ #
43
+ class EventRouter
44
+ def initialize(default_collector, emit_error_handler)
45
+ @match_rules = []
46
+ @match_cache = MatchCache.new
47
+ @default_collector = default_collector
48
+ @emit_error_handler = emit_error_handler
49
+ end
50
+
51
+ attr_accessor :default_collector
52
+ attr_accessor :emit_error_handler
53
+
54
+ class Rule
55
+ def initialize(pattern, collector)
56
+ patterns = pattern.split(/\s+/).map { |str| MatchPattern.create(str) }
57
+ @pattern = if patterns.length == 1
58
+ patterns[0]
59
+ else
60
+ OrMatchPattern.new(patterns)
61
+ end
62
+ @pattern_str = pattern
63
+ @collector = collector
64
+ end
65
+
66
+ def match?(tag)
67
+ @pattern.match(tag)
68
+ end
69
+
70
+ attr_reader :collector
71
+ attr_reader :pattern_str
72
+ end
73
+
74
+ # called by Agent to add new match pattern and collector
75
+ def add_rule(pattern, collector)
76
+ @match_rules << Rule.new(pattern, collector)
77
+ end
78
+
79
+ def emit(tag, time, record)
80
+ unless record.nil?
81
+ emit_stream(tag, OneEventStream.new(time, record))
82
+ end
83
+ end
84
+
85
+ def emit_array(tag, array)
86
+ emit_stream(tag, ArrayEventStream.new(array))
87
+ end
88
+
89
+ def emit_stream(tag, es)
90
+ match(tag).emit_events(tag, es)
91
+ rescue => e
92
+ @emit_error_handler.handle_emits_error(tag, es, e)
93
+ end
94
+
95
+ def emit_error_event(tag, time, record, error)
96
+ @emit_error_handler.emit_error_event(tag, time, record, error)
97
+ end
98
+
99
+ def match?(tag)
100
+ !!find(tag)
101
+ end
102
+
103
+ def match(tag)
104
+ collector = @match_cache.get(tag) {
105
+ find(tag) || @default_collector
106
+ }
107
+ collector
108
+ end
109
+
110
+ class MatchCache
111
+ MATCH_CACHE_SIZE = 1024
112
+
113
+ def initialize
114
+ super
115
+ @map = {}
116
+ @keys = []
117
+ end
118
+
119
+ def get(key)
120
+ if collector = @map[key]
121
+ return collector
122
+ end
123
+ collector = @map[key] = yield
124
+ if @keys.size >= MATCH_CACHE_SIZE
125
+ # expire the oldest key
126
+ @map.delete @keys.shift
127
+ end
128
+ @keys << key
129
+ collector
130
+ end
131
+ end
132
+
133
+ private
134
+
135
+ class Pipeline
136
+ def initialize
137
+ @filters = []
138
+ @output = nil
139
+ @optimizer = FilterOptimizer.new
140
+ end
141
+
142
+ def add_filter(filter)
143
+ @filters << filter
144
+ @optimizer.filters = @filters
145
+ end
146
+
147
+ def set_output(output)
148
+ @output = output
149
+ end
150
+
151
+ def emit_events(tag, es)
152
+ processed = @optimizer.filter_stream(tag, es)
153
+ @output.emit_events(tag, processed)
154
+ end
155
+
156
+ class FilterOptimizer
157
+ def initialize(filters = [])
158
+ @filters = filters
159
+ @optimizable = nil
160
+ end
161
+
162
+ def filters=(filters)
163
+ @filters = filters
164
+ reset_optimization
165
+ end
166
+
167
+ def filter_stream(tag, es)
168
+ if optimizable?
169
+ optimized_filter_stream(tag, es)
170
+ else
171
+ @filters.reduce(es) { |acc, filter| filter.filter_stream(tag, acc) }
172
+ end
173
+ end
174
+
175
+ private
176
+
177
+ def optimized_filter_stream(tag, es)
178
+ new_es = MultiEventStream.new
179
+ es.each do |time, record|
180
+ filtered_record = record
181
+ filtered_time = time
182
+
183
+ catch :break_loop do
184
+ @filters.each do |filter|
185
+ if filter.has_filter_with_time
186
+ begin
187
+ filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
188
+ throw :break_loop unless filtered_record && filtered_time
189
+ rescue => e
190
+ filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
191
+ end
192
+ else
193
+ begin
194
+ filtered_record = filter.filter(tag, filtered_time, filtered_record)
195
+ throw :break_loop unless filtered_record
196
+ rescue => e
197
+ filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
198
+ end
199
+ end
200
+ end
201
+
202
+ new_es.add(filtered_time, filtered_record)
203
+ end
204
+ end
205
+ new_es
206
+ end
207
+
208
+ def optimizable?
209
+ return @optimizable unless @optimizable.nil?
210
+ @optimizable = if filters_having_filter_stream.empty?
211
+ true
212
+ else
213
+ $log.info "Filtering works with worse performance, because #{filters_having_filter_stream.map(&:class)} uses `#filter_stream` method."
214
+ false
215
+ end
216
+ end
217
+
218
+ def filters_having_filter_stream
219
+ @filters_having_filter_stream ||= @filters.select do |filter|
220
+ filter.class.instance_methods(false).include?(:filter_stream)
221
+ end
222
+ end
223
+
224
+ def reset_optimization
225
+ @optimizable = nil
226
+ @filters_having_filter_stream = nil
227
+ end
228
+ end
229
+ end
230
+
231
+ def find(tag)
232
+ pipeline = nil
233
+ @match_rules.each_with_index { |rule, i|
234
+ if rule.match?(tag)
235
+ if rule.collector.is_a?(Plugin::Filter)
236
+ pipeline ||= Pipeline.new
237
+ pipeline.add_filter(rule.collector)
238
+ else
239
+ if pipeline
240
+ pipeline.set_output(rule.collector)
241
+ else
242
+ # Use Output directly when filter is not matched
243
+ pipeline = rule.collector
244
+ end
245
+ return pipeline
246
+ end
247
+ end
248
+ }
249
+
250
+ if pipeline
251
+ # filter is matched but no match
252
+ pipeline.set_output(@default_collector)
253
+ pipeline
254
+ else
255
+ nil
256
+ end
257
+ end
258
+ end
259
+ end