fluentd 0.14.4-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (328) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +26 -0
  4. data/.travis.yml +45 -0
  5. data/AUTHORS +2 -0
  6. data/CONTRIBUTING.md +35 -0
  7. data/COPYING +14 -0
  8. data/ChangeLog +276 -0
  9. data/Gemfile +9 -0
  10. data/README.md +51 -0
  11. data/Rakefile +53 -0
  12. data/Vagrantfile +17 -0
  13. data/appveyor.yml +41 -0
  14. data/bin/fluent-debug +5 -0
  15. data/example/copy_roundrobin.conf +39 -0
  16. data/example/filter_stdout.conf +22 -0
  17. data/example/in_forward.conf +11 -0
  18. data/example/in_http.conf +14 -0
  19. data/example/in_out_forward.conf +17 -0
  20. data/example/in_syslog.conf +15 -0
  21. data/example/in_tail.conf +14 -0
  22. data/example/in_tcp.conf +13 -0
  23. data/example/in_udp.conf +13 -0
  24. data/example/multi_filters.conf +61 -0
  25. data/example/out_buffered_null.conf +32 -0
  26. data/example/out_copy.conf +20 -0
  27. data/example/out_file.conf +13 -0
  28. data/example/out_forward.conf +35 -0
  29. data/example/out_forward_buf_file.conf +23 -0
  30. data/example/v0_12_filter.conf +78 -0
  31. data/example/v1_literal_example.conf +36 -0
  32. data/fluent.conf +139 -0
  33. data/fluentd.gemspec +51 -0
  34. data/lib/fluent/agent.rb +194 -0
  35. data/lib/fluent/command/bundler_injection.rb +45 -0
  36. data/lib/fluent/command/cat.rb +319 -0
  37. data/lib/fluent/command/debug.rb +102 -0
  38. data/lib/fluent/command/fluentd.rb +273 -0
  39. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  40. data/lib/fluent/compat/exec_util.rb +129 -0
  41. data/lib/fluent/compat/file_util.rb +54 -0
  42. data/lib/fluent/compat/filter.rb +68 -0
  43. data/lib/fluent/compat/formatter.rb +111 -0
  44. data/lib/fluent/compat/formatter_utils.rb +85 -0
  45. data/lib/fluent/compat/handle_tag_and_time_mixin.rb +62 -0
  46. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  47. data/lib/fluent/compat/input.rb +49 -0
  48. data/lib/fluent/compat/output.rb +677 -0
  49. data/lib/fluent/compat/output_chain.rb +60 -0
  50. data/lib/fluent/compat/parser.rb +180 -0
  51. data/lib/fluent/compat/parser_utils.rb +40 -0
  52. data/lib/fluent/compat/propagate_default.rb +62 -0
  53. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  54. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  55. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  56. data/lib/fluent/compat/socket_util.rb +165 -0
  57. data/lib/fluent/compat/string_util.rb +34 -0
  58. data/lib/fluent/compat/structured_format_mixin.rb +26 -0
  59. data/lib/fluent/compat/type_converter.rb +90 -0
  60. data/lib/fluent/config.rb +56 -0
  61. data/lib/fluent/config/basic_parser.rb +123 -0
  62. data/lib/fluent/config/configure_proxy.rb +366 -0
  63. data/lib/fluent/config/dsl.rb +149 -0
  64. data/lib/fluent/config/element.rb +218 -0
  65. data/lib/fluent/config/error.rb +26 -0
  66. data/lib/fluent/config/literal_parser.rb +251 -0
  67. data/lib/fluent/config/parser.rb +107 -0
  68. data/lib/fluent/config/section.rb +212 -0
  69. data/lib/fluent/config/types.rb +136 -0
  70. data/lib/fluent/config/v1_parser.rb +190 -0
  71. data/lib/fluent/configurable.rb +176 -0
  72. data/lib/fluent/daemon.rb +15 -0
  73. data/lib/fluent/engine.rb +220 -0
  74. data/lib/fluent/env.rb +27 -0
  75. data/lib/fluent/event.rb +287 -0
  76. data/lib/fluent/event_router.rb +259 -0
  77. data/lib/fluent/filter.rb +21 -0
  78. data/lib/fluent/formatter.rb +23 -0
  79. data/lib/fluent/input.rb +21 -0
  80. data/lib/fluent/label.rb +38 -0
  81. data/lib/fluent/load.rb +36 -0
  82. data/lib/fluent/log.rb +445 -0
  83. data/lib/fluent/match.rb +141 -0
  84. data/lib/fluent/mixin.rb +31 -0
  85. data/lib/fluent/msgpack_factory.rb +62 -0
  86. data/lib/fluent/output.rb +26 -0
  87. data/lib/fluent/output_chain.rb +23 -0
  88. data/lib/fluent/parser.rb +23 -0
  89. data/lib/fluent/plugin.rb +161 -0
  90. data/lib/fluent/plugin/bare_output.rb +63 -0
  91. data/lib/fluent/plugin/base.rb +130 -0
  92. data/lib/fluent/plugin/buf_file.rb +154 -0
  93. data/lib/fluent/plugin/buf_memory.rb +34 -0
  94. data/lib/fluent/plugin/buffer.rb +603 -0
  95. data/lib/fluent/plugin/buffer/chunk.rb +160 -0
  96. data/lib/fluent/plugin/buffer/file_chunk.rb +323 -0
  97. data/lib/fluent/plugin/buffer/memory_chunk.rb +90 -0
  98. data/lib/fluent/plugin/exec_util.rb +22 -0
  99. data/lib/fluent/plugin/file_util.rb +22 -0
  100. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  101. data/lib/fluent/plugin/filter.rb +93 -0
  102. data/lib/fluent/plugin/filter_grep.rb +75 -0
  103. data/lib/fluent/plugin/filter_record_transformer.rb +342 -0
  104. data/lib/fluent/plugin/filter_stdout.rb +53 -0
  105. data/lib/fluent/plugin/formatter.rb +45 -0
  106. data/lib/fluent/plugin/formatter_csv.rb +47 -0
  107. data/lib/fluent/plugin/formatter_hash.rb +29 -0
  108. data/lib/fluent/plugin/formatter_json.rb +44 -0
  109. data/lib/fluent/plugin/formatter_ltsv.rb +41 -0
  110. data/lib/fluent/plugin/formatter_msgpack.rb +29 -0
  111. data/lib/fluent/plugin/formatter_out_file.rb +78 -0
  112. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  113. data/lib/fluent/plugin/formatter_stdout.rb +74 -0
  114. data/lib/fluent/plugin/in_debug_agent.rb +64 -0
  115. data/lib/fluent/plugin/in_dummy.rb +135 -0
  116. data/lib/fluent/plugin/in_exec.rb +149 -0
  117. data/lib/fluent/plugin/in_forward.rb +366 -0
  118. data/lib/fluent/plugin/in_gc_stat.rb +52 -0
  119. data/lib/fluent/plugin/in_http.rb +422 -0
  120. data/lib/fluent/plugin/in_monitor_agent.rb +401 -0
  121. data/lib/fluent/plugin/in_object_space.rb +90 -0
  122. data/lib/fluent/plugin/in_syslog.rb +204 -0
  123. data/lib/fluent/plugin/in_tail.rb +838 -0
  124. data/lib/fluent/plugin/in_tcp.rb +41 -0
  125. data/lib/fluent/plugin/in_udp.rb +37 -0
  126. data/lib/fluent/plugin/in_unix.rb +201 -0
  127. data/lib/fluent/plugin/input.rb +33 -0
  128. data/lib/fluent/plugin/multi_output.rb +95 -0
  129. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  130. data/lib/fluent/plugin/out_buffered_stdout.rb +70 -0
  131. data/lib/fluent/plugin/out_copy.rb +42 -0
  132. data/lib/fluent/plugin/out_exec.rb +114 -0
  133. data/lib/fluent/plugin/out_exec_filter.rb +393 -0
  134. data/lib/fluent/plugin/out_file.rb +167 -0
  135. data/lib/fluent/plugin/out_forward.rb +646 -0
  136. data/lib/fluent/plugin/out_null.rb +27 -0
  137. data/lib/fluent/plugin/out_relabel.rb +28 -0
  138. data/lib/fluent/plugin/out_roundrobin.rb +80 -0
  139. data/lib/fluent/plugin/out_stdout.rb +48 -0
  140. data/lib/fluent/plugin/out_stream.rb +130 -0
  141. data/lib/fluent/plugin/output.rb +1020 -0
  142. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  143. data/lib/fluent/plugin/parser.rb +175 -0
  144. data/lib/fluent/plugin/parser_apache.rb +28 -0
  145. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  146. data/lib/fluent/plugin/parser_apache_error.rb +26 -0
  147. data/lib/fluent/plugin/parser_csv.rb +33 -0
  148. data/lib/fluent/plugin/parser_json.rb +79 -0
  149. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  150. data/lib/fluent/plugin/parser_multiline.rb +104 -0
  151. data/lib/fluent/plugin/parser_nginx.rb +28 -0
  152. data/lib/fluent/plugin/parser_none.rb +36 -0
  153. data/lib/fluent/plugin/parser_regexp.rb +73 -0
  154. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  155. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  156. data/lib/fluent/plugin/socket_util.rb +22 -0
  157. data/lib/fluent/plugin/storage.rb +84 -0
  158. data/lib/fluent/plugin/storage_local.rb +132 -0
  159. data/lib/fluent/plugin/string_util.rb +22 -0
  160. data/lib/fluent/plugin_helper.rb +42 -0
  161. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  162. data/lib/fluent/plugin_helper/compat_parameters.rb +224 -0
  163. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  164. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  165. data/lib/fluent/plugin_helper/formatter.rb +149 -0
  166. data/lib/fluent/plugin_helper/inject.rb +125 -0
  167. data/lib/fluent/plugin_helper/parser.rb +147 -0
  168. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  169. data/lib/fluent/plugin_helper/storage.rb +331 -0
  170. data/lib/fluent/plugin_helper/thread.rb +147 -0
  171. data/lib/fluent/plugin_helper/timer.rb +90 -0
  172. data/lib/fluent/plugin_id.rb +63 -0
  173. data/lib/fluent/process.rb +504 -0
  174. data/lib/fluent/registry.rb +99 -0
  175. data/lib/fluent/root_agent.rb +314 -0
  176. data/lib/fluent/rpc.rb +94 -0
  177. data/lib/fluent/supervisor.rb +680 -0
  178. data/lib/fluent/system_config.rb +122 -0
  179. data/lib/fluent/test.rb +56 -0
  180. data/lib/fluent/test/base.rb +85 -0
  181. data/lib/fluent/test/driver/base.rb +179 -0
  182. data/lib/fluent/test/driver/base_owned.rb +70 -0
  183. data/lib/fluent/test/driver/base_owner.rb +125 -0
  184. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  185. data/lib/fluent/test/driver/filter.rb +57 -0
  186. data/lib/fluent/test/driver/formatter.rb +30 -0
  187. data/lib/fluent/test/driver/input.rb +31 -0
  188. data/lib/fluent/test/driver/multi_output.rb +52 -0
  189. data/lib/fluent/test/driver/output.rb +76 -0
  190. data/lib/fluent/test/driver/parser.rb +30 -0
  191. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  192. data/lib/fluent/test/filter_test.rb +77 -0
  193. data/lib/fluent/test/formatter_test.rb +65 -0
  194. data/lib/fluent/test/helpers.rb +79 -0
  195. data/lib/fluent/test/input_test.rb +172 -0
  196. data/lib/fluent/test/log.rb +73 -0
  197. data/lib/fluent/test/output_test.rb +156 -0
  198. data/lib/fluent/test/parser_test.rb +70 -0
  199. data/lib/fluent/time.rb +175 -0
  200. data/lib/fluent/timezone.rb +133 -0
  201. data/lib/fluent/unique_id.rb +39 -0
  202. data/lib/fluent/version.rb +21 -0
  203. data/lib/fluent/winsvc.rb +71 -0
  204. data/test/compat/test_calls_super.rb +166 -0
  205. data/test/compat/test_parser.rb +82 -0
  206. data/test/config/assertions.rb +42 -0
  207. data/test/config/test_config_parser.rb +507 -0
  208. data/test/config/test_configurable.rb +1194 -0
  209. data/test/config/test_configure_proxy.rb +386 -0
  210. data/test/config/test_dsl.rb +415 -0
  211. data/test/config/test_element.rb +403 -0
  212. data/test/config/test_literal_parser.rb +297 -0
  213. data/test/config/test_section.rb +184 -0
  214. data/test/config/test_system_config.rb +120 -0
  215. data/test/config/test_types.rb +171 -0
  216. data/test/helper.rb +119 -0
  217. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  218. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  219. data/test/plugin/data/2010/01/20100102.log +0 -0
  220. data/test/plugin/data/log/bar +0 -0
  221. data/test/plugin/data/log/foo/bar.log +0 -0
  222. data/test/plugin/data/log/test.log +0 -0
  223. data/test/plugin/test_bare_output.rb +118 -0
  224. data/test/plugin/test_base.rb +75 -0
  225. data/test/plugin/test_buf_file.rb +571 -0
  226. data/test/plugin/test_buf_memory.rb +42 -0
  227. data/test/plugin/test_buffer.rb +1200 -0
  228. data/test/plugin/test_buffer_chunk.rb +168 -0
  229. data/test/plugin/test_buffer_file_chunk.rb +771 -0
  230. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  231. data/test/plugin/test_file_util.rb +96 -0
  232. data/test/plugin/test_filter.rb +353 -0
  233. data/test/plugin/test_filter_grep.rb +119 -0
  234. data/test/plugin/test_filter_record_transformer.rb +600 -0
  235. data/test/plugin/test_filter_stdout.rb +211 -0
  236. data/test/plugin/test_formatter_csv.rb +94 -0
  237. data/test/plugin/test_formatter_json.rb +30 -0
  238. data/test/plugin/test_formatter_ltsv.rb +52 -0
  239. data/test/plugin/test_formatter_msgpack.rb +28 -0
  240. data/test/plugin/test_formatter_out_file.rb +95 -0
  241. data/test/plugin/test_formatter_single_value.rb +38 -0
  242. data/test/plugin/test_in_debug_agent.rb +28 -0
  243. data/test/plugin/test_in_dummy.rb +188 -0
  244. data/test/plugin/test_in_exec.rb +133 -0
  245. data/test/plugin/test_in_forward.rb +635 -0
  246. data/test/plugin/test_in_gc_stat.rb +39 -0
  247. data/test/plugin/test_in_http.rb +442 -0
  248. data/test/plugin/test_in_monitor_agent.rb +329 -0
  249. data/test/plugin/test_in_object_space.rb +64 -0
  250. data/test/plugin/test_in_syslog.rb +205 -0
  251. data/test/plugin/test_in_tail.rb +1001 -0
  252. data/test/plugin/test_in_tcp.rb +102 -0
  253. data/test/plugin/test_in_udp.rb +121 -0
  254. data/test/plugin/test_in_unix.rb +126 -0
  255. data/test/plugin/test_input.rb +122 -0
  256. data/test/plugin/test_multi_output.rb +180 -0
  257. data/test/plugin/test_out_buffered_null.rb +79 -0
  258. data/test/plugin/test_out_buffered_stdout.rb +122 -0
  259. data/test/plugin/test_out_copy.rb +160 -0
  260. data/test/plugin/test_out_exec.rb +155 -0
  261. data/test/plugin/test_out_exec_filter.rb +262 -0
  262. data/test/plugin/test_out_file.rb +383 -0
  263. data/test/plugin/test_out_forward.rb +590 -0
  264. data/test/plugin/test_out_null.rb +29 -0
  265. data/test/plugin/test_out_relabel.rb +28 -0
  266. data/test/plugin/test_out_roundrobin.rb +146 -0
  267. data/test/plugin/test_out_stdout.rb +92 -0
  268. data/test/plugin/test_out_stream.rb +93 -0
  269. data/test/plugin/test_output.rb +568 -0
  270. data/test/plugin/test_output_as_buffered.rb +1604 -0
  271. data/test/plugin/test_output_as_buffered_overflow.rb +250 -0
  272. data/test/plugin/test_output_as_buffered_retries.rb +839 -0
  273. data/test/plugin/test_output_as_buffered_secondary.rb +817 -0
  274. data/test/plugin/test_output_as_standard.rb +374 -0
  275. data/test/plugin/test_owned_by.rb +35 -0
  276. data/test/plugin/test_parser_apache.rb +42 -0
  277. data/test/plugin/test_parser_apache2.rb +38 -0
  278. data/test/plugin/test_parser_apache_error.rb +45 -0
  279. data/test/plugin/test_parser_base.rb +32 -0
  280. data/test/plugin/test_parser_csv.rb +104 -0
  281. data/test/plugin/test_parser_json.rb +107 -0
  282. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  283. data/test/plugin/test_parser_multiline.rb +100 -0
  284. data/test/plugin/test_parser_nginx.rb +48 -0
  285. data/test/plugin/test_parser_none.rb +53 -0
  286. data/test/plugin/test_parser_regexp.rb +277 -0
  287. data/test/plugin/test_parser_syslog.rb +66 -0
  288. data/test/plugin/test_parser_time.rb +46 -0
  289. data/test/plugin/test_parser_tsv.rb +121 -0
  290. data/test/plugin/test_storage.rb +167 -0
  291. data/test/plugin/test_storage_local.rb +8 -0
  292. data/test/plugin/test_string_util.rb +26 -0
  293. data/test/plugin_helper/test_child_process.rb +608 -0
  294. data/test/plugin_helper/test_compat_parameters.rb +242 -0
  295. data/test/plugin_helper/test_event_emitter.rb +51 -0
  296. data/test/plugin_helper/test_event_loop.rb +52 -0
  297. data/test/plugin_helper/test_formatter.rb +252 -0
  298. data/test/plugin_helper/test_inject.rb +487 -0
  299. data/test/plugin_helper/test_parser.rb +263 -0
  300. data/test/plugin_helper/test_retry_state.rb +399 -0
  301. data/test/plugin_helper/test_storage.rb +521 -0
  302. data/test/plugin_helper/test_thread.rb +164 -0
  303. data/test/plugin_helper/test_timer.rb +131 -0
  304. data/test/scripts/exec_script.rb +32 -0
  305. data/test/scripts/fluent/plugin/formatter_known.rb +8 -0
  306. data/test/scripts/fluent/plugin/out_test.rb +81 -0
  307. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  308. data/test/scripts/fluent/plugin/parser_known.rb +4 -0
  309. data/test/test_config.rb +179 -0
  310. data/test/test_configdsl.rb +148 -0
  311. data/test/test_event.rb +329 -0
  312. data/test/test_event_router.rb +331 -0
  313. data/test/test_event_time.rb +184 -0
  314. data/test/test_filter.rb +121 -0
  315. data/test/test_formatter.rb +319 -0
  316. data/test/test_input.rb +31 -0
  317. data/test/test_log.rb +572 -0
  318. data/test/test_match.rb +137 -0
  319. data/test/test_mixin.rb +351 -0
  320. data/test/test_output.rb +214 -0
  321. data/test/test_plugin_classes.rb +136 -0
  322. data/test/test_plugin_helper.rb +81 -0
  323. data/test/test_process.rb +48 -0
  324. data/test/test_root_agent.rb +278 -0
  325. data/test/test_supervisor.rb +339 -0
  326. data/test/test_time_formatter.rb +186 -0
  327. data/test/test_unique_id.rb +47 -0
  328. metadata +823 -0
@@ -0,0 +1,100 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/parser'
3
+ require 'fluent/plugin/parser'
4
+
5
+ class MultilineParserTest < ::Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ def create_parser(conf)
11
+ parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::MultilineParser).configure(conf)
12
+ parser
13
+ end
14
+
15
+ def test_configure_with_invalid_params
16
+ [{'format100' => '/(?<msg>.*)/'}, {'format1' => '/(?<msg>.*)/', 'format3' => '/(?<msg>.*)/'}, 'format1' => '/(?<msg>.*)'].each { |config|
17
+ assert_raise(Fluent::ConfigError) {
18
+ create_parser(config)
19
+ }
20
+ }
21
+ end
22
+
23
+ def test_parse
24
+ parser = create_parser('format1' => '/^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/')
25
+ parser.instance.parse(<<EOS.chomp) { |time, record|
26
+ 2013-3-03 14:27:33 [main] ERROR Main - Exception
27
+ javax.management.RuntimeErrorException: null
28
+ \tat Main.main(Main.java:16) ~[bin/:na]
29
+ EOS
30
+
31
+ assert_equal(event_time('2013-3-03 14:27:33').to_i, time)
32
+ assert_equal({
33
+ "thread" => "main",
34
+ "level" => "ERROR",
35
+ "message" => " Main - Exception\njavax.management.RuntimeErrorException: null\n\tat Main.main(Main.java:16) ~[bin/:na]"
36
+ }, record)
37
+ }
38
+ end
39
+
40
+ def test_parse_with_firstline
41
+ parser = create_parser('format_firstline' => '/----/', 'format1' => '/time=(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}).*message=(?<message>.*)/')
42
+ parser.instance.parse(<<EOS.chomp) { |time, record|
43
+ ----
44
+ time=2013-3-03 14:27:33
45
+ message=test1
46
+ EOS
47
+
48
+ assert(parser.instance.firstline?('----'))
49
+ assert_equal(event_time('2013-3-03 14:27:33').to_i, time)
50
+ assert_equal({"message" => "test1"}, record)
51
+ }
52
+ end
53
+
54
+ def test_parse_with_multiple_formats
55
+ parser = create_parser('format_firstline' => '/^Started/',
56
+ 'format1' => '/Started (?<method>[^ ]+) "(?<path>[^"]+)" for (?<host>[^ ]+) at (?<time>[^ ]+ [^ ]+ [^ ]+)\n/',
57
+ 'format2' => '/Processing by (?<controller>[^\u0023]+)\u0023(?<controller_method>[^ ]+) as (?<format>[^ ]+?)\n/',
58
+ 'format3' => '/( Parameters: (?<parameters>[^ ]+)\n)?/',
59
+ 'format4' => '/ Rendered (?<template>[^ ]+) within (?<layout>.+) \([\d\.]+ms\)\n/',
60
+ 'format5' => '/Completed (?<code>[^ ]+) [^ ]+ in (?<runtime>[\d\.]+)ms \(Views: (?<view_runtime>[\d\.]+)ms \| ActiveRecord: (?<ar_runtime>[\d\.]+)ms\)/'
61
+ )
62
+ parser.instance.parse(<<EOS.chomp) { |time, record|
63
+ Started GET "/users/123/" for 127.0.0.1 at 2013-06-14 12:00:11 +0900
64
+ Processing by UsersController#show as HTML
65
+ Parameters: {"user_id"=>"123"}
66
+ Rendered users/show.html.erb within layouts/application (0.3ms)
67
+ Completed 200 OK in 4ms (Views: 3.2ms | ActiveRecord: 0.0ms)
68
+ EOS
69
+
70
+ assert(parser.instance.firstline?('Started GET "/users/123/" for 127.0.0.1...'))
71
+ assert_equal(event_time('2013-06-14 12:00:11 +0900').to_i, time)
72
+ assert_equal({
73
+ "method" => "GET",
74
+ "path" => "/users/123/",
75
+ "host" => "127.0.0.1",
76
+ "controller" => "UsersController",
77
+ "controller_method" => "show",
78
+ "format" => "HTML",
79
+ "parameters" => "{\"user_id\"=>\"123\"}",
80
+ "template" => "users/show.html.erb",
81
+ "layout" => "layouts/application",
82
+ "code" => "200",
83
+ "runtime" => "4",
84
+ "view_runtime" => "3.2",
85
+ "ar_runtime" => "0.0"
86
+ }, record)
87
+ }
88
+ end
89
+
90
+ def test_parse_with_keep_time_key
91
+ parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::MultilineParser).configure(
92
+ 'format1' => '/^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2})/',
93
+ 'keep_time_key' => 'true'
94
+ )
95
+ text = '2013-3-03 14:27:33'
96
+ parser.instance.parse(text) { |time, record|
97
+ assert_equal text, record['time']
98
+ }
99
+ end
100
+ end
@@ -0,0 +1,48 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/parser'
3
+ require 'fluent/plugin/parser_nginx'
4
+
5
+ class NginxParserTest < ::Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ @expected = {
9
+ 'remote' => '127.0.0.1',
10
+ 'host' => '192.168.0.1',
11
+ 'user' => '-',
12
+ 'method' => 'GET',
13
+ 'path' => '/',
14
+ 'code' => '200',
15
+ 'size' => '777',
16
+ 'referer' => '-',
17
+ 'agent' => 'Opera/12.0'
18
+ }
19
+ end
20
+
21
+ def create_driver
22
+ Fluent::Test::Driver::Parser.new(Fluent::Plugin::NginxParser.new).configure({})
23
+ end
24
+
25
+ def test_parse
26
+ d = create_driver
27
+ d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
28
+ assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
29
+ assert_equal(@expected, record)
30
+ }
31
+ end
32
+
33
+ def test_parse_with_empty_included_path
34
+ d = create_driver
35
+ d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
36
+ assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
37
+ assert_equal(@expected.merge('path' => '/a[ ]b'), record)
38
+ }
39
+ end
40
+
41
+ def test_parse_without_http_version
42
+ d = create_driver
43
+ d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
44
+ assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
45
+ assert_equal(@expected, record)
46
+ }
47
+ end
48
+ end
@@ -0,0 +1,53 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/parser'
3
+ require 'fluent/plugin/parser'
4
+
5
+ class NoneParserTest < ::Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ def test_config_params
11
+ parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::NoneParser)
12
+ parser.configure({})
13
+ assert_equal "message", parser.instance.message_key
14
+
15
+ parser.configure('message_key' => 'foobar')
16
+ assert_equal "foobar", parser.instance.message_key
17
+ end
18
+
19
+ def test_parse
20
+ parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('none'))
21
+ parser.configure({})
22
+ parser.instance.parse('log message!') { |time, record|
23
+ assert_equal({'message' => 'log message!'}, record)
24
+ }
25
+ end
26
+
27
+ def test_parse_with_message_key
28
+ parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::NoneParser)
29
+ parser.configure('message_key' => 'foobar')
30
+ parser.instance.parse('log message!') { |time, record|
31
+ assert_equal({'foobar' => 'log message!'}, record)
32
+ }
33
+ end
34
+
35
+ def test_parse_without_default_time
36
+ time_at_start = Time.now.to_i
37
+
38
+ parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('none'))
39
+ parser.configure({})
40
+ parser.instance.parse('log message!') { |time, record|
41
+ assert time && time >= time_at_start, "parser puts current time without time input"
42
+ assert_equal({'message' => 'log message!'}, record)
43
+ }
44
+
45
+ parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('none'))
46
+ parser.instance.estimate_current_event = false
47
+ parser.configure({})
48
+ parser.instance.parse('log message!') { |time, record|
49
+ assert_equal({'message' => 'log message!'}, record)
50
+ assert_nil time, "parser returns nil w/o time if configured so"
51
+ }
52
+ end
53
+ end
@@ -0,0 +1,277 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/parser'
3
+ require 'fluent/plugin/parser'
4
+
5
+ class RegexpParserTest < ::Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ def internal_test_case(parser)
11
+ text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] [14/Feb/2013:12:00:00 +0900] "true /,/user HTTP/1.1" 200 777'
12
+ parser.parse(text) { |time, record|
13
+ assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
14
+ assert_equal({
15
+ 'user' => '-',
16
+ 'flag' => true,
17
+ 'code' => 200.0,
18
+ 'size' => 777,
19
+ 'date' => event_time('14/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'),
20
+ 'host' => '192.168.0.1',
21
+ 'path' => ['/', '/user']
22
+ }, record)
23
+ }
24
+ end
25
+
26
+ sub_test_case "Fluent::Compat::TextParser::RegexpParser" do
27
+ def create_driver(regexp, conf = {}, initialize_conf: false)
28
+ if initialize_conf
29
+ Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
30
+ else
31
+ Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
32
+ end
33
+ end
34
+
35
+ def test_parse_with_typed
36
+ # Use Regexp.new instead of // literal to avoid different parser behaviour in 1.9 and 2.0
37
+ regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
38
+ conf = {
39
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
40
+ 'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
41
+ }
42
+ d = create_driver(regexp, conf, initialize_conf: true)
43
+ internal_test_case(d.instance)
44
+ end
45
+
46
+ def test_parse_with_configure
47
+ # Specify conf by configure method instaed of intializer
48
+ regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
49
+ conf = {
50
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
51
+ 'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
52
+ }
53
+ d = create_driver(regexp, conf)
54
+ internal_test_case(d.instance)
55
+ assert_equal(regexp, d.instance.patterns['format'])
56
+ assert_equal("%d/%b/%Y:%H:%M:%S %z", d.instance.patterns['time_format'])
57
+ end
58
+
59
+ def test_parse_with_typed_and_name_separator
60
+ regexp = Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!)
61
+ conf = {
62
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
63
+ 'types' => 'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer',
64
+ 'types_label_delimiter' => '|'
65
+ }
66
+ d = create_driver(regexp, conf)
67
+ internal_test_case(d.instance)
68
+ end
69
+
70
+ def test_parse_with_time_key
71
+ conf = {
72
+ 'time_format' => "%Y-%m-%d %H:%M:%S %z",
73
+ 'time_key' => 'logtime'
74
+ }
75
+ d = create_driver(/(?<logtime>[^\]]*)/, conf)
76
+ text = '2013-02-28 12:00:00 +0900'
77
+ d.instance.parse(text) do |time, _record|
78
+ assert_equal Fluent::EventTime.parse(text), time
79
+ end
80
+ end
81
+
82
+ def test_parse_without_time
83
+ time_at_start = Time.now.to_i
84
+ text = "tagomori_satoshi tagomoris 34\n"
85
+
86
+ regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)
87
+ conf = {
88
+ 'types' => 'name:string,user:string,age:integer'
89
+ }
90
+ d = create_driver(regexp, conf)
91
+
92
+ d.instance.parse(text) { |time, record|
93
+ assert time && time >= time_at_start, "parser puts current time without time input"
94
+ assert_equal "tagomori_satoshi", record["name"]
95
+ assert_equal "tagomoris", record["user"]
96
+ assert_equal 34, record["age"]
97
+ }
98
+ end
99
+
100
+ def test_parse_without_time_estimate_curent_event_false
101
+ text = "tagomori_satoshi tagomoris 34\n"
102
+ regexp = Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!)
103
+ conf = {
104
+ 'types' => 'name:string,user:string,age:integer'
105
+ }
106
+ d = create_driver(regexp, conf)
107
+ d.instance.estimate_current_event = false
108
+ d.instance.parse(text) { |time, record|
109
+ assert_equal "tagomori_satoshi", record["name"]
110
+ assert_equal "tagomoris", record["user"]
111
+ assert_equal 34, record["age"]
112
+
113
+ assert_nil time, "parser returns nil if configured so"
114
+ }
115
+ end
116
+
117
+ def test_parse_with_keep_time_key
118
+ regexp = Regexp.new(%q!(?<time>.*)!)
119
+ conf = {
120
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
121
+ 'keep_time_key' => 'true',
122
+ }
123
+ d = create_driver(regexp, conf)
124
+ text = '28/Feb/2013:12:00:00 +0900'
125
+ d.instance.parse(text) do |_time, record|
126
+ assert_equal text, record['time']
127
+ end
128
+ end
129
+
130
+ def test_parse_with_keep_time_key_with_typecast
131
+ regexp = Regexp.new(%q!(?<time>.*)!)
132
+ conf = {
133
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
134
+ 'keep_time_key' => 'true',
135
+ 'types' => 'time:time:%d/%b/%Y:%H:%M:%S %z',
136
+ }
137
+ d = create_driver(regexp, conf)
138
+ text = '28/Feb/2013:12:00:00 +0900'
139
+ d.instance.parse(text) do |_time, record|
140
+ assert_equal 1362020400, record['time']
141
+ end
142
+ end
143
+ end
144
+
145
+ sub_test_case "Fluent::Plugin::RegexpParser" do
146
+ def create_driver(conf)
147
+ Fluent::Test::Driver::Parser.new(Fluent::Plugin::RegexpParser.new).configure(conf)
148
+ end
149
+
150
+ sub_test_case "configure" do
151
+ def test_default_options
152
+ conf = {
153
+ 'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
154
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
155
+ 'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
156
+ }
157
+ d = create_driver(conf)
158
+ regexp = d.instance.instance_variable_get(:@regexp)
159
+ assert_equal(0, regexp.options)
160
+ end
161
+
162
+ data(
163
+ ignorecase: [{ "ignorecase" => true }, Regexp::IGNORECASE],
164
+ multiline: [{ "multiline" => true }, Regexp::MULTILINE],
165
+ ignorecase_multiline: [{ "ignorecase" => true, "multiline" => true }, Regexp::IGNORECASE | Regexp::MULTILINE],
166
+ )
167
+ def test_options(data)
168
+ regexp_option, expected = data
169
+ conf = {
170
+ 'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
171
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
172
+ 'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
173
+ }
174
+ conf = conf.merge(regexp_option)
175
+ d = create_driver(conf)
176
+ regexp = d.instance.instance_variable_get(:@regexp)
177
+ assert_equal(expected, regexp.options)
178
+ end
179
+ end
180
+
181
+ def test_parse_with_typed
182
+ conf = {
183
+ 'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
184
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
185
+ 'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
186
+ }
187
+ d = create_driver(conf)
188
+ internal_test_case(d.instance)
189
+ end
190
+
191
+ def test_parse_with_typed_and_name_separator
192
+ conf = {
193
+ 'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
194
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
195
+ 'types' => 'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer',
196
+ 'types_label_delimiter' => '|'
197
+ }
198
+ d = create_driver(conf)
199
+ internal_test_case(d.instance)
200
+ end
201
+
202
+ def test_parse_with_time_key
203
+ conf = {
204
+ 'expression' => %q!/(?<logtime>[^\]]*)/!,
205
+ 'time_format' => "%Y-%m-%d %H:%M:%S %z",
206
+ 'time_key' => 'logtime'
207
+ }
208
+ d = create_driver(conf)
209
+ text = '2013-02-28 12:00:00 +0900'
210
+ d.instance.parse(text) do |time, _record|
211
+ assert_equal Fluent::EventTime.parse(text), time
212
+ end
213
+ end
214
+
215
+ def test_parse_without_time
216
+ time_at_start = Time.now.to_i
217
+ text = "tagomori_satoshi tagomoris 34\n"
218
+
219
+ conf = {
220
+ 'expression' => %q!/^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$/!,
221
+ 'types' => 'name:string,user:string,age:integer'
222
+ }
223
+ d = create_driver(conf)
224
+
225
+ d.instance.parse(text) { |time, record|
226
+ assert time && time >= time_at_start, "parser puts current time without time input"
227
+ assert_equal "tagomori_satoshi", record["name"]
228
+ assert_equal "tagomoris", record["user"]
229
+ assert_equal 34, record["age"]
230
+ }
231
+ end
232
+
233
+ def test_parse_without_time_estimate_curent_event_false
234
+ text = "tagomori_satoshi tagomoris 34\n"
235
+ conf = {
236
+ 'expression' => %q!/^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$/!,
237
+ 'types' => 'name:string,user:string,age:integer'
238
+ }
239
+ d = create_driver(conf)
240
+ d.instance.estimate_current_event = false
241
+ d.instance.parse(text) { |time, record|
242
+ assert_equal "tagomori_satoshi", record["name"]
243
+ assert_equal "tagomoris", record["user"]
244
+ assert_equal 34, record["age"]
245
+
246
+ assert_nil time, "parser returns nil if configured so"
247
+ }
248
+ end
249
+
250
+ def test_parse_with_keep_time_key
251
+ conf = {
252
+ 'expression' => %q!/(?<time>.*)/!,
253
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
254
+ 'keep_time_key' => 'true',
255
+ }
256
+ d = create_driver(conf)
257
+ text = '28/Feb/2013:12:00:00 +0900'
258
+ d.instance.parse(text) do |_time, record|
259
+ assert_equal text, record['time']
260
+ end
261
+ end
262
+
263
+ def test_parse_with_keep_time_key_with_typecast
264
+ conf = {
265
+ 'expression' => %q!/(?<time>.*)/!,
266
+ 'time_format' => "%d/%b/%Y:%H:%M:%S %z",
267
+ 'keep_time_key' => 'true',
268
+ 'types' => 'time:time:%d/%b/%Y:%H:%M:%S %z',
269
+ }
270
+ d = create_driver(conf)
271
+ text = '28/Feb/2013:12:00:00 +0900'
272
+ d.instance.parse(text) do |_time, record|
273
+ assert_equal 1362020400, record['time']
274
+ end
275
+ end
276
+ end
277
+ end