fluentd 0.12.43 → 0.14.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (253) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE.md +6 -0
  3. data/.gitignore +2 -0
  4. data/.travis.yml +33 -21
  5. data/CONTRIBUTING.md +1 -0
  6. data/ChangeLog +1239 -0
  7. data/README.md +0 -25
  8. data/Rakefile +2 -1
  9. data/Vagrantfile +17 -0
  10. data/appveyor.yml +35 -0
  11. data/example/filter_stdout.conf +5 -5
  12. data/example/in_forward.conf +2 -2
  13. data/example/in_http.conf +2 -2
  14. data/example/in_out_forward.conf +17 -0
  15. data/example/in_syslog.conf +2 -2
  16. data/example/in_tail.conf +2 -2
  17. data/example/in_tcp.conf +2 -2
  18. data/example/in_udp.conf +2 -2
  19. data/example/out_copy.conf +4 -4
  20. data/example/out_file.conf +2 -2
  21. data/example/out_forward.conf +2 -2
  22. data/example/out_forward_buf_file.conf +23 -0
  23. data/example/v0_12_filter.conf +8 -8
  24. data/fluent.conf +29 -0
  25. data/fluentd.gemspec +18 -11
  26. data/lib/fluent/agent.rb +60 -58
  27. data/lib/fluent/command/cat.rb +1 -1
  28. data/lib/fluent/command/debug.rb +7 -5
  29. data/lib/fluent/command/fluentd.rb +97 -2
  30. data/lib/fluent/compat/call_super_mixin.rb +67 -0
  31. data/lib/fluent/compat/filter.rb +50 -0
  32. data/lib/fluent/compat/formatter.rb +109 -0
  33. data/lib/fluent/compat/input.rb +50 -0
  34. data/lib/fluent/compat/output.rb +617 -0
  35. data/lib/fluent/compat/output_chain.rb +60 -0
  36. data/lib/fluent/compat/parser.rb +163 -0
  37. data/lib/fluent/compat/propagate_default.rb +62 -0
  38. data/lib/fluent/config.rb +23 -20
  39. data/lib/fluent/config/configure_proxy.rb +119 -70
  40. data/lib/fluent/config/dsl.rb +5 -18
  41. data/lib/fluent/config/element.rb +72 -8
  42. data/lib/fluent/config/error.rb +0 -3
  43. data/lib/fluent/config/literal_parser.rb +0 -2
  44. data/lib/fluent/config/parser.rb +4 -4
  45. data/lib/fluent/config/section.rb +39 -28
  46. data/lib/fluent/config/types.rb +2 -13
  47. data/lib/fluent/config/v1_parser.rb +1 -3
  48. data/lib/fluent/configurable.rb +48 -16
  49. data/lib/fluent/daemon.rb +15 -0
  50. data/lib/fluent/engine.rb +26 -52
  51. data/lib/fluent/env.rb +6 -4
  52. data/lib/fluent/event.rb +58 -11
  53. data/lib/fluent/event_router.rb +5 -5
  54. data/lib/fluent/filter.rb +2 -50
  55. data/lib/fluent/formatter.rb +4 -293
  56. data/lib/fluent/input.rb +2 -32
  57. data/lib/fluent/label.rb +2 -2
  58. data/lib/fluent/load.rb +3 -2
  59. data/lib/fluent/log.rb +107 -38
  60. data/lib/fluent/match.rb +0 -36
  61. data/lib/fluent/mixin.rb +117 -7
  62. data/lib/fluent/msgpack_factory.rb +62 -0
  63. data/lib/fluent/output.rb +7 -612
  64. data/lib/fluent/output_chain.rb +23 -0
  65. data/lib/fluent/parser.rb +4 -800
  66. data/lib/fluent/plugin.rb +100 -121
  67. data/lib/fluent/plugin/bare_output.rb +63 -0
  68. data/lib/fluent/plugin/base.rb +121 -0
  69. data/lib/fluent/plugin/buf_file.rb +101 -182
  70. data/lib/fluent/plugin/buf_memory.rb +9 -92
  71. data/lib/fluent/plugin/buffer.rb +473 -0
  72. data/lib/fluent/plugin/buffer/chunk.rb +135 -0
  73. data/lib/fluent/plugin/buffer/file_chunk.rb +339 -0
  74. data/lib/fluent/plugin/buffer/memory_chunk.rb +100 -0
  75. data/lib/fluent/plugin/exec_util.rb +80 -75
  76. data/lib/fluent/plugin/file_util.rb +33 -28
  77. data/lib/fluent/plugin/file_wrapper.rb +120 -0
  78. data/lib/fluent/plugin/filter.rb +51 -0
  79. data/lib/fluent/plugin/filter_grep.rb +13 -40
  80. data/lib/fluent/plugin/filter_record_transformer.rb +22 -18
  81. data/lib/fluent/plugin/formatter.rb +93 -0
  82. data/lib/fluent/plugin/formatter_csv.rb +48 -0
  83. data/lib/fluent/plugin/formatter_hash.rb +32 -0
  84. data/lib/fluent/plugin/formatter_json.rb +47 -0
  85. data/lib/fluent/plugin/formatter_ltsv.rb +42 -0
  86. data/lib/fluent/plugin/formatter_msgpack.rb +32 -0
  87. data/lib/fluent/plugin/formatter_out_file.rb +45 -0
  88. data/lib/fluent/plugin/formatter_single_value.rb +34 -0
  89. data/lib/fluent/plugin/formatter_stdout.rb +39 -0
  90. data/lib/fluent/plugin/in_debug_agent.rb +4 -0
  91. data/lib/fluent/plugin/in_dummy.rb +22 -18
  92. data/lib/fluent/plugin/in_exec.rb +18 -8
  93. data/lib/fluent/plugin/in_forward.rb +36 -79
  94. data/lib/fluent/plugin/in_gc_stat.rb +4 -0
  95. data/lib/fluent/plugin/in_http.rb +21 -18
  96. data/lib/fluent/plugin/in_monitor_agent.rb +15 -48
  97. data/lib/fluent/plugin/in_object_space.rb +6 -1
  98. data/lib/fluent/plugin/in_stream.rb +7 -3
  99. data/lib/fluent/plugin/in_syslog.rb +46 -95
  100. data/lib/fluent/plugin/in_tail.rb +58 -640
  101. data/lib/fluent/plugin/in_tcp.rb +8 -1
  102. data/lib/fluent/plugin/in_udp.rb +8 -18
  103. data/lib/fluent/plugin/input.rb +33 -0
  104. data/lib/fluent/plugin/multi_output.rb +95 -0
  105. data/lib/fluent/plugin/out_buffered_null.rb +59 -0
  106. data/lib/fluent/plugin/out_copy.rb +11 -7
  107. data/lib/fluent/plugin/out_exec.rb +15 -11
  108. data/lib/fluent/plugin/out_exec_filter.rb +18 -10
  109. data/lib/fluent/plugin/out_file.rb +34 -5
  110. data/lib/fluent/plugin/out_forward.rb +25 -19
  111. data/lib/fluent/plugin/out_null.rb +0 -14
  112. data/lib/fluent/plugin/out_roundrobin.rb +11 -7
  113. data/lib/fluent/plugin/out_stdout.rb +5 -7
  114. data/lib/fluent/plugin/out_stream.rb +3 -1
  115. data/lib/fluent/plugin/output.rb +979 -0
  116. data/lib/fluent/plugin/owned_by_mixin.rb +42 -0
  117. data/lib/fluent/plugin/parser.rb +244 -0
  118. data/lib/fluent/plugin/parser_apache.rb +24 -0
  119. data/lib/fluent/plugin/parser_apache2.rb +84 -0
  120. data/lib/fluent/plugin/parser_apache_error.rb +21 -0
  121. data/lib/fluent/plugin/parser_csv.rb +31 -0
  122. data/lib/fluent/plugin/parser_json.rb +79 -0
  123. data/lib/fluent/plugin/parser_ltsv.rb +50 -0
  124. data/lib/fluent/plugin/parser_multiline.rb +102 -0
  125. data/lib/fluent/plugin/parser_nginx.rb +24 -0
  126. data/lib/fluent/plugin/parser_none.rb +36 -0
  127. data/lib/fluent/plugin/parser_syslog.rb +82 -0
  128. data/lib/fluent/plugin/parser_tsv.rb +37 -0
  129. data/lib/fluent/plugin/socket_util.rb +119 -117
  130. data/lib/fluent/plugin/storage.rb +84 -0
  131. data/lib/fluent/plugin/storage_local.rb +116 -0
  132. data/lib/fluent/plugin/string_util.rb +16 -13
  133. data/lib/fluent/plugin_helper.rb +39 -0
  134. data/lib/fluent/plugin_helper/child_process.rb +298 -0
  135. data/lib/fluent/plugin_helper/compat_parameters.rb +99 -0
  136. data/lib/fluent/plugin_helper/event_emitter.rb +80 -0
  137. data/lib/fluent/plugin_helper/event_loop.rb +118 -0
  138. data/lib/fluent/plugin_helper/retry_state.rb +177 -0
  139. data/lib/fluent/plugin_helper/storage.rb +308 -0
  140. data/lib/fluent/plugin_helper/thread.rb +147 -0
  141. data/lib/fluent/plugin_helper/timer.rb +85 -0
  142. data/lib/fluent/plugin_id.rb +63 -0
  143. data/lib/fluent/process.rb +21 -30
  144. data/lib/fluent/registry.rb +21 -9
  145. data/lib/fluent/root_agent.rb +115 -40
  146. data/lib/fluent/supervisor.rb +330 -320
  147. data/lib/fluent/system_config.rb +42 -18
  148. data/lib/fluent/test.rb +6 -1
  149. data/lib/fluent/test/base.rb +23 -3
  150. data/lib/fluent/test/driver/base.rb +247 -0
  151. data/lib/fluent/test/driver/event_feeder.rb +98 -0
  152. data/lib/fluent/test/driver/filter.rb +35 -0
  153. data/lib/fluent/test/driver/input.rb +31 -0
  154. data/lib/fluent/test/driver/output.rb +78 -0
  155. data/lib/fluent/test/driver/test_event_router.rb +45 -0
  156. data/lib/fluent/test/filter_test.rb +0 -1
  157. data/lib/fluent/test/formatter_test.rb +2 -1
  158. data/lib/fluent/test/input_test.rb +23 -17
  159. data/lib/fluent/test/output_test.rb +28 -39
  160. data/lib/fluent/test/parser_test.rb +1 -1
  161. data/lib/fluent/time.rb +104 -1
  162. data/lib/fluent/{status.rb → unique_id.rb} +15 -24
  163. data/lib/fluent/version.rb +1 -1
  164. data/lib/fluent/winsvc.rb +72 -0
  165. data/test/compat/test_calls_super.rb +164 -0
  166. data/test/config/test_config_parser.rb +83 -0
  167. data/test/config/test_configurable.rb +547 -274
  168. data/test/config/test_configure_proxy.rb +146 -29
  169. data/test/config/test_dsl.rb +3 -181
  170. data/test/config/test_element.rb +274 -0
  171. data/test/config/test_literal_parser.rb +1 -1
  172. data/test/config/test_section.rb +79 -7
  173. data/test/config/test_system_config.rb +21 -0
  174. data/test/config/test_types.rb +3 -26
  175. data/test/helper.rb +78 -8
  176. data/test/plugin/test_bare_output.rb +118 -0
  177. data/test/plugin/test_base.rb +75 -0
  178. data/test/plugin/test_buf_file.rb +420 -521
  179. data/test/plugin/test_buf_memory.rb +32 -194
  180. data/test/plugin/test_buffer.rb +981 -0
  181. data/test/plugin/test_buffer_chunk.rb +110 -0
  182. data/test/plugin/test_buffer_file_chunk.rb +770 -0
  183. data/test/plugin/test_buffer_memory_chunk.rb +265 -0
  184. data/test/plugin/test_filter.rb +255 -0
  185. data/test/plugin/test_filter_grep.rb +2 -73
  186. data/test/plugin/test_filter_record_transformer.rb +24 -68
  187. data/test/plugin/test_filter_stdout.rb +6 -6
  188. data/test/plugin/test_in_debug_agent.rb +2 -0
  189. data/test/plugin/test_in_dummy.rb +11 -17
  190. data/test/plugin/test_in_exec.rb +6 -25
  191. data/test/plugin/test_in_forward.rb +112 -151
  192. data/test/plugin/test_in_gc_stat.rb +2 -0
  193. data/test/plugin/test_in_http.rb +106 -157
  194. data/test/plugin/test_in_object_space.rb +21 -5
  195. data/test/plugin/test_in_stream.rb +14 -13
  196. data/test/plugin/test_in_syslog.rb +30 -275
  197. data/test/plugin/test_in_tail.rb +95 -282
  198. data/test/plugin/test_in_tcp.rb +14 -0
  199. data/test/plugin/test_in_udp.rb +21 -67
  200. data/test/plugin/test_input.rb +122 -0
  201. data/test/plugin/test_multi_output.rb +180 -0
  202. data/test/plugin/test_out_buffered_null.rb +79 -0
  203. data/test/plugin/test_out_copy.rb +15 -2
  204. data/test/plugin/test_out_exec.rb +75 -25
  205. data/test/plugin/test_out_exec_filter.rb +74 -8
  206. data/test/plugin/test_out_file.rb +61 -7
  207. data/test/plugin/test_out_forward.rb +92 -15
  208. data/test/plugin/test_out_roundrobin.rb +1 -0
  209. data/test/plugin/test_out_stdout.rb +22 -13
  210. data/test/plugin/test_out_stream.rb +18 -0
  211. data/test/plugin/test_output.rb +515 -0
  212. data/test/plugin/test_output_as_buffered.rb +1540 -0
  213. data/test/plugin/test_output_as_buffered_overflow.rb +247 -0
  214. data/test/plugin/test_output_as_buffered_retries.rb +808 -0
  215. data/test/plugin/test_output_as_buffered_secondary.rb +776 -0
  216. data/test/plugin/test_output_as_standard.rb +362 -0
  217. data/test/plugin/test_owned_by.rb +35 -0
  218. data/test/plugin/test_storage.rb +167 -0
  219. data/test/plugin/test_storage_local.rb +8 -0
  220. data/test/plugin_helper/test_child_process.rb +599 -0
  221. data/test/plugin_helper/test_compat_parameters.rb +175 -0
  222. data/test/plugin_helper/test_event_emitter.rb +51 -0
  223. data/test/plugin_helper/test_event_loop.rb +52 -0
  224. data/test/plugin_helper/test_retry_state.rb +399 -0
  225. data/test/plugin_helper/test_storage.rb +411 -0
  226. data/test/plugin_helper/test_thread.rb +164 -0
  227. data/test/plugin_helper/test_timer.rb +100 -0
  228. data/test/scripts/exec_script.rb +0 -6
  229. data/test/scripts/fluent/plugin/out_test.rb +3 -0
  230. data/test/test_config.rb +13 -4
  231. data/test/test_event.rb +24 -13
  232. data/test/test_event_router.rb +8 -7
  233. data/test/test_event_time.rb +187 -0
  234. data/test/test_formatter.rb +13 -51
  235. data/test/test_input.rb +1 -1
  236. data/test/test_log.rb +239 -16
  237. data/test/test_mixin.rb +1 -1
  238. data/test/test_output.rb +53 -66
  239. data/test/test_parser.rb +105 -323
  240. data/test/test_plugin_helper.rb +81 -0
  241. data/test/test_root_agent.rb +4 -52
  242. data/test/test_supervisor.rb +272 -0
  243. data/test/test_unique_id.rb +47 -0
  244. metadata +181 -55
  245. data/CHANGELOG.md +0 -710
  246. data/lib/fluent/buffer.rb +0 -365
  247. data/lib/fluent/plugin/filter_parser.rb +0 -107
  248. data/lib/fluent/plugin/in_status.rb +0 -76
  249. data/lib/fluent/test/helpers.rb +0 -86
  250. data/test/plugin/data/log/foo/bar2 +0 -0
  251. data/test/plugin/test_filter_parser.rb +0 -744
  252. data/test/plugin/test_in_status.rb +0 -38
  253. data/test/test_buffer.rb +0 -624
@@ -0,0 +1,362 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/output'
3
+ require 'fluent/plugin/buffer'
4
+ require 'fluent/msgpack_factory'
5
+ require 'fluent/event'
6
+
7
+ require 'json'
8
+ require 'time'
9
+ require 'timeout'
10
+
11
+ require 'flexmock/test_unit'
12
+
13
+ module FluentPluginStandardBufferedOutputTest
14
+ class DummyBareOutput < Fluent::Plugin::Output
15
+ def register(name, &block)
16
+ instance_variable_set("@#{name}", block)
17
+ end
18
+ end
19
+ class DummyAsyncOutput < DummyBareOutput
20
+ def format(tag, time, record)
21
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
22
+ end
23
+ def write(chunk)
24
+ @write ? @write.call(chunk) : nil
25
+ end
26
+ end
27
+ class DummyAsyncStandardOutput < DummyBareOutput
28
+ def write(chunk)
29
+ @write ? @write.call(chunk) : nil
30
+ end
31
+ end
32
+ end
33
+
34
+ class StandardBufferedOutputTest < Test::Unit::TestCase
35
+ def create_output(type=:full)
36
+ case type
37
+ when :bare then FluentPluginStandardBufferedOutputTest::DummyBareOutput.new
38
+ when :buffered then FluentPluginStandardBufferedOutputTest::DummyAsyncOutput.new
39
+ when :standard then FluentPluginStandardBufferedOutputTest::DummyAsyncStandardOutput.new
40
+ else
41
+ raise ArgumentError, "unknown type: #{type}"
42
+ end
43
+ end
44
+ def create_metadata(timekey: nil, tag: nil, variables: nil)
45
+ Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
46
+ end
47
+ def waiting(seconds)
48
+ begin
49
+ Timeout.timeout(seconds) do
50
+ yield
51
+ end
52
+ rescue Timeout::Error
53
+ STDERR.print(*@i.log.out.logs)
54
+ raise
55
+ end
56
+ end
57
+ def test_event_stream
58
+ es = Fluent::MultiEventStream.new
59
+ es.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
60
+ es.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
61
+ es.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
62
+ es.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
63
+ es.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
64
+ es.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
65
+ es
66
+ end
67
+
68
+ setup do
69
+ @i = nil
70
+ end
71
+
72
+ teardown do
73
+ if @i
74
+ @i.stop unless @i.stopped?
75
+ @i.before_shutdown unless @i.before_shutdown?
76
+ @i.shutdown unless @i.shutdown?
77
+ @i.after_shutdown unless @i.after_shutdown?
78
+ @i.close unless @i.closed?
79
+ @i.terminate unless @i.terminated?
80
+ end
81
+ end
82
+
83
+ sub_test_case 'standard buffered without any chunk keys' do
84
+ test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format' do
85
+ @i = create_output(:standard)
86
+ @i.configure(config_element())
87
+ @i.start
88
+
89
+ m = create_metadata()
90
+ es = test_event_stream
91
+
92
+ buffer_mock = flexmock(@i.buffer)
93
+ buffer_mock.should_receive(:write).once.with({m => [es.to_msgpack_stream, es.size]}, bulk: true, enqueue: false)
94
+
95
+ @i.execute_chunking("mytag.test", es)
96
+ end
97
+
98
+ test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format, but time will be int if time_as_integer specified' do
99
+ @i = create_output(:standard)
100
+ @i.configure(config_element('ROOT','',{"time_as_integer"=>"true"}))
101
+ @i.start
102
+
103
+ m = create_metadata()
104
+ es = test_event_stream
105
+
106
+ buffer_mock = flexmock(@i.buffer)
107
+ buffer_mock.should_receive(:write).once.with({m => [es.to_msgpack_stream(time_int: true), es.size]}, bulk: true, enqueue: false)
108
+
109
+ @i.execute_chunking("mytag.test", es)
110
+ end
111
+ end
112
+
113
+ sub_test_case 'standard buffered with tag chunk key' do
114
+ test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format' do
115
+ @i = create_output(:standard)
116
+ @i.configure(config_element('ROOT','',{},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))
117
+ @i.start
118
+
119
+ m = create_metadata(tag: "mytag.test")
120
+ es = test_event_stream
121
+
122
+ buffer_mock = flexmock(@i.buffer)
123
+ buffer_mock.should_receive(:write).once.with({m => [es.to_msgpack_stream, es.size]}, bulk: true, enqueue: false)
124
+
125
+ @i.execute_chunking("mytag.test", es)
126
+ end
127
+
128
+ test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format, but time will be int if time_as_integer specified' do
129
+ @i = create_output(:standard)
130
+ @i.configure(config_element('ROOT','',{"time_as_integer"=>"true"},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))
131
+ @i.start
132
+
133
+ m = create_metadata(tag: "mytag.test")
134
+ es = test_event_stream
135
+
136
+ buffer_mock = flexmock(@i.buffer)
137
+ buffer_mock.should_receive(:write).once.with({m => [es.to_msgpack_stream(time_int: true), es.size]}, bulk: true, enqueue: false)
138
+
139
+ @i.execute_chunking("mytag.test", es)
140
+ end
141
+ end
142
+
143
+ sub_test_case 'standard buffered with time chunk key' do
144
+ test '#execute_chunking calls @buffer.write(bulk: true) with predefined msgpack format' do
145
+ @i = create_output(:standard)
146
+ @i.configure(config_element('ROOT','',{},[config_element('buffer','time',{"timekey" => "60",'flush_thread_burst_interval' => 0.01})]))
147
+ @i.start
148
+
149
+ m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)
150
+ m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)
151
+ m3 = create_metadata(timekey: Time.parse('2016-04-21 17:21:00 -0700').to_i)
152
+
153
+ es1 = Fluent::MultiEventStream.new
154
+ es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
155
+ es1.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
156
+ es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
157
+
158
+ es2 = Fluent::MultiEventStream.new
159
+ es2.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
160
+ es2.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
161
+
162
+ es3 = Fluent::MultiEventStream.new
163
+ es3.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
164
+
165
+ buffer_mock = flexmock(@i.buffer)
166
+ buffer_mock.should_receive(:write).once.with({
167
+ m1 => [es1.to_msgpack_stream, 3],
168
+ m2 => [es2.to_msgpack_stream, 2],
169
+ m3 => [es3.to_msgpack_stream, 1],
170
+ }, bulk: true, enqueue: false)
171
+
172
+ es = test_event_stream
173
+ @i.execute_chunking("mytag.test", es)
174
+ end
175
+
176
+ test '#execute_chunking calls @buffer.write(bulk: true) with predefined msgpack format, but time will be int if time_as_integer specified' do
177
+ @i = create_output(:standard)
178
+ @i.configure(config_element('ROOT','',{"time_as_integer" => "true"},[config_element('buffer','time',{"timekey" => "60",'flush_thread_burst_interval' => 0.01})]))
179
+ @i.start
180
+
181
+ m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)
182
+ m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)
183
+ m3 = create_metadata(timekey: Time.parse('2016-04-21 17:21:00 -0700').to_i)
184
+
185
+ es1 = Fluent::MultiEventStream.new
186
+ es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
187
+ es1.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
188
+ es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
189
+
190
+ es2 = Fluent::MultiEventStream.new
191
+ es2.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
192
+ es2.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
193
+
194
+ es3 = Fluent::MultiEventStream.new
195
+ es3.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
196
+
197
+ buffer_mock = flexmock(@i.buffer)
198
+ buffer_mock.should_receive(:write).with({
199
+ m1 => [es1.to_msgpack_stream(time_int: true), 3],
200
+ m2 => [es2.to_msgpack_stream(time_int: true), 2],
201
+ m3 => [es3.to_msgpack_stream(time_int: true), 1],
202
+ }, bulk: true, enqueue: false)
203
+
204
+ es = test_event_stream
205
+ @i.execute_chunking("mytag.test", es)
206
+ end
207
+ end
208
+
209
+ sub_test_case 'standard buffered with variable chunk keys' do
210
+ test '#execute_chunking calls @buffer.write(bulk: true) with predefined msgpack format' do
211
+ @i = create_output(:standard)
212
+ @i.configure(config_element('ROOT','',{},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))
213
+ @i.start
214
+
215
+ m1 = create_metadata(variables: {key: "my value", name: "moris1"})
216
+ es1 = Fluent::MultiEventStream.new
217
+ es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
218
+ es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
219
+ es1.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
220
+ es1.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
221
+ es1.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
222
+
223
+ m2 = create_metadata(variables: {key: "my value", name: "moris2"})
224
+ es2 = Fluent::MultiEventStream.new
225
+ es2.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
226
+
227
+ buffer_mock = flexmock(@i.buffer)
228
+ buffer_mock.should_receive(:write).with({
229
+ m1 => [es1.to_msgpack_stream, 5],
230
+ m2 => [es2.to_msgpack_stream, 1],
231
+ }, bulk: true, enqueue: false).once
232
+
233
+ es = test_event_stream
234
+ @i.execute_chunking("mytag.test", es)
235
+ end
236
+
237
+ test '#execute_chunking calls @buffer.write(bulk: true) in times of # of variable variations with predefined msgpack format, but time will be int if time_as_integer specified' do
238
+ @i = create_output(:standard)
239
+ @i.configure(config_element('ROOT','',{"time_as_integer" => "true"},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))
240
+ @i.start
241
+
242
+ m1 = create_metadata(variables: {key: "my value", name: "moris1"})
243
+ es1 = Fluent::MultiEventStream.new
244
+ es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
245
+ es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
246
+ es1.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
247
+ es1.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
248
+ es1.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
249
+
250
+ m2 = create_metadata(variables: {key: "my value", name: "moris2"})
251
+ es2 = Fluent::MultiEventStream.new
252
+ es2.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
253
+
254
+ buffer_mock = flexmock(@i.buffer)
255
+ buffer_mock.should_receive(:write).with({
256
+ m1 => [es1.to_msgpack_stream(time_int: true), 5],
257
+ m2 => [es2.to_msgpack_stream(time_int: true), 1],
258
+ }, bulk: true, enqueue: false).once
259
+
260
+ es = test_event_stream
261
+ @i.execute_chunking("mytag.test", es)
262
+ end
263
+ end
264
+
265
+ sub_test_case 'custom format buffered without any chunk keys' do
266
+ test '#execute_chunking calls @buffer.write(bulk: true) just once with customized format' do
267
+ @i = create_output(:buffered)
268
+ @i.register(:format){|tag, time, record| [time, record].to_json }
269
+ @i.configure(config_element())
270
+ @i.start
271
+
272
+ m = create_metadata()
273
+ es = test_event_stream
274
+
275
+ buffer_mock = flexmock(@i.buffer)
276
+ buffer_mock.should_receive(:write).once.with({m => [es.map{|t,r| [t,r].to_json }.join, es.size]}, bulk: true, enqueue: false)
277
+
278
+ @i.execute_chunking("mytag.test", es)
279
+ end
280
+ end
281
+
282
+ sub_test_case 'custom format buffered with tag chunk key' do
283
+ test '#execute_chunking calls @buffer.write(bulk: true) just once with customized format' do
284
+ @i = create_output(:buffered)
285
+ @i.register(:format){|tag, time, record| [time, record].to_json }
286
+ @i.configure(config_element('ROOT','',{},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))
287
+ @i.start
288
+
289
+ m = create_metadata(tag: "mytag.test")
290
+ es = test_event_stream
291
+
292
+ buffer_mock = flexmock(@i.buffer)
293
+ buffer_mock.should_receive(:write).once.with({m => [es.map{|t,r| [t,r].to_json }.join, es.size]}, bulk: true, enqueue: false)
294
+
295
+ @i.execute_chunking("mytag.test", es)
296
+ end
297
+ end
298
+ sub_test_case 'custom format buffered with time chunk key' do
299
+ test '#execute_chunking calls @buffer.write with customized format' do
300
+ @i = create_output(:buffered)
301
+ @i.register(:format){|tag, time, record| [time, record].to_json }
302
+ @i.configure(config_element('ROOT','',{},[config_element('buffer','time',{"timekey" => "60",'flush_thread_burst_interval' => 0.01})]))
303
+ @i.start
304
+
305
+ m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)
306
+ m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)
307
+ m3 = create_metadata(timekey: Time.parse('2016-04-21 17:21:00 -0700').to_i)
308
+
309
+ es1 = Fluent::MultiEventStream.new
310
+ es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
311
+ es1.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
312
+ es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
313
+
314
+ es2 = Fluent::MultiEventStream.new
315
+ es2.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
316
+ es2.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
317
+
318
+ es3 = Fluent::MultiEventStream.new
319
+ es3.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
320
+
321
+ buffer_mock = flexmock(@i.buffer)
322
+ buffer_mock.should_receive(:write).with({
323
+ m1 => es1.map{|t,r| [t,r].to_json },
324
+ m2 => es2.map{|t,r| [t,r].to_json },
325
+ m3 => es3.map{|t,r| [t,r].to_json },
326
+ }, bulk: false, enqueue: false).once
327
+
328
+ es = test_event_stream
329
+ @i.execute_chunking("mytag.test", es)
330
+ end
331
+ end
332
+
333
+ sub_test_case 'custom format buffered with variable chunk keys' do
334
+ test '#execute_chunking calls @buffer.write in times of # of variable variations with customized format' do
335
+ @i = create_output(:buffered)
336
+ @i.register(:format){|tag, time, record| [time, record].to_json }
337
+ @i.configure(config_element('ROOT','',{},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))
338
+ @i.start
339
+
340
+ m1 = create_metadata(variables: {key: "my value", name: "moris1"})
341
+ es1 = Fluent::MultiEventStream.new
342
+ es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
343
+ es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
344
+ es1.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
345
+ es1.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
346
+ es1.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
347
+
348
+ m2 = create_metadata(variables: {key: "my value", name: "moris2"})
349
+ es2 = Fluent::MultiEventStream.new
350
+ es2.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
351
+
352
+ buffer_mock = flexmock(@i.buffer)
353
+ buffer_mock.should_receive(:write).with({
354
+ m1 => es1.map{|t,r| [t,r].to_json },
355
+ m2 => es2.map{|t,r| [t,r].to_json },
356
+ }, bulk: false, enqueue: false).once
357
+
358
+ es = test_event_stream
359
+ @i.execute_chunking("mytag.test", es)
360
+ end
361
+ end
362
+ end
@@ -0,0 +1,35 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/base'
3
+ require 'fluent/plugin/input'
4
+ require 'fluent/plugin/owned_by_mixin'
5
+
6
+ module OwnedByMixinTestEnv
7
+ class DummyParent < Fluent::Plugin::Input
8
+ Fluent::Plugin.register_input('dummy_parent', self)
9
+ end
10
+ class DummyChild < Fluent::Plugin::Base
11
+ include Fluent::Plugin::OwnedByMixin
12
+ Fluent::Plugin.register_parser('dummy_child', self)
13
+ end
14
+ end
15
+
16
+ class OwnedByMixinTest < Test::Unit::TestCase
17
+ sub_test_case 'Owned plugins' do
18
+ setup do
19
+ Fluent::Test.setup
20
+ end
21
+
22
+ test 'inherits plugin id and logger from parent' do
23
+ parent = Fluent::Plugin.new_input('dummy_parent')
24
+ parent.configure(config_element('ROOT', '', {'@id' => 'my_parent_id', '@log_level' => 'trace'}))
25
+ child = Fluent::Plugin.new_parser('dummy_child', parent: parent)
26
+
27
+ assert_equal parent.object_id, child.owner.object_id
28
+
29
+ assert child.instance_eval{ @_plugin_id_configured }
30
+ assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id }
31
+
32
+ assert_equal Fluent::Log::LEVEL_TRACE, child.log.level
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,167 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/storage'
3
+ require 'fluent/plugin/base'
4
+
5
+ class DummyPlugin < Fluent::Plugin::TestBase
6
+ end
7
+
8
+ class BareStorage < Fluent::Plugin::Storage
9
+ Fluent::Plugin.register_storage('bare', self)
10
+ end
11
+
12
+ class BasicStorage < Fluent::Plugin::Storage
13
+ Fluent::Plugin.register_storage('example', self)
14
+
15
+ attr_reader :data, :saved
16
+
17
+ def initialize
18
+ super
19
+ @data = @saved = nil
20
+ end
21
+ def load
22
+ @data = {}
23
+ end
24
+ def save
25
+ @saved = @data.dup
26
+ end
27
+ def get(key)
28
+ @data[key]
29
+ end
30
+ def fetch(key, defval)
31
+ @data.fetch(key, defval)
32
+ end
33
+ def put(key, value)
34
+ @data[key] = value
35
+ end
36
+ def delete(key)
37
+ @data.delete(key)
38
+ end
39
+ def update(key, &block)
40
+ @data[key] = block.call(@data[key])
41
+ end
42
+ def close
43
+ @data = {}
44
+ super
45
+ end
46
+ def terminate
47
+ @saved = {}
48
+ super
49
+ end
50
+ end
51
+
52
+ class StorageTest < Test::Unit::TestCase
53
+ sub_test_case 'BareStorage' do
54
+ setup do
55
+ plugin = DummyPlugin.new
56
+ @s = BareStorage.new
57
+ @s.configure(config_element())
58
+ @s.owner = plugin
59
+ end
60
+
61
+ test 'is configured with plugin information and system config' do
62
+ plugin = DummyPlugin.new
63
+ plugin.system_config_override({'process_name' => 'mytest'})
64
+ plugin.configure(config_element('ROOT', '', {'@id' => '1'}))
65
+ s = BareStorage.new
66
+ s.configure(config_element())
67
+ s.owner = plugin
68
+
69
+ assert_equal 'mytest', s.owner.system_config.process_name
70
+ assert_equal '1', s.instance_eval{ @_plugin_id }
71
+ assert_equal true, s.instance_eval{ @_plugin_id_configured }
72
+ end
73
+
74
+ test 'does NOT have features for high-performance/high-consistent storages' do
75
+ assert_equal false, @s.persistent_always?
76
+ assert_equal false, @s.synchronized?
77
+ end
78
+
79
+ test 'does have default values which is conservative for almost all users' do
80
+ assert_equal false, @s.persistent
81
+ assert_equal true, @s.autosave
82
+ assert_equal 10, @s.autosave_interval
83
+ assert_equal true, @s.save_at_shutdown
84
+ end
85
+
86
+ test 'load/save doesn NOT anything: just as memory storage' do
87
+ assert_nothing_raised{ @s.load }
88
+ assert_nothing_raised{ @s.save }
89
+ end
90
+
91
+ test 'all operations are not defined yet' do
92
+ assert_raise NotImplementedError do
93
+ @s.get('key')
94
+ end
95
+ assert_raise NotImplementedError do
96
+ @s.fetch('key', 'value')
97
+ end
98
+ assert_raise NotImplementedError do
99
+ @s.put('key', 'value')
100
+ end
101
+ assert_raise NotImplementedError do
102
+ @s.delete('key')
103
+ end
104
+ assert_raise NotImplementedError do
105
+ @s.update('key'){ |v| v + '2' }
106
+ end
107
+ end
108
+ end
109
+
110
+ sub_test_case 'ExampleStorage' do
111
+ setup do
112
+ plugin = DummyPlugin.new
113
+ plugin.configure(config_element('ROOT', '', {'@id' => '1'}))
114
+ @s = BasicStorage.new
115
+ @s.configure(config_element())
116
+ @s.owner = plugin
117
+ end
118
+
119
+ test 'load/save works well as plugin internal state operations' do
120
+ plugin = DummyPlugin.new
121
+ plugin.configure(config_element('ROOT', '', {'@id' => '0'}))
122
+ s = BasicStorage.new
123
+ s.owner = plugin
124
+
125
+ assert_nothing_raised{ s.load }
126
+ assert s.data
127
+ assert_nil s.saved
128
+
129
+ assert_nothing_raised{ s.save }
130
+ assert s.saved
131
+ assert{ s.data == s.saved }
132
+ assert{ s.data.object_id != s.saved.object_id }
133
+ end
134
+
135
+ test 'all operations work well' do
136
+ @s.load
137
+
138
+ assert_nil @s.get('key')
139
+ assert_equal 'value', @s.fetch('key', 'value')
140
+ assert_nil @s.get('key')
141
+
142
+ assert_equal 'value', @s.put('key', 'value')
143
+ assert_equal 'value', @s.get('key')
144
+
145
+ assert_equal 'valuevalue', @s.update('key'){|v| v * 2 }
146
+
147
+ assert_equal 'valuevalue', @s.delete('key')
148
+ end
149
+
150
+ test 'close and terminate work to operate internal states' do
151
+ @s.load
152
+ @s.put('k1', 'v1')
153
+ @s.put('k2', 'v2')
154
+ assert_equal 2, @s.data.size
155
+ @s.save
156
+ assert_equal @s.data.size, @s.saved.size
157
+
158
+ assert_nothing_raised{ @s.close }
159
+ assert @s.data.empty?
160
+ assert !@s.saved.empty?
161
+
162
+ assert_nothing_raised{ @s.terminate }
163
+ assert @s.data.empty?
164
+ assert @s.saved.empty?
165
+ end
166
+ end
167
+ end