fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -0,0 +1,258 @@
1
+ require_relative '../helper'
2
+ require 'fluent/counter/store'
3
+ require 'fluent/time'
4
+ require 'timecop'
5
+
6
+ class CounterStoreTest < ::Test::Unit::TestCase
7
+ setup do
8
+ @name = 'key_name'
9
+ @scope = "server\tworker\tplugin"
10
+
11
+ # timecop isn't compatible with EventTime
12
+ t = Time.parse('2016-09-22 16:59:59 +0900')
13
+ Timecop.freeze(t)
14
+ @now = Fluent::EventTime.now
15
+ end
16
+
17
+ shutdown do
18
+ Timecop.return
19
+ end
20
+
21
+ def extract_value_from_counter(counter, key)
22
+ store = counter.instance_variable_get(:@storage).instance_variable_get(:@store)
23
+ store[key]
24
+ end
25
+
26
+ def travel(sec)
27
+ # Since Timecop.travel() causes test failures on Windows/AppVeyor by inducing
28
+ # rounding errors to Time.now, we need to use Timecop.freeze() instead.
29
+ Timecop.freeze(Time.now + sec)
30
+ end
31
+
32
+ sub_test_case 'init' do
33
+ setup do
34
+ @reset_interval = 10
35
+ @store = Fluent::Counter::Store.new
36
+ @data = { 'name' => @name, 'reset_interval' => @reset_interval }
37
+ @key = Fluent::Counter::Store.gen_key(@scope, @name)
38
+ end
39
+
40
+ test 'create new value in the counter' do
41
+ v = @store.init(@key, @data)
42
+
43
+ assert_equal @name, v['name']
44
+ assert_equal @reset_interval, v['reset_interval']
45
+
46
+ v2 = extract_value_from_counter(@store, @key)
47
+ v2 = @store.send(:build_response, v2)
48
+ assert_equal v, v2
49
+ end
50
+
51
+ test 'raise an error when a passed key already exists' do
52
+ @store.init(@key, @data)
53
+
54
+ assert_raise Fluent::Counter::InvalidParams do
55
+ @store.init(@key, @data)
56
+ end
57
+ end
58
+
59
+ test 'return a value when passed key already exists and a ignore option is true' do
60
+ v = @store.init(@key, @data)
61
+ v1 = extract_value_from_counter(@store, @key)
62
+ v1 = @store.send(:build_response, v1)
63
+ v2 = @store.init(@key, @data, ignore: true)
64
+ assert_equal v, v2
65
+ assert_equal v1, v2
66
+ end
67
+ end
68
+
69
+ sub_test_case 'get' do
70
+ setup do
71
+ @store = Fluent::Counter::Store.new
72
+ data = { 'name' => @name, 'reset_interval' => 10 }
73
+ @key = Fluent::Counter::Store.gen_key(@scope, @name)
74
+ @store.init(@key, data)
75
+ end
76
+
77
+ test 'return a value from the counter' do
78
+ v = extract_value_from_counter(@store, @key)
79
+ expected = @store.send(:build_response, v)
80
+ assert_equal expected, @store.get(@key)
81
+ end
82
+
83
+ test 'return a raw value from the counter when raw option is true' do
84
+ v = extract_value_from_counter(@store, @key)
85
+ assert_equal v, @store.get(@key, raw: true)
86
+ end
87
+
88
+ test "return nil when a passed key doesn't exist" do
89
+ assert_equal nil, @store.get('unknown_key')
90
+ end
91
+
92
+ test "raise a error when when a passed key doesn't exist and raise_error option is true" do
93
+ assert_raise Fluent::Counter::UnknownKey do
94
+ @store.get('unknown_key', raise_error: true)
95
+ end
96
+ end
97
+ end
98
+
99
+ sub_test_case 'key?' do
100
+ setup do
101
+ @store = Fluent::Counter::Store.new
102
+ data = { 'name' => @name, 'reset_interval' => 10 }
103
+ @key = Fluent::Counter::Store.gen_key(@scope, @name)
104
+ @store.init(@key, data)
105
+ end
106
+
107
+ test 'return true when passed key exists' do
108
+ assert_true @store.key?(@key)
109
+ end
110
+
111
+ test "return false when passed key doesn't exist" do
112
+ assert_true !@store.key?('unknown_key')
113
+ end
114
+ end
115
+
116
+ sub_test_case 'delete' do
117
+ setup do
118
+ @store = Fluent::Counter::Store.new
119
+ data = { 'name' => @name, 'reset_interval' => 10 }
120
+ @key = Fluent::Counter::Store.gen_key(@scope, @name)
121
+ @init_value = @store.init(@key, data)
122
+ end
123
+
124
+ test 'delete a value from the counter' do
125
+ v = @store.delete(@key)
126
+ assert_equal @init_value, v
127
+ assert_nil extract_value_from_counter(@store, @key)
128
+ end
129
+
130
+ test "raise an error when passed key doesn't exist" do
131
+ assert_raise Fluent::Counter::UnknownKey do
132
+ @store.delete('unknown_key')
133
+ end
134
+ end
135
+ end
136
+
137
+ sub_test_case 'inc' do
138
+ setup do
139
+ @store = Fluent::Counter::Store.new
140
+ @init_data = { 'name' => @name, 'reset_interval' => 10 }
141
+ @travel_sec = 10
142
+ end
143
+
144
+ data(
145
+ positive: 10,
146
+ negative: -10
147
+ )
148
+ test 'increment or decrement a value in the counter' do |value|
149
+ key = Fluent::Counter::Store.gen_key(@scope, @name)
150
+ @store.init(key, @init_data)
151
+ travel(@travel_sec)
152
+ v = @store.inc(key, { 'value' => value })
153
+
154
+ assert_equal value, v['total']
155
+ assert_equal value, v['current']
156
+ assert_equal @now, v['last_reset_at'] # last_reset_at doesn't change
157
+
158
+ v1 = extract_value_from_counter(@store, key)
159
+ v1 = @store.send(:build_response, v1)
160
+ assert_equal v, v1
161
+ end
162
+
163
+ test "raise an error when passed key doesn't exist" do
164
+ assert_raise Fluent::Counter::UnknownKey do
165
+ @store.inc('unknown_key', { 'value' => 1 })
166
+ end
167
+ end
168
+
169
+ test 'raise an error when a type of passed value is incompatible with a stored value' do
170
+ key1 = Fluent::Counter::Store.gen_key(@scope, @name)
171
+ key2 = Fluent::Counter::Store.gen_key(@scope, 'name2')
172
+ key3 = Fluent::Counter::Store.gen_key(@scope, 'name3')
173
+ v1 = @store.init(key1, @init_data.merge('type' => 'integer'))
174
+ v2 = @store.init(key2, @init_data.merge('type' => 'float'))
175
+ v3 = @store.init(key3, @init_data.merge('type' => 'numeric'))
176
+ assert_equal 'integer', v1['type']
177
+ assert_equal 'float', v2['type']
178
+ assert_equal 'numeric', v3['type']
179
+
180
+ assert_raise Fluent::Counter::InvalidParams do
181
+ @store.inc(key1, { 'value' => 1.1 })
182
+ end
183
+
184
+ assert_raise Fluent::Counter::InvalidParams do
185
+ @store.inc(key2, { 'value' => 1 })
186
+ end
187
+
188
+ assert_nothing_raised do
189
+ @store.inc(key3, { 'value' => 1 })
190
+ @store.inc(key3, { 'value' => 1.0 })
191
+ end
192
+ end
193
+ end
194
+
195
+ sub_test_case 'reset' do
196
+ setup do
197
+ @store = Fluent::Counter::Store.new
198
+ @travel_sec = 10
199
+
200
+ @inc_value = 10
201
+ @key = Fluent::Counter::Store.gen_key(@scope, @name)
202
+ @store.init(@key, { 'name' => @name, 'reset_interval' => 10 })
203
+ @store.inc(@key, { 'value' => 10 })
204
+ end
205
+
206
+ test 'reset a value in the counter' do
207
+ travel(@travel_sec)
208
+
209
+ v = @store.reset(@key)
210
+ assert_equal @travel_sec, v['elapsed_time']
211
+ assert_true v['success']
212
+ counter = v['counter_data']
213
+
214
+ assert_equal @name, counter['name']
215
+ assert_equal @inc_value, counter['total']
216
+ assert_equal @inc_value, counter['current']
217
+ assert_equal 'numeric', counter['type']
218
+ assert_equal @now, counter['last_reset_at']
219
+ assert_equal 10, counter['reset_interval']
220
+
221
+ v1 = extract_value_from_counter(@store, @key)
222
+ assert_equal 0, v1['current']
223
+ assert_true v1['current'].is_a?(Integer)
224
+ assert_equal @inc_value, v1['total']
225
+ assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])
226
+ assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])
227
+ end
228
+
229
+ test 'reset a value after `reset_interval` passed' do
230
+ first_travel_sec = 5
231
+ travel(first_travel_sec) # jump time less than reset_interval
232
+ v = @store.reset(@key)
233
+
234
+ assert_equal false, v['success']
235
+ assert_equal first_travel_sec, v['elapsed_time']
236
+ store = extract_value_from_counter(@store, @key)
237
+ assert_equal 10, store['current']
238
+ assert_equal @now, Fluent::EventTime.new(*store['last_reset_at'])
239
+
240
+ # time is passed greater than reset_interval
241
+ travel(@travel_sec)
242
+ v = @store.reset(@key)
243
+ assert_true v['success']
244
+ assert_equal @travel_sec + first_travel_sec, v['elapsed_time']
245
+
246
+ v1 = extract_value_from_counter(@store, @key)
247
+ assert_equal 0, v1['current']
248
+ assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])
249
+ assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])
250
+ end
251
+
252
+ test "raise an error when passed key doesn't exist" do
253
+ assert_raise Fluent::Counter::UnknownKey do
254
+ @store.reset('unknown_key')
255
+ end
256
+ end
257
+ end
258
+ end
@@ -0,0 +1,137 @@
1
+ require_relative '../helper'
2
+ require 'fluent/counter/validator'
3
+
4
+ class CounterValidatorTest < ::Test::Unit::TestCase
5
+ data(
6
+ invalid_name1: '',
7
+ invalid_name3: '_',
8
+ invalid_name4: 'A',
9
+ invalid_name5: 'a*',
10
+ invalid_name6: "a\t",
11
+ invalid_name7: "\n",
12
+ )
13
+ test 'invalid name' do |invalid_name|
14
+ assert_nil(Fluent::Counter::Validator::VALID_NAME =~ invalid_name)
15
+ end
16
+
17
+ sub_test_case 'request' do
18
+ test 'return an empty array' do
19
+ data = { 'id' => 0, 'method' => 'init' }
20
+ errors = Fluent::Counter::Validator.request(data)
21
+ assert_empty errors
22
+ end
23
+
24
+ data(
25
+ missing_id: [
26
+ { 'method' => 'init' },
27
+ { 'code' => 'invalid_request', 'message' => 'Request should include `id`' }
28
+ ],
29
+ missing_method: [
30
+ { 'id' => 0 },
31
+ { 'code' => 'invalid_request', 'message' => 'Request should include `method`' }
32
+ ],
33
+ invalid_method: [
34
+ { 'id' => 0, 'method' => "A\t" },
35
+ { 'code' => 'invalid_request', 'message' => '`method` is the invalid format' }
36
+ ],
37
+ unknown_method: [
38
+ { 'id' => 0, 'method' => 'unknown_method' },
39
+ { 'code' => 'method_not_found', 'message' => 'Unknown method name passed: unknown_method' }
40
+ ]
41
+ )
42
+ test 'return an error array' do |(data, expected_error)|
43
+ errors = Fluent::Counter::Validator.request(data)
44
+ assert_equal [expected_error], errors
45
+ end
46
+ end
47
+
48
+ sub_test_case 'call' do
49
+ test "return an error hash when passed method doesn't exist" do
50
+ v = Fluent::Counter::Validator.new(:unknown)
51
+ success, errors = v.call(['key1'])
52
+ assert_empty success
53
+ assert_equal 'internal_server_error', errors.first.to_hash['code']
54
+ end
55
+ end
56
+
57
+ test 'validate_empty!' do
58
+ v = Fluent::Counter::Validator.new(:empty)
59
+ success, errors = v.call([])
60
+ assert_empty success
61
+ assert_equal [Fluent::Counter::InvalidParams.new('One or more `params` are required')], errors
62
+ end
63
+ end
64
+
65
+ class CounterArrayValidatorTest < ::Test::Unit::TestCase
66
+ test 'validate_key!' do
67
+ ary = ['key', 100, '_']
68
+ error_expected = [
69
+ { 'code' => 'invalid_params', 'message' => 'The type of `key` should be String' },
70
+ { 'code' => 'invalid_params', 'message' => '`key` is the invalid format' }
71
+ ]
72
+ v = Fluent::Counter::ArrayValidator.new(:key)
73
+ valid_params, errors = v.call(ary)
74
+
75
+ assert_equal ['key'], valid_params
76
+ assert_equal error_expected, errors.map(&:to_hash)
77
+ end
78
+ end
79
+
80
+ class CounterHashValidatorTest < ::Test::Unit::TestCase
81
+ test 'validate_name!' do
82
+ hash = [
83
+ { 'name' => 'key' },
84
+ {},
85
+ { 'name' => 10 },
86
+ { 'name' => '_' }
87
+ ]
88
+ error_expected = [
89
+ { 'code' => 'invalid_params', 'message' => '`name` is required' },
90
+ { 'code' => 'invalid_params', 'message' => 'The type of `name` should be String' },
91
+ { 'code' => 'invalid_params', 'message' => '`name` is the invalid format' },
92
+ ]
93
+ v = Fluent::Counter::HashValidator.new(:name)
94
+ success, errors = v.call(hash)
95
+
96
+ assert_equal [{ 'name' => 'key' }], success
97
+ assert_equal error_expected, errors.map(&:to_hash)
98
+ end
99
+
100
+ test 'validate_value!' do
101
+ hash = [
102
+ { 'value' => 1 },
103
+ { 'value' => -1 },
104
+ {},
105
+ { 'value' => 'str' }
106
+ ]
107
+ error_expected = [
108
+ { 'code' => 'invalid_params', 'message' => '`value` is required' },
109
+ { 'code' => 'invalid_params', 'message' => 'The type of `value` type should be Numeric' },
110
+ ]
111
+ v = Fluent::Counter::HashValidator.new(:value)
112
+ valid_params, errors = v.call(hash)
113
+
114
+ assert_equal [{ 'value' => 1 }, { 'value' => -1 }], valid_params
115
+ assert_equal error_expected, errors.map(&:to_hash)
116
+ end
117
+
118
+ test 'validate_reset_interval!' do
119
+ hash = [
120
+ { 'reset_interval' => 1 },
121
+ { 'reset_interval' => 1.0 },
122
+ {},
123
+ { 'reset_interval' => -1 },
124
+ { 'reset_interval' => 'str' }
125
+ ]
126
+ error_expected = [
127
+ { 'code' => 'invalid_params', 'message' => '`reset_interval` is required' },
128
+ { 'code' => 'invalid_params', 'message' => '`reset_interval` should be a positive number' },
129
+ { 'code' => 'invalid_params', 'message' => 'The type of `reset_interval` should be Numeric' },
130
+ ]
131
+ v = Fluent::Counter::HashValidator.new(:reset_interval)
132
+ valid_params, errors = v.call(hash)
133
+
134
+ assert_equal [{ 'reset_interval' => 1 }, { 'reset_interval' => 1.0 }], valid_params
135
+ assert_equal error_expected.map(&:to_hash), errors.map(&:to_hash)
136
+ end
137
+ end
@@ -840,4 +840,128 @@ class FileBufferTest < Test::Unit::TestCase
840
840
  assert File.exist?(@not_chunk)
841
841
  end
842
842
  end
843
+
844
+ sub_test_case 'there are existing broken file chunks' do
845
+ setup do
846
+ @bufdir = File.expand_path('../../tmp/broken_buffer_file', __FILE__)
847
+ FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir)
848
+ @bufpath = File.join(@bufdir, 'broken_test.*.log')
849
+
850
+ Fluent::Test.setup
851
+ @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
852
+ @p = Fluent::Plugin::FileBuffer.new
853
+ @p.owner = @d
854
+ @p.configure(config_element('buffer', '', {'path' => @bufpath}))
855
+ end
856
+
857
+ teardown do
858
+ if @p
859
+ @p.stop unless @p.stopped?
860
+ @p.before_shutdown unless @p.before_shutdown?
861
+ @p.shutdown unless @p.shutdown?
862
+ @p.after_shutdown unless @p.after_shutdown?
863
+ @p.close unless @p.closed?
864
+ @p.terminate unless @p.terminated?
865
+ end
866
+ if @bufdir
867
+ Dir.glob(File.join(@bufdir, '*')).each do |path|
868
+ next if ['.', '..'].include?(File.basename(path))
869
+ File.delete(path)
870
+ end
871
+ end
872
+ end
873
+
874
+ def create_first_chunk(mode)
875
+ cid = Fluent::UniqueId.generate
876
+ path = File.join(@bufdir, "broken_test.#{mode}#{Fluent::UniqueId.hex(cid)}.log")
877
+ File.open(path, 'wb') do |f|
878
+ f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
879
+ f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
880
+ f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
881
+ f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n"
882
+ end
883
+ write_metadata(
884
+ path + '.meta', cid, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),
885
+ 4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i
886
+ )
887
+
888
+ return cid, path
889
+ end
890
+
891
+ def create_second_chunk(mode)
892
+ cid = Fluent::UniqueId.generate
893
+ path = File.join(@bufdir, "broken_test.#{mode}#{Fluent::UniqueId.hex(cid)}.log")
894
+ File.open(path, 'wb') do |f|
895
+ f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
896
+ f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
897
+ f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
898
+ end
899
+ write_metadata(
900
+ path + '.meta', cid, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i),
901
+ 3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i
902
+ )
903
+
904
+ return cid, path
905
+ end
906
+
907
+ def compare_staged_chunk(staged, id, time, num, mode)
908
+ assert_equal 1, staged.size
909
+ m = metadata(timekey: event_time(time).to_i)
910
+ assert_equal id, staged[m].unique_id
911
+ assert_equal num, staged[m].size
912
+ assert_equal mode, staged[m].state
913
+ end
914
+
915
+ def compare_queued_chunk(queued, id, num, mode)
916
+ assert_equal 1, queued.size
917
+ assert_equal id, queued[0].unique_id
918
+ assert_equal num, queued[0].size
919
+ assert_equal mode, queued[0].state
920
+ end
921
+
922
+ def compare_log(plugin, msg)
923
+ logs = plugin.log.out.logs
924
+ assert { logs.any? { |log| log.include?(msg) } }
925
+ end
926
+
927
+ test '#resume ignores staged empty chunk' do
928
+ _, p1 = create_first_chunk('b')
929
+ File.open(p1, 'wb') { |f| } # create staged empty chunk file
930
+ c2id, _ = create_second_chunk('b')
931
+
932
+ @p.start
933
+ compare_staged_chunk(@p.stage, c2id, '2016-04-17 14:01:00 -0700', 3, :staged)
934
+ compare_log(@p, 'staged file chunk is empty')
935
+ end
936
+
937
+ test '#resume ignores staged broken metadata' do
938
+ c1id, _ = create_first_chunk('b')
939
+ _, p2 = create_second_chunk('b')
940
+ File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
941
+
942
+ @p.start
943
+ compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
944
+ compare_log(@p, 'staged meta file is broken')
945
+ end
946
+
947
+ test '#resume ignores enqueued empty chunk' do
948
+ _, p1 = create_first_chunk('q')
949
+ File.open(p1, 'wb') { |f| } # create enqueued empty chunk file
950
+ c2id, _ = create_second_chunk('q')
951
+
952
+ @p.start
953
+ compare_queued_chunk(@p.queue, c2id, 3, :queued)
954
+ compare_log(@p, 'enqueued file chunk is empty')
955
+ end
956
+
957
+ test '#resume ignores enqueued broken metadata' do
958
+ c1id, _ = create_first_chunk('q')
959
+ _, p2 = create_second_chunk('q')
960
+ File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create enqueued broken meta file
961
+
962
+ @p.start
963
+ compare_queued_chunk(@p.queue, c1id, 4, :queued)
964
+ compare_log(@p, 'enqueued meta file is broken')
965
+ end
966
+ end
843
967
  end