fluentd 0.14.10-x86-mingw32 → 0.14.11-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +14 -6
  3. data/ChangeLog +28 -2
  4. data/appveyor.yml +1 -0
  5. data/lib/fluent/engine.rb +4 -7
  6. data/lib/fluent/error.rb +30 -0
  7. data/lib/fluent/log.rb +0 -7
  8. data/lib/fluent/plugin/base.rb +11 -0
  9. data/lib/fluent/plugin/buf_file.rb +9 -7
  10. data/lib/fluent/plugin/formatter_csv.rb +4 -2
  11. data/lib/fluent/plugin/in_forward.rb +46 -17
  12. data/lib/fluent/plugin/in_http.rb +2 -0
  13. data/lib/fluent/plugin/in_monitor_agent.rb +27 -2
  14. data/lib/fluent/plugin/in_syslog.rb +52 -36
  15. data/lib/fluent/plugin/in_tail.rb +1 -0
  16. data/lib/fluent/plugin/out_forward.rb +39 -29
  17. data/lib/fluent/plugin/output.rb +17 -0
  18. data/lib/fluent/plugin/storage_local.rb +16 -13
  19. data/lib/fluent/plugin_helper/storage.rb +21 -9
  20. data/lib/fluent/plugin_id.rb +17 -0
  21. data/lib/fluent/supervisor.rb +73 -45
  22. data/lib/fluent/system_config.rb +24 -21
  23. data/lib/fluent/version.rb +1 -1
  24. data/test/command/test_fluentd.rb +348 -0
  25. data/test/config/test_system_config.rb +39 -31
  26. data/test/plugin/test_base.rb +20 -0
  27. data/test/plugin/test_buf_file.rb +40 -0
  28. data/test/plugin/test_formatter_csv.rb +8 -0
  29. data/test/plugin/test_in_forward.rb +56 -21
  30. data/test/plugin/test_in_monitor_agent.rb +80 -8
  31. data/test/plugin/test_in_syslog.rb +75 -45
  32. data/test/plugin/test_out_file.rb +0 -1
  33. data/test/plugin/test_out_forward.rb +19 -11
  34. data/test/plugin/test_output.rb +44 -0
  35. data/test/plugin/test_storage_local.rb +290 -2
  36. data/test/plugin_helper/test_child_process.rb +40 -39
  37. data/test/plugin_helper/test_storage.rb +4 -3
  38. data/test/test_log.rb +1 -1
  39. data/test/test_output.rb +3 -0
  40. data/test/test_plugin_id.rb +101 -0
  41. data/test/test_supervisor.rb +3 -0
  42. metadata +7 -2
@@ -3,21 +3,8 @@ require 'fluent/test/driver/input'
3
3
  require 'fluent/plugin/in_syslog'
4
4
 
5
5
  class SyslogInputTest < Test::Unit::TestCase
6
- class << self
7
- def startup
8
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
9
- @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
10
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
11
- end
12
-
13
- def shutdown
14
- @server.close
15
- end
16
- end
17
-
18
6
  def setup
19
7
  Fluent::Test.setup
20
- require 'fluent/plugin/socket_util'
21
8
  end
22
9
 
23
10
  PORT = unused_port
@@ -37,41 +24,45 @@ class SyslogInputTest < Test::Unit::TestCase
37
24
  Fluent::Test::Driver::Input.new(Fluent::Plugin::SyslogInput).configure(conf)
38
25
  end
39
26
 
40
- def test_configure
41
- configs = {'127.0.0.1' => CONFIG}
42
- configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
43
-
44
- configs.each_pair { |k, v|
45
- d = create_driver(v)
46
- assert_equal PORT, d.instance.port
47
- assert_equal k, d.instance.bind
48
- }
27
+ data(
28
+ ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
29
+ ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],
30
+ )
31
+ def test_configure(data)
32
+ bind_addr, config, family = data
33
+ omit "IPv6 unavailable" if family == ::Socket::AF_INET6 && !ipv6_enabled?
34
+
35
+ d = create_driver(config)
36
+ assert_equal PORT, d.instance.port
37
+ assert_equal bind_addr, d.instance.bind
49
38
  end
50
39
 
51
- def test_time_format
52
- configs = {'127.0.0.1' => CONFIG}
53
- configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
54
-
55
- configs.each_pair { |k, v|
56
- d = create_driver(v)
40
+ data(
41
+ ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
42
+ ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],
43
+ )
44
+ def test_time_format(data)
45
+ bind_addr, config, family = data
46
+ omit "IPv6 unavailable" if family == ::Socket::AF_INET6 && !ipv6_enabled?
57
47
 
58
- tests = [
59
- {'msg' => '<6>Dec 11 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec 11 00:00:00', '%b %d %H:%M:%S'))},
60
- {'msg' => '<6>Dec 1 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec 1 00:00:00', '%b %d %H:%M:%S'))},
61
- ]
62
- d.run(expect_emits: 2) do
63
- u = Fluent::SocketUtil.create_udp_socket(k)
64
- u.connect(k, PORT)
65
- tests.each {|test|
66
- u.send(test['msg'], 0)
67
- }
68
- end
48
+ d = create_driver(config)
69
49
 
70
- events = d.events
71
- assert(events.size > 0)
72
- events.each_index {|i|
73
- assert_equal_event_time(tests[i]['expected'], events[i][1])
50
+ tests = [
51
+ {'msg' => '<6>Dec 11 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec 11 00:00:00', '%b %d %H:%M:%S'))},
52
+ {'msg' => '<6>Dec 1 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec 1 00:00:00', '%b %d %H:%M:%S'))},
53
+ ]
54
+ d.run(expect_emits: 2) do
55
+ u = UDPSocket.new(family)
56
+ u.connect(bind_addr, PORT)
57
+ tests.each {|test|
58
+ u.send(test['msg'], 0)
74
59
  }
60
+ end
61
+
62
+ events = d.events
63
+ assert(events.size > 0)
64
+ events.each_index {|i|
65
+ assert_equal_event_time(tests[i]['expected'], events[i][1])
75
66
  }
76
67
  end
77
68
 
@@ -179,7 +170,7 @@ class SyslogInputTest < Test::Unit::TestCase
179
170
  compare_test_result(d.events, tests, {host: host})
180
171
  end
181
172
 
182
- def test_msg_size_with_include_priority
173
+ def test_msg_size_with_priority_key
183
174
  d = create_driver([CONFIG, 'priority_key priority'].join("\n"))
184
175
  tests = create_test_case
185
176
 
@@ -196,7 +187,7 @@ class SyslogInputTest < Test::Unit::TestCase
196
187
  compare_test_result(d.events, tests, {priority: priority})
197
188
  end
198
189
 
199
- def test_msg_size_with_include_facility
190
+ def test_msg_size_with_facility_key
200
191
  d = create_driver([CONFIG, 'facility_key facility'].join("\n"))
201
192
  tests = create_test_case
202
193
 
@@ -213,6 +204,43 @@ class SyslogInputTest < Test::Unit::TestCase
213
204
  compare_test_result(d.events, tests, {facility: facility})
214
205
  end
215
206
 
207
+ def test_msg_size_with_source_address_key
208
+ d = create_driver([CONFIG, 'source_address_key source_address'].join("\n"))
209
+ tests = create_test_case
210
+
211
+ address = nil
212
+ d.run(expect_emits: 2) do
213
+ u = UDPSocket.new
214
+ u.connect('127.0.0.1', PORT)
215
+ address = u.peeraddr[3]
216
+ tests.each {|test|
217
+ u.send(test['msg'], 0)
218
+ }
219
+ end
220
+
221
+ assert(d.events.size > 0)
222
+ compare_test_result(d.events, tests, {address: address})
223
+ end
224
+
225
+ def test_msg_size_with_source_hostname_key
226
+ d = create_driver([CONFIG, 'source_hostname_key source_hostname'].join("\n"))
227
+ tests = create_test_case
228
+
229
+ hostname = nil
230
+ d.run(expect_emits: 2) do
231
+ u = UDPSocket.new
232
+ u.do_not_reverse_lookup = false
233
+ u.connect('127.0.0.1', PORT)
234
+ hostname = u.peeraddr[2]
235
+ tests.each {|test|
236
+ u.send(test['msg'], 0)
237
+ }
238
+ end
239
+
240
+ assert(d.events.size > 0)
241
+ compare_test_result(d.events, tests, {hostname: hostname})
242
+ end
243
+
216
244
  def create_test_case(large_message: false)
217
245
  # actual syslog message has "\n"
218
246
  if large_message
@@ -234,6 +262,8 @@ class SyslogInputTest < Test::Unit::TestCase
234
262
  assert_equal('syslog.kern.info', events[i][0]) # <6> means kern.info
235
263
  assert_equal(tests[i]['expected'], events[i][2]['message'])
236
264
  assert_equal(options[:host], events[i][2]['source_host']) if options[:host]
265
+ assert_equal(options[:address], events[i][2]['source_address']) if options[:address]
266
+ assert_equal(options[:hostname], events[i][2]['source_hostname']) if options[:hostname]
237
267
  assert_equal(options[:priority], events[i][2]['priority']) if options[:priority]
238
268
  assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
239
269
  }
@@ -136,7 +136,6 @@ class FileOutputTest < Test::Unit::TestCase
136
136
 
137
137
  test 'configured as secondary with primary using chunk_key_tag and not using chunk_key_time' do
138
138
  require 'fluent/plugin/out_null'
139
- port = unused_port
140
139
  conf = config_element('match', '**', {
141
140
  }, [
142
141
  config_element('buffer', 'tag', {
@@ -35,16 +35,25 @@ class ForwardOutputTest < Test::Unit::TestCase
35
35
 
36
36
  def create_driver(conf=CONFIG)
37
37
  Fluent::Test::Driver::Output.new(Fluent::Plugin::ForwardOutput) {
38
- attr_reader :responses, :exceptions
38
+ attr_reader :response_chunk_ids, :exceptions, :sent_chunk_ids
39
39
 
40
40
  def initialize
41
41
  super
42
- @responses = []
42
+ @sent_chunk_ids = []
43
+ @response_chunk_ids = []
43
44
  @exceptions = []
44
45
  end
45
46
 
47
+ def try_write(chunk)
48
+ retval = super
49
+ @sent_chunk_ids << chunk.unique_id
50
+ retval
51
+ end
52
+
46
53
  def read_ack_from_sock(sock, unpacker)
47
- @responses << super
54
+ retval = super
55
+ @response_chunk_ids << retval
56
+ retval
48
57
  rescue => e
49
58
  @exceptions << e
50
59
  raise e
@@ -302,7 +311,7 @@ EOL
302
311
  assert_equal ['test', time, records[0]], events[0]
303
312
  assert_equal ['test', time, records[1]], events[1]
304
313
 
305
- assert_empty d.instance.responses # not attempt to receive responses, so it's empty
314
+ assert_empty d.instance.response_chunk_ids # not attempt to receive responses, so it's empty
306
315
  assert_empty d.instance.exceptions
307
316
  end
308
317
 
@@ -329,7 +338,7 @@ EOL
329
338
  assert_equal ['test', time, records[0]], events[0]
330
339
  assert_equal ['test', time, records[1]], events[1]
331
340
 
332
- assert_empty d.instance.responses # not attempt to receive responses, so it's empty
341
+ assert_empty d.instance.response_chunk_ids # not attempt to receive responses, so it's empty
333
342
  assert_empty d.instance.exceptions
334
343
  end
335
344
 
@@ -354,7 +363,7 @@ EOL
354
363
  {"a" => 2}
355
364
  ]
356
365
  target_input_driver.run(expect_records: 2) do
357
- d.end_if{ d.instance.responses.length > 0 }
366
+ d.end_if{ d.instance.response_chunk_ids.length > 0 }
358
367
  d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do
359
368
  d.feed([[time, records[0]], [time,records[1]]])
360
369
  end
@@ -364,7 +373,8 @@ EOL
364
373
  assert_equal ['test', time, records[0]], events[0]
365
374
  assert_equal ['test', time, records[1]], events[1]
366
375
 
367
- assert_equal 1, d.instance.responses.length
376
+ assert_equal 1, d.instance.response_chunk_ids.size
377
+ assert_equal d.instance.sent_chunk_ids.first, d.instance.response_chunk_ids.first
368
378
  assert_empty d.instance.exceptions
369
379
  end
370
380
 
@@ -400,7 +410,7 @@ EOL
400
410
  end
401
411
  end
402
412
 
403
- assert_equal 1, delayed_commit_timeout_value
413
+ assert_equal (1 + 2), delayed_commit_timeout_value
404
414
 
405
415
  events = target_input_driver.events
406
416
  assert_equal ['test', time, records[0]], events[0]
@@ -409,7 +419,6 @@ EOL
409
419
  assert{ d.instance.rollback_count > 0 }
410
420
 
411
421
  logs = d.instance.log.logs
412
- assert{ logs.any?{|log| log.include?("failed to flush the buffer chunk, timeout to commit.") } }
413
422
  assert{ logs.any?{|log| log.include?("no response from node. regard it as unavailable.") } }
414
423
  end
415
424
 
@@ -445,7 +454,7 @@ EOL
445
454
  end
446
455
  end
447
456
 
448
- assert_equal 5, delayed_commit_timeout_value
457
+ assert_equal (5 + 2), delayed_commit_timeout_value
449
458
 
450
459
  events = target_input_driver.events
451
460
  assert_equal ['test', time, records[0]], events[0]
@@ -454,7 +463,6 @@ EOL
454
463
  assert{ d.instance.rollback_count > 0 }
455
464
 
456
465
  logs = d.instance.log.logs
457
- assert{ logs.any?{|log| log.include?("failed to flush the buffer chunk, timeout to commit.") } }
458
466
  assert{ logs.any?{|log| log.include?("no response from node. regard it as unavailable.") } }
459
467
  end
460
468
 
@@ -799,4 +799,48 @@ class OutputTest < Test::Unit::TestCase
799
799
  assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc
800
800
  end
801
801
  end
802
+
803
+ sub_test_case 'slow_flush_log_threshold' do
804
+ def invoke_slow_flush_log_threshold_test(i)
805
+ i.configure(config_element('ROOT', '', {'slow_flush_log_threshold' => 0.5}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
806
+ i.start
807
+ i.after_start
808
+
809
+ t = event_time()
810
+ i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
811
+ i.force_flush
812
+
813
+ waiting(4) { Thread.pass until i.test_finished? }
814
+
815
+ yield
816
+ ensure
817
+ i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
818
+ end
819
+
820
+ test '#write flush took longer time than slow_flush_log_threshold' do
821
+ i = create_output(:buffered)
822
+ write_called = false
823
+ i.register(:write) { |chunk| sleep 1 }
824
+ i.define_singleton_method(:test_finished?) { write_called }
825
+ i.define_singleton_method(:try_flush) { super(); write_called = true }
826
+
827
+ invoke_slow_flush_log_threshold_test(i) {
828
+ assert write_called
829
+ assert_equal 1, i.log.out.logs.select { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold: elapsed_time/ }.size
830
+ }
831
+ end
832
+
833
+ test '#try_write flush took longer time than slow_flush_log_threshold' do
834
+ i = create_output(:delayed)
835
+ try_write_called = false
836
+ i.register(:try_write){ |chunk| sleep 1 }
837
+ i.define_singleton_method(:test_finished?) { try_write_called }
838
+ i.define_singleton_method(:try_flush) { super(); try_write_called = true }
839
+
840
+ invoke_slow_flush_log_threshold_test(i) {
841
+ assert try_write_called
842
+ assert_equal 1, i.log.out.logs.select { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold: elapsed_time/ }.size
843
+ }
844
+ end
845
+ end
802
846
  end
@@ -1,8 +1,296 @@
1
1
  require_relative '../helper'
2
2
  require 'fluent/plugin/storage_local'
3
+ require 'fluent/plugin/input'
4
+ require 'fluent/system_config'
5
+ require 'fileutils'
3
6
 
4
7
  class LocalStorageTest < Test::Unit::TestCase
5
- test 'syntax' do
6
- assert true
8
+ TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/storage_local#{ENV['TEST_ENV_NUMBER']}")
9
+
10
+ class MyInput < Fluent::Plugin::Input
11
+ helpers :storage
12
+ config_section :storage do
13
+ config_set_default :@type, 'local'
14
+ end
15
+ end
16
+
17
+ setup do
18
+ FileUtils.rm_rf(TMP_DIR)
19
+ FileUtils.mkdir_p(TMP_DIR)
20
+ Fluent::Test.setup
21
+ @d = MyInput.new
22
+ end
23
+
24
+ teardown do
25
+ @d.stop unless @d.stopped?
26
+ @d.before_shutdown unless @d.before_shutdown?
27
+ @d.shutdown unless @d.shutdown?
28
+ @d.after_shutdown unless @d.after_shutdown?
29
+ @d.close unless @d.closed?
30
+ @d.terminate unless @d.terminated?
31
+ end
32
+
33
+ sub_test_case 'without any configuration' do
34
+ test 'works as on-memory storage' do
35
+ conf = config_element()
36
+
37
+ @d.configure(conf)
38
+ @d.start
39
+ @p = @d.storage_create()
40
+
41
+ assert_nil @p.path
42
+ assert @p.store.empty?
43
+
44
+ assert_nil @p.get('key1')
45
+ assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')
46
+
47
+ @p.put('key1', '1')
48
+ assert_equal '1', @p.get('key1')
49
+
50
+ @p.update('key1') do |v|
51
+ (v.to_i * 2).to_s
52
+ end
53
+ assert_equal '2', @p.get('key1')
54
+
55
+ @p.save # on-memory storage does nothing...
56
+
57
+ @d.stop; @d.before_shutdown; @d.shutdown; @d.after_shutdown; @d.close; @d.terminate
58
+
59
+ # re-create to reload storage contents
60
+ @d = MyInput.new
61
+ @d.configure(conf)
62
+ @d.start
63
+ @p = @d.storage_create()
64
+
65
+ assert @p.store.empty?
66
+ end
67
+
68
+ test 'warns about on-memory storage if autosave is true' do
69
+ @d.configure(config_element())
70
+ @d.start
71
+ @p = @d.storage_create()
72
+
73
+ logs = @d.log.out.logs
74
+ assert{ logs.any?{|log| log.include?("[warn]: both of Plugin @id and path for <storage> are not specified. Using on-memory store.") } }
75
+ end
76
+
77
+ test 'show info log about on-memory storage if autosave is false' do
78
+ @d.configure(config_element('ROOT', '', {}, [config_element('storage', '', {'autosave' => 'false'})]))
79
+ @d.start
80
+ @p = @d.storage_create()
81
+
82
+ logs = @d.log.out.logs
83
+ assert{ logs.any?{|log| log.include?("[info]: both of Plugin @id and path for <storage> are not specified. Using on-memory store.") } }
84
+ end
85
+ end
86
+
87
+ sub_test_case 'configured with path' do
88
+ test 'works as storage which stores data on disk' do
89
+ storage_path = File.join(TMP_DIR, 'my_store.json')
90
+ conf = config_element('ROOT', '', {}, [config_element('storage', '', {'path' => storage_path})])
91
+ @d.configure(conf)
92
+ @d.start
93
+ @p = @d.storage_create()
94
+
95
+ assert_equal storage_path, @p.path
96
+ assert @p.store.empty?
97
+
98
+ assert_nil @p.get('key1')
99
+ assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')
100
+
101
+ @p.put('key1', '1')
102
+ assert_equal '1', @p.get('key1')
103
+
104
+ @p.update('key1') do |v|
105
+ (v.to_i * 2).to_s
106
+ end
107
+ assert_equal '2', @p.get('key1')
108
+
109
+ @p.save # stores all data into file
110
+
111
+ assert File.exist?(storage_path)
112
+
113
+ @p.put('key2', 4)
114
+
115
+ @d.stop; @d.before_shutdown; @d.shutdown; @d.after_shutdown; @d.close; @d.terminate
116
+
117
+ assert_equal({'key1' => '2', 'key2' => 4}, File.open(storage_path){|f| JSON.parse(f.read) })
118
+
119
+ # re-create to reload storage contents
120
+ @d = MyInput.new
121
+ @d.configure(conf)
122
+ @d.start
123
+ @p = @d.storage_create()
124
+
125
+ assert_false @p.store.empty?
126
+
127
+ assert_equal '2', @p.get('key1')
128
+ assert_equal 4, @p.get('key2')
129
+ end
130
+ end
131
+
132
+ sub_test_case 'configured with root-dir and plugin id' do
133
+ test 'works as storage which stores data under root dir' do
134
+ root_dir = File.join(TMP_DIR, 'root')
135
+ expected_storage_path = File.join(root_dir, 'worker0', 'local_storage_test', 'storage.json')
136
+ conf = config_element('ROOT', '', {'@id' => 'local_storage_test'})
137
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => root_dir) do
138
+ @d.configure(conf)
139
+ end
140
+ @d.start
141
+ @p = @d.storage_create()
142
+
143
+ assert_equal expected_storage_path, @p.path
144
+ assert @p.store.empty?
145
+
146
+ assert_nil @p.get('key1')
147
+ assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')
148
+
149
+ @p.put('key1', '1')
150
+ assert_equal '1', @p.get('key1')
151
+
152
+ @p.update('key1') do |v|
153
+ (v.to_i * 2).to_s
154
+ end
155
+ assert_equal '2', @p.get('key1')
156
+
157
+ @p.save # stores all data into file
158
+
159
+ assert File.exist?(expected_storage_path)
160
+
161
+ @p.put('key2', 4)
162
+
163
+ @d.stop; @d.before_shutdown; @d.shutdown; @d.after_shutdown; @d.close; @d.terminate
164
+
165
+ assert_equal({'key1' => '2', 'key2' => 4}, File.open(expected_storage_path){|f| JSON.parse(f.read) })
166
+
167
+ # re-create to reload storage contents
168
+ @d = MyInput.new
169
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => root_dir) do
170
+ @d.configure(conf)
171
+ end
172
+ @d.start
173
+ @p = @d.storage_create()
174
+
175
+ assert_false @p.store.empty?
176
+
177
+ assert_equal '2', @p.get('key1')
178
+ assert_equal 4, @p.get('key2')
179
+ end
180
+ end
181
+
182
+ sub_test_case 'persistent specified' do
183
+ test 'works well with path' do
184
+ omit "It's hard to test on Windows" if Fluent.windows?
185
+
186
+ storage_path = File.join(TMP_DIR, 'my_store.json')
187
+ conf = config_element('ROOT', '', {}, [config_element('storage', '', {'path' => storage_path, 'persistent' => 'true'})])
188
+ @d.configure(conf)
189
+ @d.start
190
+ @p = @d.storage_create()
191
+
192
+ assert_equal storage_path, @p.path
193
+ assert @p.store.empty?
194
+
195
+ assert_nil @p.get('key1')
196
+ assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')
197
+
198
+ @p.put('key1', '1')
199
+ assert_equal({'key1' => '1'}, File.open(storage_path){|f| JSON.parse(f.read) })
200
+
201
+ @p.update('key1') do |v|
202
+ (v.to_i * 2).to_s
203
+ end
204
+ assert_equal({'key1' => '2'}, File.open(storage_path){|f| JSON.parse(f.read) })
205
+ end
206
+
207
+ test 'works well with root-dir and plugin id' do
208
+ omit "It's hard to test on Windows" if Fluent.windows?
209
+
210
+ root_dir = File.join(TMP_DIR, 'root')
211
+ expected_storage_path = File.join(root_dir, 'worker0', 'local_storage_test', 'storage.json')
212
+ conf = config_element('ROOT', '', {'@id' => 'local_storage_test'}, [config_element('storage', '', {'persistent' => 'true'})])
213
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => root_dir) do
214
+ @d.configure(conf)
215
+ end
216
+ @d.start
217
+ @p = @d.storage_create()
218
+
219
+ assert_equal expected_storage_path, @p.path
220
+ assert @p.store.empty?
221
+
222
+ assert_nil @p.get('key1')
223
+ assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')
224
+
225
+ @p.put('key1', '1')
226
+ assert_equal({'key1' => '1'}, File.open(expected_storage_path){|f| JSON.parse(f.read) })
227
+
228
+ @p.update('key1') do |v|
229
+ (v.to_i * 2).to_s
230
+ end
231
+ assert_equal({'key1' => '2'}, File.open(expected_storage_path){|f| JSON.parse(f.read) })
232
+ end
233
+
234
+ test 'raises error if it is configured to use on-memory storage' do
235
+ assert_raise Fluent::ConfigError.new("Plugin @id or path for <storage> required when 'persistent' is true") do
236
+ @d.configure(config_element('ROOT', '', {}, [config_element('storage', '', {'persistent' => 'true'})]))
237
+ end
238
+ end
239
+ end
240
+
241
+ sub_test_case 'with various configurations' do
242
+ test 'mode and dir_mode controls permissions of stored data' do
243
+ omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
244
+
245
+ storage_path = File.join(TMP_DIR, 'dir', 'my_store.json')
246
+ storage_conf = {
247
+ 'path' => storage_path,
248
+ 'mode' => '0600',
249
+ 'dir_mode' => '0777',
250
+ }
251
+ conf = config_element('ROOT', '', {}, [config_element('storage', '', storage_conf)])
252
+ @d.configure(conf)
253
+ @d.start
254
+ @p = @d.storage_create()
255
+
256
+ assert_equal storage_path, @p.path
257
+ assert @p.store.empty?
258
+
259
+ @p.put('key1', '1')
260
+ assert_equal '1', @p.get('key1')
261
+
262
+ @p.save # stores all data into file
263
+
264
+ assert File.exist?(storage_path)
265
+ assert_equal 0600, (File.stat(storage_path).mode % 01000)
266
+ assert_equal 0777, (File.stat(File.dirname(storage_path)).mode % 01000)
267
+ end
268
+
269
+ test 'pretty_print controls to write data in files as human-easy-to-read' do
270
+ storage_path = File.join(TMP_DIR, 'dir', 'my_store.json')
271
+ storage_conf = {
272
+ 'path' => storage_path,
273
+ 'pretty_print' => 'true',
274
+ }
275
+ conf = config_element('ROOT', '', {}, [config_element('storage', '', storage_conf)])
276
+ @d.configure(conf)
277
+ @d.start
278
+ @p = @d.storage_create()
279
+
280
+ assert_equal storage_path, @p.path
281
+ assert @p.store.empty?
282
+
283
+ @p.put('key1', '1')
284
+ assert_equal '1', @p.get('key1')
285
+
286
+ @p.save # stores all data into file
287
+
288
+ expected_pp_json = <<JSON.chomp
289
+ {
290
+ "key1": "1"
291
+ }
292
+ JSON
293
+ assert_equal expected_pp_json, File.read(storage_path)
294
+ end
7
295
  end
8
296
  end