fluentd 0.14.7-x64-mingw32 → 0.14.10-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +2 -0
  4. data/CONTRIBUTING.md +6 -1
  5. data/ChangeLog +95 -0
  6. data/Rakefile +21 -0
  7. data/appveyor.yml +1 -0
  8. data/code-of-conduct.md +3 -0
  9. data/example/out_exec_filter.conf +42 -0
  10. data/fluentd.gemspec +1 -1
  11. data/lib/fluent/agent.rb +2 -2
  12. data/lib/fluent/command/binlog_reader.rb +1 -1
  13. data/lib/fluent/command/cat.rb +15 -4
  14. data/lib/fluent/compat/output.rb +14 -9
  15. data/lib/fluent/compat/parser.rb +141 -11
  16. data/lib/fluent/config/configure_proxy.rb +2 -11
  17. data/lib/fluent/config/section.rb +8 -1
  18. data/lib/fluent/configurable.rb +1 -3
  19. data/lib/fluent/env.rb +1 -1
  20. data/lib/fluent/log.rb +1 -1
  21. data/lib/fluent/plugin/base.rb +17 -0
  22. data/lib/fluent/plugin/filter_parser.rb +108 -0
  23. data/lib/fluent/plugin/filter_record_transformer.rb +14 -35
  24. data/lib/fluent/plugin/filter_stdout.rb +1 -1
  25. data/lib/fluent/plugin/formatter.rb +5 -0
  26. data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
  27. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  28. data/lib/fluent/plugin/formatter_tsv.rb +34 -0
  29. data/lib/fluent/plugin/in_exec.rb +48 -93
  30. data/lib/fluent/plugin/in_forward.rb +66 -265
  31. data/lib/fluent/plugin/in_http.rb +68 -65
  32. data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
  33. data/lib/fluent/plugin/in_syslog.rb +42 -58
  34. data/lib/fluent/plugin/in_tail.rb +29 -14
  35. data/lib/fluent/plugin/in_tcp.rb +54 -14
  36. data/lib/fluent/plugin/in_udp.rb +49 -13
  37. data/lib/fluent/plugin/multi_output.rb +1 -3
  38. data/lib/fluent/plugin/out_exec.rb +58 -71
  39. data/lib/fluent/plugin/out_exec_filter.rb +199 -279
  40. data/lib/fluent/plugin/out_file.rb +172 -81
  41. data/lib/fluent/plugin/out_forward.rb +229 -206
  42. data/lib/fluent/plugin/out_stdout.rb +6 -21
  43. data/lib/fluent/plugin/output.rb +90 -59
  44. data/lib/fluent/plugin/parser.rb +121 -61
  45. data/lib/fluent/plugin/parser_csv.rb +9 -3
  46. data/lib/fluent/plugin/parser_json.rb +37 -35
  47. data/lib/fluent/plugin/parser_ltsv.rb +11 -19
  48. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  49. data/lib/fluent/plugin/parser_regexp.rb +15 -42
  50. data/lib/fluent/plugin/parser_tsv.rb +8 -3
  51. data/lib/fluent/plugin_helper.rb +10 -1
  52. data/lib/fluent/plugin_helper/child_process.rb +139 -73
  53. data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
  54. data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
  55. data/lib/fluent/plugin_helper/event_loop.rb +24 -6
  56. data/lib/fluent/plugin_helper/extract.rb +16 -4
  57. data/lib/fluent/plugin_helper/formatter.rb +9 -11
  58. data/lib/fluent/plugin_helper/inject.rb +16 -1
  59. data/lib/fluent/plugin_helper/parser.rb +3 -3
  60. data/lib/fluent/plugin_helper/server.rb +494 -0
  61. data/lib/fluent/plugin_helper/socket.rb +101 -0
  62. data/lib/fluent/plugin_helper/socket_option.rb +84 -0
  63. data/lib/fluent/plugin_helper/timer.rb +1 -0
  64. data/lib/fluent/root_agent.rb +1 -1
  65. data/lib/fluent/test/driver/base.rb +95 -49
  66. data/lib/fluent/test/driver/base_owner.rb +18 -8
  67. data/lib/fluent/test/driver/multi_output.rb +2 -1
  68. data/lib/fluent/test/driver/output.rb +29 -6
  69. data/lib/fluent/test/helpers.rb +3 -1
  70. data/lib/fluent/test/log.rb +4 -0
  71. data/lib/fluent/test/startup_shutdown.rb +13 -0
  72. data/lib/fluent/time.rb +14 -8
  73. data/lib/fluent/version.rb +1 -1
  74. data/lib/fluent/winsvc.rb +1 -1
  75. data/test/command/test_binlog_reader.rb +5 -1
  76. data/test/compat/test_parser.rb +10 -0
  77. data/test/config/test_configurable.rb +193 -0
  78. data/test/config/test_configure_proxy.rb +0 -43
  79. data/test/helper.rb +36 -1
  80. data/test/plugin/test_base.rb +16 -0
  81. data/test/plugin/test_filter_parser.rb +665 -0
  82. data/test/plugin/test_filter_record_transformer.rb +36 -100
  83. data/test/plugin/test_filter_stdout.rb +18 -27
  84. data/test/plugin/test_in_dummy.rb +1 -1
  85. data/test/plugin/test_in_exec.rb +206 -94
  86. data/test/plugin/test_in_forward.rb +268 -347
  87. data/test/plugin/test_in_http.rb +310 -186
  88. data/test/plugin/test_in_monitor_agent.rb +65 -35
  89. data/test/plugin/test_in_syslog.rb +39 -3
  90. data/test/plugin/test_in_tcp.rb +78 -62
  91. data/test/plugin/test_in_udp.rb +101 -80
  92. data/test/plugin/test_out_exec.rb +223 -68
  93. data/test/plugin/test_out_exec_filter.rb +520 -169
  94. data/test/plugin/test_out_file.rb +637 -177
  95. data/test/plugin/test_out_forward.rb +242 -234
  96. data/test/plugin/test_out_null.rb +1 -1
  97. data/test/plugin/test_out_secondary_file.rb +4 -2
  98. data/test/plugin/test_out_stdout.rb +14 -35
  99. data/test/plugin/test_output_as_buffered.rb +60 -2
  100. data/test/plugin/test_parser.rb +359 -0
  101. data/test/plugin/test_parser_csv.rb +1 -2
  102. data/test/plugin/test_parser_json.rb +3 -4
  103. data/test/plugin/test_parser_labeled_tsv.rb +1 -2
  104. data/test/plugin/test_parser_none.rb +1 -2
  105. data/test/plugin/test_parser_regexp.rb +8 -4
  106. data/test/plugin/test_parser_tsv.rb +4 -3
  107. data/test/plugin_helper/test_child_process.rb +184 -0
  108. data/test/plugin_helper/test_compat_parameters.rb +88 -1
  109. data/test/plugin_helper/test_extract.rb +0 -1
  110. data/test/plugin_helper/test_formatter.rb +5 -2
  111. data/test/plugin_helper/test_inject.rb +21 -0
  112. data/test/plugin_helper/test_parser.rb +6 -5
  113. data/test/plugin_helper/test_server.rb +905 -0
  114. data/test/test_event_time.rb +3 -1
  115. data/test/test_output.rb +53 -2
  116. data/test/test_plugin_classes.rb +20 -0
  117. data/test/test_root_agent.rb +139 -0
  118. data/test/test_test_drivers.rb +135 -0
  119. metadata +28 -8
  120. data/test/plugin/test_parser_base.rb +0 -32
@@ -18,123 +18,211 @@ class ExecOutputTest < Test::Unit::TestCase
18
18
 
19
19
  TMP_DIR = File.dirname(__FILE__) + "/../tmp/out_exec#{ENV['TEST_ENV_NUMBER']}"
20
20
 
21
- CONFIG = %[
21
+ def create_driver(config)
22
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecOutput).configure(config)
23
+ end
24
+
25
+ def create_test_data
26
+ time = event_time("2011-01-02 13:14:15.123")
27
+ records = [{"k1"=>"v1","kx"=>"vx"}, {"k1"=>"v2","kx"=>"vx"}]
28
+ return time, records
29
+ end
30
+
31
+ DEFAULT_CONFIG_ONLY_WITH_KEYS = %[
32
+ command cat >#{TMP_DIR}/out
33
+ <format>
34
+ keys ["k1", "kx"]
35
+ </format>
36
+ ]
37
+
38
+ test 'configure in default' do
39
+ d = create_driver DEFAULT_CONFIG_ONLY_WITH_KEYS
40
+ assert{ d.instance.formatter.is_a? Fluent::Plugin::TSVFormatter }
41
+ assert_equal ["k1", "kx"], d.instance.formatter.keys
42
+ assert_nil d.instance.inject_config
43
+ end
44
+
45
+ TSV_CONFIG = %[
46
+ command cat >#{TMP_DIR}/out
47
+ <inject>
48
+ tag_key tag
49
+ time_key time
50
+ time_format %Y-%m-%d %H:%M:%S
51
+ localtime yes
52
+ </inject>
53
+ <format>
54
+ @type tsv
55
+ keys time, tag, k1
56
+ </format>
57
+ ]
58
+ TSV_CONFIG_WITH_SUBSEC = %[
59
+ command cat >#{TMP_DIR}/out
60
+ <inject>
61
+ tag_key tag
62
+ time_key time
63
+ time_format %Y-%m-%d %H:%M:%S.%3N
64
+ localtime yes
65
+ </inject>
66
+ <format>
67
+ @type tsv
68
+ keys time, tag, k1
69
+ </format>
70
+ ]
71
+ TSV_CONFIG_WITH_BUFFER = TSV_CONFIG + %[
72
+ <buffer time>
73
+ @type memory
74
+ timekey 3600
75
+ flush_thread_count 5
76
+ chunk_limit_size 50m
77
+ total_limit_size #{50 * 1024 * 1024 * 128}
78
+ flush_at_shutdown yes
79
+ </buffer>
80
+ ]
81
+ JSON_CONFIG = %[
82
+ command cat >#{TMP_DIR}/out
83
+ <format>
84
+ @type json
85
+ </format>
86
+ ]
87
+ MSGPACK_CONFIG = %[
88
+ command cat >#{TMP_DIR}/out
89
+ <format>
90
+ @type msgpack
91
+ </format>
92
+ ]
93
+
94
+ CONFIG_COMPAT = %[
22
95
  buffer_path #{TMP_DIR}/buffer
23
96
  command cat >#{TMP_DIR}/out
24
97
  localtime
25
98
  ]
26
- TSV_CONFIG = %[
99
+ TSV_CONFIG_COMPAT = %[
27
100
  keys "time,tag,k1"
28
101
  tag_key "tag"
29
102
  time_key "time"
30
103
  time_format %Y-%m-%d %H:%M:%S
31
104
  ]
105
+ BUFFER_CONFIG_COMPAT = %[
106
+ buffer_type memory
107
+ time_slice_format %Y%m%d%H
108
+ num_threads 5
109
+ buffer_chunk_limit 50m
110
+ buffer_queue_limit 128
111
+ flush_at_shutdown yes
112
+ ]
113
+ TSV_CONFIG_WITH_SUBSEC_COMPAT = %[
114
+ keys "time,tag,k1"
115
+ tag_key "tag"
116
+ time_key "time"
117
+ time_format %Y-%m-%d %H:%M:%S.%3N
118
+ ]
32
119
 
33
- def create_driver(conf = TSV_CONFIG)
34
- config = CONFIG + conf
35
- Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecOutput).configure(config)
36
- end
37
-
38
- def create_test_case
39
- time = Time.parse("2011-01-02 13:14:15").to_i
40
- tests = [{"k1"=>"v1","kx"=>"vx"}, {"k1"=>"v2","kx"=>"vx"}]
41
- return time, tests
42
- end
43
-
44
- def test_configure
45
- d = create_driver
120
+ data(
121
+ 'with sections' => TSV_CONFIG,
122
+ 'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT,
123
+ )
124
+ test 'configure for tsv' do |conf|
125
+ d = create_driver(conf)
46
126
 
47
- assert_equal ["time","tag","k1"], d.instance.keys
48
- assert_equal "tag", d.instance.tag_key
49
- assert_equal "time", d.instance.time_key
50
- assert_equal "%Y-%m-%d %H:%M:%S", d.instance.time_format
51
- assert_equal true, d.instance.localtime
127
+ assert_equal ["time","tag","k1"], d.instance.formatter.keys
128
+ assert_equal "tag", d.instance.inject_config.tag_key
129
+ assert_equal "time", d.instance.inject_config.time_key
130
+ assert_equal "%Y-%m-%d %H:%M:%S", d.instance.inject_config.time_format
131
+ assert_equal true, d.instance.inject_config.localtime
52
132
  end
53
133
 
54
- def test_configure_with_compat_buffer_parameters
55
- conf = TSV_CONFIG + %[
56
- buffer_type memory
57
- time_slice_format %Y%m%d%H
58
- num_threads 5
59
- buffer_chunk_limit 50m
60
- buffer_queue_limit 128
61
- flush_at_shutdown yes
62
- ]
134
+ data(
135
+ 'with sections' => TSV_CONFIG_WITH_BUFFER,
136
+ 'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT + BUFFER_CONFIG_COMPAT,
137
+ )
138
+ test 'configure_with_compat_buffer_parameters' do |conf|
63
139
  d = create_driver(conf)
64
140
  assert_equal 3600, d.instance.buffer_config.timekey
65
141
  assert_equal 5, d.instance.buffer_config.flush_thread_count
66
142
  assert_equal 50*1024*1024, d.instance.buffer.chunk_limit_size
67
- assert_equal 128, d.instance.buffer.queue_length_limit
143
+ assert_equal 50*1024*1024*128, d.instance.buffer.total_limit_size
68
144
  assert d.instance.buffer_config.flush_at_shutdown
69
145
  end
70
146
 
71
- def test_format
72
- d = create_driver
73
- time, tests = create_test_case
147
+ data(
148
+ 'with sections' => TSV_CONFIG,
149
+ 'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT,
150
+ )
151
+ test 'format' do |conf|
152
+ d = create_driver(conf)
153
+ time, records = create_test_data
74
154
 
75
155
  d.run(default_tag: 'test') do
76
- d.feed(time, tests[0])
77
- d.feed(time, tests[1])
156
+ d.feed(time, records[0])
157
+ d.feed(time, records[1])
78
158
  end
79
159
 
80
160
  assert_equal %[2011-01-02 13:14:15\ttest\tv1\n], d.formatted[0]
81
161
  assert_equal %[2011-01-02 13:14:15\ttest\tv2\n], d.formatted[1]
82
162
  end
83
163
 
84
- def test_format_json
85
- d = create_driver("format json")
86
- time, tests = create_test_case
164
+ data(
165
+ 'with sections' => JSON_CONFIG,
166
+ 'traditional' => CONFIG_COMPAT + "format json",
167
+ )
168
+ test 'format_json' do |conf|
169
+ d = create_driver(conf)
170
+ time, records = create_test_data
87
171
 
88
172
  d.run(default_tag: 'test') do
89
- d.feed(time, tests[0])
90
- d.feed(time, tests[1])
173
+ d.feed(time, records[0])
174
+ d.feed(time, records[1])
91
175
  end
92
176
 
93
- assert_equal Yajl.dump(tests[0]) + "\n", d.formatted[0]
94
- assert_equal Yajl.dump(tests[1]) + "\n", d.formatted[1]
177
+ assert_equal Yajl.dump(records[0]) + "\n", d.formatted[0]
178
+ assert_equal Yajl.dump(records[1]) + "\n", d.formatted[1]
95
179
  end
96
180
 
97
- def test_format_msgpack
98
- d = create_driver("format msgpack")
99
- time, tests = create_test_case
181
+ data(
182
+ 'with sections' => MSGPACK_CONFIG,
183
+ 'traditional' => CONFIG_COMPAT + "format msgpack"
184
+ )
185
+ test 'format_msgpack' do |conf|
186
+ d = create_driver(conf)
187
+ time, records = create_test_data
100
188
 
101
189
  d.run(default_tag: 'test') do
102
- d.feed(time, tests[0])
103
- d.feed(time, tests[1])
190
+ d.feed(time, records[0])
191
+ d.feed(time, records[1])
104
192
  end
105
193
 
106
- assert_equal tests[0].to_msgpack, d.formatted[0]
107
- assert_equal tests[1].to_msgpack, d.formatted[1]
194
+ assert_equal records[0].to_msgpack, d.formatted[0]
195
+ assert_equal records[1].to_msgpack, d.formatted[1]
108
196
  end
109
197
 
110
- def test_format_time
111
- config = %[
112
- keys "time,tag,k1"
113
- tag_key "tag"
114
- time_key "time"
115
- time_format %Y-%m-%d %H:%M:%S.%3N
116
- ]
117
- d = create_driver(config)
118
-
119
- time = event_time("2011-01-02 13:14:15.123")
120
- tests = [{"k1"=>"v1","kx"=>"vx"}, {"k1"=>"v2","kx"=>"vx"}]
198
+ data(
199
+ 'with sections' => TSV_CONFIG_WITH_SUBSEC,
200
+ 'traditional' => CONFIG_COMPAT + TSV_CONFIG_WITH_SUBSEC_COMPAT,
201
+ )
202
+ test 'format subsecond time' do |conf|
203
+ d = create_driver(conf)
204
+ time, records = create_test_data
121
205
 
122
206
  d.run(default_tag: 'test') do
123
- d.feed(time, tests[0])
124
- d.feed(time, tests[1])
207
+ d.feed(time, records[0])
208
+ d.feed(time, records[1])
125
209
  end
126
210
 
127
211
  assert_equal %[2011-01-02 13:14:15.123\ttest\tv1\n], d.formatted[0]
128
212
  assert_equal %[2011-01-02 13:14:15.123\ttest\tv2\n], d.formatted[1]
129
213
  end
130
214
 
131
- def test_write
132
- d = create_driver
133
- time, tests = create_test_case
215
+ data(
216
+ 'with sections' => TSV_CONFIG,
217
+ 'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT,
218
+ )
219
+ test 'write' do |conf|
220
+ d = create_driver(conf)
221
+ time, records = create_test_data
134
222
 
135
223
  d.run(default_tag: 'test', flush: true) do
136
- d.feed(time, tests[0])
137
- d.feed(time, tests[1])
224
+ d.feed(time, records[0])
225
+ d.feed(time, records[1])
138
226
  end
139
227
 
140
228
  expect_path = "#{TMP_DIR}/out"
@@ -151,5 +239,72 @@ class ExecOutputTest < Test::Unit::TestCase
151
239
  %[2011-01-02 13:14:15\ttest\tv2\n]
152
240
  assert_equal expect_data, data
153
241
  end
154
- end
155
242
 
243
+ sub_test_case 'when executed process dies unexpectedly' do
244
+ setup do
245
+ @gen_config = ->(num){ <<EOC
246
+ command ruby -e "ARGV.first.to_i == 0 ? open(ARGV[1]){|f| STDOUT.write f.read} : (sleep 1 ; exit ARGV.first.to_i)" #{num} >#{TMP_DIR}/fail_out
247
+ <inject>
248
+ tag_key tag
249
+ time_key time
250
+ time_format %Y-%m-%d %H:%M:%S
251
+ localtime yes
252
+ </inject>
253
+ <format>
254
+ @type tsv
255
+ keys time, tag, k1
256
+ </format>
257
+ EOC
258
+ }
259
+ end
260
+
261
+ test 'flushed chunk will be committed after child process successfully exits' do
262
+ d = create_driver(@gen_config.call(0))
263
+ time, records = create_test_data
264
+
265
+ expect_path = "#{TMP_DIR}/fail_out"
266
+
267
+ d.end_if{ File.exist?(expect_path) }
268
+ d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do
269
+ d.feed(time, records[0])
270
+ d.feed(time, records[1])
271
+ end
272
+
273
+ assert{ File.exist?(expect_path) }
274
+
275
+ data = File.read(expect_path)
276
+ expect_data =
277
+ %[2011-01-02 13:14:15\ttest\tv1\n] +
278
+ %[2011-01-02 13:14:15\ttest\tv2\n]
279
+ assert_equal expect_data, data
280
+
281
+ assert{ d.instance.buffer.queue.empty? }
282
+ assert{ d.instance.dequeued_chunks.empty? }
283
+
284
+ d.instance_shutdown
285
+ end
286
+
287
+ test 'flushed chunk will be taken back after child process unexpectedly exits' do
288
+ d = create_driver(@gen_config.call(3))
289
+ time, records = create_test_data
290
+
291
+ expect_path = "#{TMP_DIR}/fail_out"
292
+
293
+ d.end_if{ d.instance.log.out.logs.any?{|line| line.include?("command exits with error code") } }
294
+ d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do
295
+ d.feed(time, records[0])
296
+ d.feed(time, records[1])
297
+ end
298
+
299
+ assert{ d.instance.dequeued_chunks.empty? } # because it's already taken back
300
+ assert{ d.instance.buffer.queue.size == 1 }
301
+
302
+ logs = d.instance.log.out.logs
303
+ assert{ logs.any?{|line| line.include?("command exits with error code") && line.include?("status=3") } }
304
+
305
+ assert{ File.exist?(expect_path) && File.size(expect_path) == 0 }
306
+
307
+ d.instance_shutdown
308
+ end
309
+ end
310
+ end
@@ -1,5 +1,5 @@
1
1
  require_relative '../helper'
2
- require 'fluent/test'
2
+ require 'fluent/test/driver/output'
3
3
  require 'fluent/plugin/out_exec_filter'
4
4
  require 'fileutils'
5
5
 
@@ -9,6 +9,29 @@ class ExecFilterOutputTest < Test::Unit::TestCase
9
9
  end
10
10
 
11
11
  CONFIG = %[
12
+ command cat
13
+ num_children 3
14
+ <inject>
15
+ tag_key tag
16
+ time_key time_in
17
+ time_type string
18
+ time_format %Y-%m-%d %H:%M:%S
19
+ </inject>
20
+ <format>
21
+ keys ["time_in", "tag", "k1"]
22
+ </format>
23
+ <parse>
24
+ keys ["time_out", "tag", "k2"]
25
+ </parse>
26
+ <extract>
27
+ tag_key tag
28
+ time_key time_out
29
+ time_type string
30
+ time_format %Y-%m-%d %H:%M:%S
31
+ </extract>
32
+ ]
33
+
34
+ CONFIG_COMPAT = %[
12
35
  command cat
13
36
  in_keys time_in,tag,k1
14
37
  out_keys time_out,tag,k2
@@ -20,35 +43,34 @@ class ExecFilterOutputTest < Test::Unit::TestCase
20
43
  num_children 3
21
44
  ]
22
45
 
23
- def create_driver(conf = CONFIG, tag = 'test')
24
- Fluent::Test::BufferedOutputTestDriver.new(Fluent::ExecFilterOutput, tag).configure(conf)
25
- end
26
-
27
- def sed_unbuffered_support?
28
- @sed_unbuffered_support ||= lambda {
29
- system("echo xxx | sed --unbuffered -l -e 's/x/y/g' >#{IO::NULL} 2>&1")
30
- $?.success?
31
- }.call
32
- end
33
-
34
- def sed_unbuffered_option
35
- sed_unbuffered_support? ? '--unbuffered' : ''
46
+ def create_driver(conf)
47
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecFilterOutput).configure(conf)
36
48
  end
37
49
 
38
- def test_configure
39
- d = create_driver
40
-
41
- assert d.instance.instance_eval{ @overrides_format_stream }
42
-
43
- assert_equal ["time_in","tag","k1"], d.instance.in_keys
44
- assert_equal ["time_out","tag","k2"], d.instance.out_keys
45
- assert_equal "tag", d.instance.out_tag_key
46
- assert_equal "tag", d.instance.in_tag_key
47
- assert_equal "time_in", d.instance.in_time_key
48
- assert_equal "time_out", d.instance.out_time_key
49
- assert_equal "%Y-%m-%d %H:%M:%S", d.instance.in_time_format
50
- assert_equal "%Y-%m-%d %H:%M:%S", d.instance.out_time_format
51
- assert_equal true, d.instance.localtime
50
+ SED_SUPPORT_UNBUFFERED_OPTION = ->(){
51
+ system("echo xxx | sed --unbuffered -l -e 's/x/y/g' >#{IO::NULL} 2>&1")
52
+ $?.success?
53
+ }.call
54
+ SED_UNBUFFERED_OPTION = SED_SUPPORT_UNBUFFERED_OPTION ? '--unbuffered' : ''
55
+
56
+ data(
57
+ 'with sections' => CONFIG,
58
+ 'traditional' => CONFIG_COMPAT,
59
+ )
60
+ test 'configure' do |conf|
61
+ d = create_driver(conf)
62
+
63
+ assert_false d.instance.parser.estimate_current_event
64
+
65
+ assert_equal ["time_in","tag","k1"], d.instance.formatter.keys
66
+ assert_equal ["time_out","tag","k2"], d.instance.parser.keys
67
+ assert_equal "tag", d.instance.inject_config.tag_key
68
+ assert_equal "tag", d.instance.extract_config.tag_key
69
+ assert_equal "time_in", d.instance.inject_config.time_key
70
+ assert_equal "time_out", d.instance.extract_config.time_key
71
+ assert_equal "%Y-%m-%d %H:%M:%S", d.instance.inject_config.time_format
72
+ assert_equal "%Y-%m-%d %H:%M:%S", d.instance.extract_config.time_format
73
+ assert_equal true, d.instance.inject_config.localtime
52
74
  assert_equal 3, d.instance.num_children
53
75
 
54
76
  d = create_driver %[
@@ -61,7 +83,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
61
83
  ]
62
84
  assert_equal "sed -l -e s/foo/bar/", d.instance.command
63
85
 
64
- d = create_driver(CONFIG + %[
86
+ d = create_driver(conf + %[
65
87
  remove_prefix before
66
88
  add_prefix after
67
89
  ])
@@ -69,194 +91,523 @@ class ExecFilterOutputTest < Test::Unit::TestCase
69
91
  assert_equal "after" , d.instance.add_prefix
70
92
  end
71
93
 
72
- def test_emit_1
73
- d = create_driver
74
-
75
- time = Fluent::EventTime.parse("2011-01-02 13:14:15")
76
-
77
- d.expected_emits_length = 2
78
- d.run do
79
- d.emit({"k1"=>1}, time)
80
- d.emit({"k1"=>2}, time)
94
+ data(
95
+ 'with sections' => CONFIG,
96
+ 'traditional' => CONFIG_COMPAT,
97
+ )
98
+ test 'emit events with TSV format' do |conf|
99
+ d = create_driver(conf)
100
+ time = event_time("2011-01-02 13:14:15")
101
+
102
+ d.run(default_tag: 'test', expect_emits: 2, timeout: 10) do
103
+ # sleep 0.1 until d.instance.children && !d.instance.children.empty? && d.instance.children.all?{|c| c.finished == false }
104
+ d.feed(time, {"k1"=>1})
105
+ d.feed(time, {"k1"=>2})
81
106
  end
82
107
 
83
- emits = d.emits
84
- assert_equal 2, emits.length
85
- assert_equal ["test", time, {"k2"=>"1"}], emits[0]
86
- assert_equal_event_time time, emits[0][1]
87
- assert_equal ["test", time, {"k2"=>"2"}], emits[1]
88
- assert_equal_event_time time, emits[1][1]
108
+ assert_equal "2011-01-02 13:14:15\ttest\t1\n", d.formatted[0]
109
+ assert_equal "2011-01-02 13:14:15\ttest\t2\n", d.formatted[1]
110
+
111
+ events = d.events
112
+ assert_equal 2, events.length
113
+ assert_equal_event_time time, events[0][1]
114
+ assert_equal ["test", time, {"k2"=>"1"}], events[0]
115
+ assert_equal_event_time time, events[1][1]
116
+ assert_equal ["test", time, {"k2"=>"2"}], events[1]
89
117
  end
90
118
 
91
- def test_emit_2
92
- d = create_driver %[
93
- command cat
94
- in_keys time,k1
95
- out_keys time,k2
96
- tag xxx
119
+ CONFIG_WITHOUT_TIME_FORMAT = %[
120
+ command cat
121
+ num_children 3
122
+ tag xxx
123
+ <inject>
97
124
  time_key time
98
- num_children 3
99
- ]
100
-
101
- time = Fluent::EventTime.parse("2011-01-02 13:14:15")
125
+ time_type unixtime
126
+ </inject>
127
+ <format>
128
+ keys time,k1
129
+ </format>
130
+ <parse>
131
+ keys time,k2
132
+ time_key time
133
+ time_type unixtime
134
+ </parse>
135
+ ]
136
+ CONFIG_WITHOUT_TIME_FORMAT_COMPAT = %[
137
+ command cat
138
+ in_keys time,k1
139
+ out_keys time,k2
140
+ tag xxx
141
+ time_key time
142
+ num_children 3
143
+ ]
102
144
 
103
- d.expected_emits_length = 2
104
- d.run do
105
- d.emit({"k1"=>1}, time)
106
- d.emit({"k1"=>2}, time)
145
+ data(
146
+ 'with sections' => CONFIG_WITHOUT_TIME_FORMAT,
147
+ 'traditional' => CONFIG_WITHOUT_TIME_FORMAT_COMPAT,
148
+ )
149
+ test 'emit events without time format configuration' do |conf|
150
+ d = create_driver(conf)
151
+ time = event_time("2011-01-02 13:14:15 +0900")
152
+
153
+ d.run(default_tag: 'test', expect_emits: 2, timeout: 10) do
154
+ d.feed(time, {"k1"=>1})
155
+ d.feed(time, {"k1"=>2})
107
156
  end
108
157
 
109
- emits = d.emits
110
- assert_equal 2, emits.length
111
- assert_equal ["xxx", time, {"k2"=>"1"}], emits[0]
112
- assert_equal_event_time time, emits[0][1]
113
- assert_equal ["xxx", time, {"k2"=>"2"}], emits[1]
114
- assert_equal_event_time time, emits[1][1]
158
+ assert_equal "1293941655\t1\n", d.formatted[0]
159
+ assert_equal "1293941655\t2\n", d.formatted[1]
160
+
161
+ events = d.events
162
+ assert_equal 2, events.length
163
+ assert_equal_event_time time, events[0][1]
164
+ assert_equal ["xxx", time, {"k2"=>"1"}], events[0]
165
+ assert_equal_event_time time, events[1][1]
166
+ assert_equal ["xxx", time, {"k2"=>"2"}], events[1]
115
167
  end
116
168
 
117
- def test_emit_3
118
- d = create_driver %[
119
- command grep --line-buffered -v poo
120
- in_keys time,val1
121
- out_keys time,val2
122
- tag xxx
169
+ CONFIG_TO_DO_GREP = %[
170
+ command grep --line-buffered -v poo
171
+ num_children 3
172
+ tag xxx
173
+ <inject>
123
174
  time_key time
124
- num_children 3
125
- ]
126
-
127
- time = Fluent::EventTime.parse("2011-01-02 13:14:15")
175
+ time_type unixtime
176
+ </inject>
177
+ <format>
178
+ keys time, val1
179
+ </format>
180
+ <parse>
181
+ keys time, val2
182
+ time_key time
183
+ time_type unixtime
184
+ </parse>
185
+ ]
186
+ CONFIG_TO_DO_GREP_COMPAT = %[
187
+ command grep --line-buffered -v poo
188
+ in_keys time,val1
189
+ out_keys time,val2
190
+ tag xxx
191
+ time_key time
192
+ num_children 3
193
+ ]
128
194
 
129
- d.expected_emits_length = 2
130
- d.run do
131
- d.emit({"val1"=>"sed-ed value foo"}, time)
132
- d.emit({"val1"=>"sed-ed value poo"}, time)
195
+ data(
196
+ 'with sections' => CONFIG_TO_DO_GREP,
197
+ 'traditional' => CONFIG_TO_DO_GREP_COMPAT,
198
+ )
199
+ test 'emit events through grep command' do |conf|
200
+ d = create_driver(conf)
201
+ time = event_time("2011-01-02 13:14:15 +0900")
202
+
203
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10) do
204
+ d.feed(time, {"val1"=>"sed-ed value poo"})
205
+ d.feed(time, {"val1"=>"sed-ed value foo"})
133
206
  end
134
207
 
135
- emits = d.emits
136
- assert_equal 1, emits.length
137
- assert_equal ["xxx", time, {"val2"=>"sed-ed value foo"}], emits[0]
138
- assert_equal_event_time time, emits[0][1]
208
+ assert_equal "1293941655\tsed-ed value poo\n", d.formatted[0]
209
+ assert_equal "1293941655\tsed-ed value foo\n", d.formatted[1]
139
210
 
140
- d = create_driver %[
141
- command sed #{sed_unbuffered_option} -l -e s/foo/bar/
142
- in_keys time,val1
143
- out_keys time,val2
144
- tag xxx
145
- time_key time
146
- num_children 3
147
- ]
211
+ events = d.events
212
+ assert_equal 1, events.length
213
+ assert_equal_event_time time, events[0][1]
214
+ assert_equal ["xxx", time, {"val2"=>"sed-ed value foo"}], events[0]
215
+ end
148
216
 
149
- time = Fluent::EventTime.parse("2011-01-02 13:14:15")
217
+ CONFIG_TO_DO_SED = %[
218
+ command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
219
+ num_children 3
220
+ tag xxx
221
+ <inject>
222
+ time_key time
223
+ time_type unixtime
224
+ </inject>
225
+ <format>
226
+ keys time, val1
227
+ </format>
228
+ <parse>
229
+ keys time, val2
230
+ time_key time
231
+ time_type unixtime
232
+ </parse>
233
+ ]
234
+ CONFIG_TO_DO_SED_COMPAT = %[
235
+ command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
236
+ in_keys time,val1
237
+ out_keys time,val2
238
+ tag xxx
239
+ time_key time
240
+ num_children 3
241
+ ]
150
242
 
151
- d.expected_emits_length = 2
152
- d.run do
153
- d.emit({"val1"=>"sed-ed value foo"}, time)
154
- d.emit({"val1"=>"sed-ed value poo"}, time)
243
+ data(
244
+ 'with sections' => CONFIG_TO_DO_SED,
245
+ 'traditional' => CONFIG_TO_DO_SED_COMPAT,
246
+ )
247
+ test 'emit events through sed command' do |conf|
248
+ d = create_driver(conf)
249
+ time = event_time("2011-01-02 13:14:15 +0900")
250
+
251
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10) do
252
+ d.feed(time, {"val1"=>"sed-ed value poo"})
253
+ d.feed(time, {"val1"=>"sed-ed value foo"})
155
254
  end
156
255
 
157
- emits = d.emits
158
- assert_equal 2, emits.length
159
- assert_equal ["xxx", time, {"val2"=>"sed-ed value bar"}], emits[0]
160
- assert_equal_event_time time, emits[0][1]
161
- assert_equal ["xxx", time, {"val2"=>"sed-ed value poo"}], emits[1]
162
- assert_equal_event_time time, emits[1][1]
256
+ assert_equal "1293941655\tsed-ed value poo\n", d.formatted[0]
257
+ assert_equal "1293941655\tsed-ed value foo\n", d.formatted[1]
258
+
259
+ events = d.events
260
+ assert_equal 2, events.length
261
+ assert_equal_event_time time, events[0][1]
262
+ assert_equal ["xxx", time, {"val2"=>"sed-ed value poo"}], events[0]
263
+ assert_equal_event_time time, events[1][1]
264
+ assert_equal ["xxx", time, {"val2"=>"sed-ed value bar"}], events[1]
163
265
  end
164
266
 
165
- def test_emit_4
166
- d = create_driver(%[
167
- command sed #{sed_unbuffered_option} -l -e s/foo/bar/
168
- in_keys tag,time,val1
169
- remove_prefix input
170
- out_keys tag,time,val2
171
- add_prefix output
267
+ CONFIG_TO_DO_SED_WITH_TAG_MODIFY = %[
268
+ command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
269
+ num_children 3
270
+ remove_prefix input
271
+ add_prefix output
272
+ <inject>
172
273
  tag_key tag
173
274
  time_key time
174
- num_children 3
175
- ], 'input.test')
275
+ </inject>
276
+ <format>
277
+ keys tag, time, val1
278
+ </format>
279
+ <parse>
280
+ keys tag, time, val2
281
+ </parse>
282
+ <extract>
283
+ tag_key tag
284
+ time_key time
285
+ </extract>
286
+ ]
287
+ CONFIG_TO_DO_SED_WITH_TAG_MODIFY_COMPAT = %[
288
+ command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
289
+ in_keys tag,time,val1
290
+ remove_prefix input
291
+ out_keys tag,time,val2
292
+ add_prefix output
293
+ tag_key tag
294
+ time_key time
295
+ num_children 3
296
+ ]
297
+
298
+ data(
299
+ 'with sections' => CONFIG_TO_DO_SED_WITH_TAG_MODIFY,
300
+ 'traditional' => CONFIG_TO_DO_SED_WITH_TAG_MODIFY_COMPAT,
301
+ )
302
+ test 'emit events with add/remove tag prefix' do |conf|
303
+ d = create_driver(conf)
176
304
 
177
- time = Fluent::EventTime.parse("2011-01-02 13:14:15")
305
+ time = event_time("2011-01-02 13:14:15 +0900")
178
306
 
179
- d.expected_emits_length = 2
180
- d.run do
181
- d.emit({"val1"=>"sed-ed value foo"}, time)
182
- d.emit({"val1"=>"sed-ed value poo"}, time)
307
+ d.run(default_tag: 'input.test', expect_emits: 2, timeout: 10) do
308
+ d.feed(time, {"val1"=>"sed-ed value foo"})
309
+ d.feed(time, {"val1"=>"sed-ed value poo"})
183
310
  end
184
311
 
185
- emits = d.emits
186
- assert_equal 2, emits.length
187
- assert_equal ["output.test", time, {"val2"=>"sed-ed value bar"}], emits[0]
188
- assert_equal_event_time time, emits[0][1]
189
- assert_equal ["output.test", time, {"val2"=>"sed-ed value poo"}], emits[1]
190
- assert_equal_event_time time, emits[1][1]
312
+ assert_equal "test\t1293941655\tsed-ed value foo\n", d.formatted[0]
313
+ assert_equal "test\t1293941655\tsed-ed value poo\n", d.formatted[1]
314
+
315
+ events = d.events
316
+ assert_equal 2, events.length
317
+ assert_equal_event_time time, events[0][1]
318
+ assert_equal ["output.test", time, {"val2"=>"sed-ed value bar"}], events[0]
319
+ assert_equal_event_time time, events[1][1]
320
+ assert_equal ["output.test", time, {"val2"=>"sed-ed value poo"}], events[1]
191
321
  end
192
322
 
193
- def test_json_1
194
- d = create_driver(%[
195
- command cat
196
- in_keys message
197
- out_format json
198
- time_key time
323
+ CONFIG_JSON = %[
324
+ command cat
325
+ <format>
326
+ @type tsv
327
+ keys message
328
+ </format>
329
+ <parse>
330
+ @type json
331
+ </parse>
332
+ <extract>
199
333
  tag_key tag
200
- ], 'input.test')
201
-
202
- time = Fluent::EventTime.parse("2011-01-02 13:14:15")
334
+ time_key time
335
+ </extract>
336
+ ]
337
+ CONFIG_JSON_COMPAT = %[
338
+ command cat
339
+ in_keys message
340
+ out_format json
341
+ time_key time
342
+ tag_key tag
343
+ ]
203
344
 
204
- d.expected_emits_length = 1
205
- d.run do
206
- d.emit({"message"=>%[{"time":#{time},"tag":"t1","k1":"v1"}]}, time+10)
345
+ data(
346
+ 'with sections' => CONFIG_JSON,
347
+ 'traditional' => CONFIG_JSON_COMPAT,
348
+ )
349
+ test 'using json format' do |conf|
350
+ d = create_driver(conf)
351
+ time = event_time("2011-01-02 13:14:15 +0900")
352
+
353
+ d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
354
+ i = d.instance
355
+ assert{ i.router }
356
+ d.feed(time, {"message"=>%[{"time":#{time},"tag":"t1","k1":"v1"}]})
207
357
  end
208
358
 
209
- emits = d.emits
210
- assert_equal 1, emits.length
211
- assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
212
- assert_equal_event_time time, emits[0][1]
359
+ assert_equal '{"time":1293941655,"tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
360
+
361
+ events = d.events
362
+ assert_equal 1, events.length
363
+ assert_equal_event_time time, events[0][1]
364
+ assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
213
365
  end
214
366
 
215
- def test_json_with_float_time
216
- d = create_driver(%[
217
- command cat
218
- in_keys message
219
- out_format json
220
- time_key time
367
+ CONFIG_JSON_WITH_FLOAT_TIME = %[
368
+ command cat
369
+ <format>
370
+ @type tsv
371
+ keys message
372
+ </format>
373
+ <parse>
374
+ @type json
375
+ </parse>
376
+ <extract>
221
377
  tag_key tag
222
- ], 'input.test')
378
+ time_key time
379
+ </extract>
380
+ ]
381
+ CONFIG_JSON_WITH_FLOAT_TIME_COMPAT = %[
382
+ command cat
383
+ in_keys message
384
+ out_format json
385
+ time_key time
386
+ tag_key tag
387
+ ]
223
388
 
224
- float_time = Time.parse("2011-01-02 13:14:15").to_f
225
- time = Fluent::EventTime.from_time(Time.at(float_time))
389
+ data(
390
+ 'with sections' => CONFIG_JSON_WITH_FLOAT_TIME,
391
+ 'traditional' => CONFIG_JSON_WITH_FLOAT_TIME_COMPAT,
392
+ )
393
+ test 'using json format with float time' do |conf|
394
+ d = create_driver(conf)
395
+ time = event_time("2011-01-02 13:14:15.123 +0900")
226
396
 
227
- d.expected_emits_length = 1
228
- d.run do
229
- d.emit({"message"=>%[{"time":#{float_time},"tag":"t1","k1":"v1"}]}, time+10)
397
+ d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
398
+ d.feed(time + 10, {"message"=>%[{"time":#{time.sec}.#{time.nsec},"tag":"t1","k1":"v1"}]})
230
399
  end
231
400
 
232
- emits = d.emits
233
- assert_equal 1, emits.length
234
- assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
235
- assert_equal_event_time time, emits[0][1]
401
+ assert_equal '{"time":1293941655.123000000,"tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
402
+
403
+ events = d.events
404
+ assert_equal 1, events.length
405
+ assert_equal_event_time time, events[0][1]
406
+ assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
236
407
  end
237
408
 
238
- def test_json_with_time_format
239
- d = create_driver(%[
240
- command cat
241
- in_keys message
242
- out_format json
409
+ CONFIG_JSON_WITH_TIME_FORMAT = %[
410
+ command cat
411
+ <format>
412
+ @type tsv
413
+ keys message
414
+ </format>
415
+ <parse>
416
+ @type json
417
+ </parse>
418
+ <extract>
419
+ tag_key tag
243
420
  time_key time
421
+ time_type string
244
422
  time_format %d/%b/%Y %H:%M:%S.%N %z
245
- tag_key tag
246
- ], 'input.test')
423
+ </extract>
424
+ ]
425
+ CONFIG_JSON_WITH_TIME_FORMAT_COMPAT = %[
426
+ command cat
427
+ in_keys message
428
+ out_format json
429
+ time_key time
430
+ time_format %d/%b/%Y %H:%M:%S.%N %z
431
+ tag_key tag
432
+ ]
247
433
 
434
+ data(
435
+ 'with sections' => CONFIG_JSON_WITH_TIME_FORMAT,
436
+ 'traditional' => CONFIG_JSON_WITH_TIME_FORMAT_COMPAT,
437
+ )
438
+ test 'using json format with custom time format' do |conf|
439
+ d = create_driver(conf)
248
440
  time_str = "28/Feb/2013 12:00:00.123456789 +0900"
249
- time = Fluent::EventTime.from_time(Time.strptime(time_str, "%d/%b/%Y %H:%M:%S.%N %z"))
441
+ time = event_time(time_str, format: "%d/%b/%Y %H:%M:%S.%N %z")
250
442
 
251
- d.expected_emits_length = 1
252
- d.run do
253
- d.emit({"message"=>%[{"time":"#{time_str}","tag":"t1","k1":"v1"}]}, time+10)
443
+ d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
444
+ d.feed(time + 10, {"message"=>%[{"time":"#{time_str}","tag":"t1","k1":"v1"}]})
254
445
  end
255
446
 
256
- emits = d.emits
257
- assert_equal 1, emits.length
258
- assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
259
- assert_equal_event_time time, emits[0][1]
447
+ assert_equal '{"time":"28/Feb/2013 12:00:00.123456789 +0900","tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
448
+
449
+ events = d.events
450
+ assert_equal 1, events.length
451
+ assert_equal_event_time time, events[0][1]
452
+ assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
260
453
  end
261
- end
262
454
 
455
+ CONFIG_ROUND_ROBIN = %[
456
+ command ruby -e 'STDOUT.sync = true; STDIN.each_line{|line| puts line.chomp + "\t" + Process.pid.to_s }'
457
+ num_children 3
458
+ <inject>
459
+ tag_key tag
460
+ time_key time_in
461
+ time_type string
462
+ time_format %Y-%m-%d %H:%M:%S
463
+ </inject>
464
+ <format>
465
+ keys ["time_in", "tag", "k1"]
466
+ </format>
467
+ <parse>
468
+ keys ["time_out", "tag", "k2", "child_pid"]
469
+ </parse>
470
+ <extract>
471
+ tag_key tag
472
+ time_key time_out
473
+ time_type string
474
+ time_format %Y-%m-%d %H:%M:%S
475
+ </extract>
476
+ ]
477
+ CONFIG_ROUND_ROBIN_COMPAT = %[
478
+ command ruby -e 'STDOUT.sync = true; STDIN.each_line{|line| puts line.chomp + "\t" + Process.pid.to_s }'
479
+ in_keys time_in,tag,k1
480
+ out_keys time_out,tag,k2,child_pid
481
+ tag_key tag
482
+ in_time_key time_in
483
+ out_time_key time_out
484
+ time_format %Y-%m-%d %H:%M:%S
485
+ localtime
486
+ num_children 3
487
+ ]
488
+
489
+ data(
490
+ 'with sections' => CONFIG_ROUND_ROBIN,
491
+ 'traditional' => CONFIG_ROUND_ROBIN_COMPAT,
492
+ )
493
+ test 'using child processes by round robin' do |conf|
494
+ d = create_driver(conf)
495
+ time = event_time("2011-01-02 13:14:15")
496
+
497
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: true, shutdown: false){ d.feed(time, {"k1"=>1}) }
498
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>2}) }
499
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>3}) }
500
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>4}) }
501
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>5}) }
502
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>6}) }
503
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>7}) }
504
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>8}) }
505
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: true ){ d.feed(time, {"k1"=>9}) }
506
+
507
+ assert_equal "2011-01-02 13:14:15\ttest\t1\n", d.formatted[0]
508
+ assert_equal "2011-01-02 13:14:15\ttest\t2\n", d.formatted[1]
509
+ assert_equal "2011-01-02 13:14:15\ttest\t3\n", d.formatted[2]
510
+ assert_equal "2011-01-02 13:14:15\ttest\t4\n", d.formatted[3]
511
+ assert_equal "2011-01-02 13:14:15\ttest\t5\n", d.formatted[4]
512
+ assert_equal "2011-01-02 13:14:15\ttest\t6\n", d.formatted[5]
513
+ assert_equal "2011-01-02 13:14:15\ttest\t7\n", d.formatted[6]
514
+ assert_equal "2011-01-02 13:14:15\ttest\t8\n", d.formatted[7]
515
+ assert_equal "2011-01-02 13:14:15\ttest\t9\n", d.formatted[8]
516
+
517
+ events = d.events
518
+ assert_equal 9, events.length
519
+
520
+ pid_list = []
521
+ events.each do |event|
522
+ pid = event[2]['child_pid']
523
+ pid_list << pid unless pid_list.include?(pid)
524
+ end
525
+ assert_equal 3, pid_list.size, "the number of pids should be same with number of child processes: #{pid_list.inspect}"
526
+
527
+ assert_equal pid_list[0], events[0][2]['child_pid']
528
+ assert_equal pid_list[1], events[1][2]['child_pid']
529
+ assert_equal pid_list[2], events[2][2]['child_pid']
530
+ assert_equal pid_list[0], events[3][2]['child_pid']
531
+ assert_equal pid_list[1], events[4][2]['child_pid']
532
+ assert_equal pid_list[2], events[5][2]['child_pid']
533
+ assert_equal pid_list[0], events[6][2]['child_pid']
534
+ assert_equal pid_list[1], events[7][2]['child_pid']
535
+ assert_equal pid_list[2], events[8][2]['child_pid']
536
+ end
537
+
538
+ # child process exits per 3 lines
539
+ CONFIG_RESPAWN = %[
540
+ command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; proc.call; proc.call; proc.call'
541
+ num_children 4
542
+ child_respawn -1
543
+ <inject>
544
+ tag_key tag
545
+ time_key time_in
546
+ time_type unixtime
547
+ </inject>
548
+ <format>
549
+ keys ["time_in", "tag", "k1"]
550
+ </format>
551
+ <parse>
552
+ keys ["time_out", "tag", "k2", "child_pid"]
553
+ </parse>
554
+ <extract>
555
+ tag_key tag
556
+ time_key time_out
557
+ time_type unixtime
558
+ </extract>
559
+ ]
560
+
561
+ CONFIG_RESPAWN_COMPAT = %[
562
+ command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; proc.call; proc.call; proc.call'
563
+ num_children 4
564
+ child_respawn -1
565
+ in_keys time_in,tag,k1
566
+ out_keys time_out,tag,k2,child_pid
567
+ tag_key tag
568
+ in_time_key time_in
569
+ out_time_key time_out
570
+ # time_format %Y-%m-%d %H:%M:%S
571
+ # localtime
572
+ ]
573
+
574
+ data(
575
+ 'with sections' => CONFIG_RESPAWN,
576
+ 'traditional' => CONFIG_RESPAWN_COMPAT,
577
+ )
578
+ test 'emit events via child processes which exits sometimes' do |conf|
579
+ d = create_driver(conf)
580
+ time = event_time("2011-01-02 13:14:15")
581
+
582
+ countup = 0
583
+
584
+ d.run(start: true, shutdown: false)
585
+
586
+ assert_equal 4, d.instance.instance_eval{ @_child_process_processes.size }
587
+
588
+ 20.times do
589
+ d.run(default_tag: 'test', expect_emits: 1, timeout: 10, force_flush_retry: true, start: false, shutdown: false) do
590
+ d.feed(time, {"k1"=>countup}); countup += 1
591
+ d.feed(time, {"k1"=>countup}); countup += 1
592
+ d.feed(time, {"k1"=>countup}); countup += 1
593
+ end
594
+ end
595
+
596
+ events = d.events
597
+ assert_equal 60, events.length
598
+
599
+ pid_list = []
600
+ events.each do |event|
601
+ pid = event[2]['child_pid']
602
+ pid_list << pid unless pid_list.include?(pid)
603
+ end
604
+ # the number of pids should be same with number of child processes
605
+ assert{ pid_list.size >= 18 }
606
+
607
+ logs = d.instance.log.out.logs
608
+ assert{ logs.select{|l| l.include?("child process exits with error code") }.size >= 18 } # 20
609
+ assert{ logs.select{|l| l.include?("respawning child process") }.size >= 18 } # 20
610
+
611
+ d.run(start: false, shutdown: true)
612
+ end
613
+ end