fluentd 0.14.4-x86-mingw32 → 0.14.5-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/ChangeLog +18 -0
  3. data/example/in_forward.conf +3 -0
  4. data/example/in_forward_client.conf +37 -0
  5. data/example/in_forward_shared_key.conf +15 -0
  6. data/example/in_forward_users.conf +24 -0
  7. data/example/out_forward.conf +13 -13
  8. data/example/out_forward_client.conf +109 -0
  9. data/example/out_forward_shared_key.conf +36 -0
  10. data/example/out_forward_users.conf +65 -0
  11. data/example/{out_buffered_null.conf → out_null.conf} +10 -6
  12. data/example/secondary_file.conf +41 -0
  13. data/lib/fluent/agent.rb +3 -1
  14. data/lib/fluent/plugin/buffer.rb +5 -1
  15. data/lib/fluent/plugin/in_forward.rb +300 -50
  16. data/lib/fluent/plugin/in_tail.rb +41 -85
  17. data/lib/fluent/plugin/multi_output.rb +4 -0
  18. data/lib/fluent/plugin/out_forward.rb +326 -209
  19. data/lib/fluent/plugin/out_null.rb +37 -0
  20. data/lib/fluent/plugin/out_secondary_file.rb +128 -0
  21. data/lib/fluent/plugin/out_stdout.rb +38 -2
  22. data/lib/fluent/plugin/output.rb +13 -5
  23. data/lib/fluent/root_agent.rb +1 -1
  24. data/lib/fluent/test/startup_shutdown.rb +33 -0
  25. data/lib/fluent/version.rb +1 -1
  26. data/test/plugin/test_in_forward.rb +906 -441
  27. data/test/plugin/test_in_monitor_agent.rb +4 -0
  28. data/test/plugin/test_in_tail.rb +681 -663
  29. data/test/plugin/test_out_forward.rb +150 -208
  30. data/test/plugin/test_out_null.rb +85 -9
  31. data/test/plugin/test_out_secondary_file.rb +432 -0
  32. data/test/plugin/test_out_stdout.rb +143 -45
  33. data/test/test_root_agent.rb +42 -0
  34. metadata +14 -9
  35. data/lib/fluent/plugin/out_buffered_null.rb +0 -59
  36. data/lib/fluent/plugin/out_buffered_stdout.rb +0 -70
  37. data/test/plugin/test_out_buffered_null.rb +0 -79
  38. data/test/plugin/test_out_buffered_stdout.rb +0 -122
@@ -119,6 +119,8 @@ EOC
119
119
  "@id"=>"null",
120
120
  "@type" => "null"
121
121
  },
122
+ "buffer_queue_length" => 0,
123
+ "buffer_total_queued_size" => 0,
122
124
  "output_plugin" => true,
123
125
  "plugin_category" => "output",
124
126
  "plugin_id" => "null",
@@ -285,6 +287,8 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
285
287
  "@id" => "null",
286
288
  "@type" => "null"
287
289
  },
290
+ "buffer_queue_length" => 0,
291
+ "buffer_total_queued_size" => 0,
288
292
  "output_plugin" => true,
289
293
  "plugin_category" => "output",
290
294
  "plugin_id" => "null",
@@ -1,5 +1,5 @@
1
1
  require_relative '../helper'
2
- require 'fluent/test'
2
+ require 'fluent/test/driver/input'
3
3
  require 'fluent/plugin/in_tail'
4
4
  require 'fluent/plugin/buffer'
5
5
  require 'fluent/system_config'
@@ -16,7 +16,7 @@ class TailInputTest < Test::Unit::TestCase
16
16
  # ensure files are closed for Windows, on which deleted files
17
17
  # are still visible from filesystem
18
18
  GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
19
- FileUtils.remove_entry_secure(TMP_DIR)
19
+ FileUtils.remove_entry_secure(TMP_DIR, true)
20
20
  end
21
21
  FileUtils.mkdir_p(TMP_DIR)
22
22
  end
@@ -28,102 +28,206 @@ class TailInputTest < Test::Unit::TestCase
28
28
 
29
29
  TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
30
30
 
31
- CONFIG = %[
32
- path #{TMP_DIR}/tail.txt
33
- tag t1
34
- rotate_wait 2s
35
- ]
36
- COMMON_CONFIG = CONFIG + %[
37
- pos_file #{TMP_DIR}/tail.pos
38
- ]
39
- CONFIG_READ_FROM_HEAD = %[
40
- read_from_head true
41
- ]
42
- CONFIG_ENABLE_WATCH_TIMER = %[
43
- enable_watch_timer false
44
- ]
45
- SINGLE_LINE_CONFIG = %[
46
- format /(?<message>.*)/
47
- ]
31
+ CONFIG = config_element("ROOT", "", {
32
+ "path" => "#{TMP_DIR}/tail.txt",
33
+ "tag" => "t1",
34
+ "rotate_wait" => "2s"
35
+ })
36
+ COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
37
+ CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
38
+ CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
39
+ SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
40
+ PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
41
+ MULTILINE_CONFIG = config_element(
42
+ "", "", {
43
+ "format" => "multiline",
44
+ "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
45
+ "format_firstline" => "/^[s]/"
46
+ })
47
+ PARSE_MULTILINE_CONFIG = config_element(
48
+ "", "", {},
49
+ [config_element("parse", "", {
50
+ "@type" => "multiline",
51
+ "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
52
+ "format_firstline" => "/^[s]/"
53
+ })
54
+ ])
48
55
 
49
56
  def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
50
57
  config = use_common_conf ? COMMON_CONFIG + conf : conf
51
- Fluent::Test::InputTestDriver.new(Fluent::NewTailInput).configure(config)
58
+ Fluent::Test::Driver::Input.new(Fluent::Plugin::TailInput).configure(config)
52
59
  end
53
60
 
54
- def test_configure
55
- d = create_driver
56
- assert_equal ["#{TMP_DIR}/tail.txt"], d.instance.paths
57
- assert_equal "t1", d.instance.tag
58
- assert_equal 2, d.instance.rotate_wait
59
- assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
60
- assert_equal 1000, d.instance.read_lines_limit
61
- end
61
+ sub_test_case "configure" do
62
+ test "plain single line" do
63
+ d = create_driver
64
+ assert_equal ["#{TMP_DIR}/tail.txt"], d.instance.paths
65
+ assert_equal "t1", d.instance.tag
66
+ assert_equal 2, d.instance.rotate_wait
67
+ assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
68
+ assert_equal 1000, d.instance.read_lines_limit
69
+ end
70
+
71
+ data("empty" => config_element,
72
+ "w/o @type" => config_element("", "", {}, [config_element("parse", "", {})]))
73
+ test "w/o parse section" do |conf|
74
+ assert_raise(Fluent::ConfigError) do
75
+ create_driver(conf)
76
+ end
77
+ end
62
78
 
63
- def test_configure_encoding
64
- # valid encoding
65
- d = create_driver(SINGLE_LINE_CONFIG + 'encoding utf-8')
66
- assert_equal Encoding::UTF_8, d.instance.encoding
79
+ sub_test_case "encoding" do
80
+ test "valid" do
81
+ conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" })
82
+ d = create_driver(conf)
83
+ assert_equal Encoding::UTF_8, d.instance.encoding
84
+ end
67
85
 
68
- # invalid encoding
69
- assert_raise(Fluent::ConfigError) do
70
- create_driver(SINGLE_LINE_CONFIG + 'encoding no-such-encoding')
86
+ test "invalid" do
87
+ conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "no-such-encoding" })
88
+ assert_raise(Fluent::ConfigError) do
89
+ create_driver(conf)
90
+ end
91
+ end
71
92
  end
72
- end
73
93
 
74
- def test_configure_from_encoding
75
- # If only specified from_encoding raise ConfigError
76
- assert_raise(Fluent::ConfigError) do
77
- create_driver(SINGLE_LINE_CONFIG + 'from_encoding utf-8')
78
- end
94
+ sub_test_case "from_encoding" do
95
+ test "only specified from_encoding raise ConfigError" do
96
+ conf = SINGLE_LINE_CONFIG + config_element("", "", { "from_encoding" => "utf-8" })
97
+ assert_raise(Fluent::ConfigError) do
98
+ create_driver(conf)
99
+ end
100
+ end
79
101
 
80
- # valid setting
81
- d = create_driver %[
82
- format /(?<message>.*)/
83
- read_from_head true
84
- from_encoding utf-8
85
- encoding utf-8
86
- ]
87
- assert_equal Encoding::UTF_8, d.instance.from_encoding
102
+ test "valid" do
103
+ conf = SINGLE_LINE_CONFIG + config_element("", "", {
104
+ "from_encoding" => "utf-8",
105
+ "encoding" => "utf-8"
106
+ })
107
+ d = create_driver(conf)
108
+ assert_equal(Encoding::UTF_8, d.instance.from_encoding)
109
+ end
88
110
 
89
- # invalid from_encoding
90
- assert_raise(Fluent::ConfigError) do
91
- d = create_driver %[
92
- format /(?<message>.*)/
93
- read_from_head true
94
- from_encoding no-such-encoding
95
- encoding utf-8
96
- ]
111
+ test "invalid" do
112
+ conf = SINGLE_LINE_CONFIG + config_element("", "", {
113
+ "from_encoding" => "no-such-encoding",
114
+ "encoding" => "utf-8"
115
+ })
116
+ assert_raise(Fluent::ConfigError) do
117
+ create_driver(conf)
118
+ end
119
+ end
97
120
  end
98
121
  end
99
122
 
100
- # TODO: Should using more better approach instead of sleep wait
123
+ sub_test_case "singleline" do
124
+ data(flat: SINGLE_LINE_CONFIG,
125
+ parse: PARSE_SINGLE_LINE_CONFIG)
126
+ def test_emit(data)
127
+ config = data
128
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
129
+ f.puts "test1"
130
+ f.puts "test2"
131
+ }
132
+
133
+ d = create_driver(config)
101
134
 
102
- def test_emit
103
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
104
- f.puts "test1"
105
- f.puts "test2"
106
- }
135
+ d.run(expect_emits: 1) do
136
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
137
+ f.puts "test3"
138
+ f.puts "test4"
139
+ }
140
+ end
107
141
 
108
- d = create_driver
142
+ events = d.events
143
+ assert_equal(true, events.length > 0)
144
+ assert_equal({"message" => "test3"}, events[0][2])
145
+ assert_equal({"message" => "test4"}, events[1][2])
146
+ assert(events[0][1].is_a?(Fluent::EventTime))
147
+ assert(events[1][1].is_a?(Fluent::EventTime))
148
+ assert_equal(1, d.emit_count)
149
+ end
150
+
151
+ data('flat 1' => [:flat, 1, 2],
152
+ 'flat 10' => [:flat, 10, 1],
153
+ 'parse 1' => [:parse, 1, 2],
154
+ 'parse 10' => [:parse, 10, 1])
155
+ def test_emit_with_read_lines_limit(data)
156
+ config_style, limit, num_events = data
157
+ case config_style
158
+ when :flat
159
+ config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit })
160
+ when :parse
161
+ config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit }) + PARSE_SINGLE_LINE_CONFIG
162
+ end
163
+ d = create_driver(config)
164
+ msg = 'test' * 500 # in_tail reads 2048 bytes at once.
109
165
 
110
- d.run do
111
- sleep 1
166
+ d.run(expect_emits: 1) do
167
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
168
+ f.puts msg
169
+ f.puts msg
170
+ }
171
+ end
112
172
 
113
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
114
- f.puts "test3"
115
- f.puts "test4"
173
+ events = d.events
174
+ assert_equal(true, events.length > 0)
175
+ assert_equal({"message" => msg}, events[0][2])
176
+ assert_equal({"message" => msg}, events[1][2])
177
+ assert_equal(num_events, d.emit_count)
178
+ end
179
+
180
+ data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
181
+ parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
182
+ def test_emit_with_read_from_head(data)
183
+ config = data
184
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
185
+ f.puts "test1"
186
+ f.puts "test2"
116
187
  }
117
- sleep 1
188
+
189
+ d = create_driver(config)
190
+
191
+ d.run(expect_emits: 2) do
192
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
193
+ f.puts "test3"
194
+ f.puts "test4"
195
+ }
196
+ end
197
+
198
+ events = d.events
199
+ assert(events.length > 0)
200
+ assert_equal({"message" => "test1"}, events[0][2])
201
+ assert_equal({"message" => "test2"}, events[1][2])
202
+ assert_equal({"message" => "test3"}, events[2][2])
203
+ assert_equal({"message" => "test4"}, events[3][2])
118
204
  end
119
205
 
120
- emits = d.emits
121
- assert_equal(true, emits.length > 0)
122
- assert_equal({"message" => "test3"}, emits[0][2])
123
- assert_equal({"message" => "test4"}, emits[1][2])
124
- assert(emits[0][1].is_a?(Fluent::EventTime))
125
- assert(emits[1][1].is_a?(Fluent::EventTime))
126
- assert_equal(1, d.emit_streams.size)
206
+ data(flat: CONFIG_ENABLE_WATCH_TIMER + SINGLE_LINE_CONFIG,
207
+ parse: CONFIG_ENABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG)
208
+ def test_emit_with_enable_watch_timer(data)
209
+ config = data
210
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
211
+ f.puts "test1"
212
+ f.puts "test2"
213
+ }
214
+
215
+ d = create_driver(config)
216
+
217
+ d.run(expect_emits: 1) do
218
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
219
+ f.puts "test3"
220
+ f.puts "test4"
221
+ }
222
+ # according to cool.io's stat_watcher.c, systems without inotify will use
223
+ # an "automatic" value, typically around 5 seconds
224
+ end
225
+
226
+ events = d.events
227
+ assert(events.length > 0)
228
+ assert_equal({"message" => "test3"}, events[0][2])
229
+ assert_equal({"message" => "test4"}, events[1][2])
230
+ end
127
231
  end
128
232
 
129
233
  class TestWithSystem < self
@@ -163,198 +267,133 @@ class TailInputTest < Test::Unit::TestCase
163
267
 
164
268
  d = create_driver
165
269
 
166
- d.run do
167
- sleep 1
168
-
270
+ d.run(expect_emits: 1) do
169
271
  File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
170
272
  f.puts "test3"
171
273
  f.puts "test4"
172
274
  }
173
- sleep 1
174
275
  end
175
276
 
176
- emits = d.emits
177
- assert_equal(true, emits.length > 0)
178
- assert_equal({"message" => "test3"}, emits[0][2])
179
- assert_equal({"message" => "test4"}, emits[1][2])
180
- assert(emits[0][1].is_a?(Fluent::EventTime))
181
- assert(emits[1][1].is_a?(Fluent::EventTime))
182
- assert_equal(1, d.emit_streams.size)
277
+ events = d.events
278
+ assert_equal(true, events.length > 0)
279
+ assert_equal({"message" => "test3"}, events[0][2])
280
+ assert_equal({"message" => "test4"}, events[1][2])
281
+ assert(events[0][1].is_a?(Fluent::EventTime))
282
+ assert(events[1][1].is_a?(Fluent::EventTime))
283
+ assert_equal(1, d.emit_count)
183
284
  pos = d.instance.instance_variable_get(:@pf_file)
184
285
  mode = "%o" % File.stat(pos).mode
185
286
  assert_equal OVERRIDE_FILE_PERMISSION, mode[-3, 3].to_i
186
287
  end
187
288
  end
188
289
 
189
- data('1' => [1, 2], '10' => [10, 1])
190
- def test_emit_with_read_lines_limit(data)
191
- limit, num_emits = data
192
- d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + "read_lines_limit #{limit}")
193
- msg = 'test' * 500 # in_tail reads 2048 bytes at once.
194
-
195
- d.run do
196
- sleep 1
197
-
198
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
199
- f.puts msg
200
- f.puts msg
201
- }
202
- sleep 1
203
- end
204
-
205
- emits = d.emits
206
- assert_equal(true, emits.length > 0)
207
- assert_equal({"message" => msg}, emits[0][2])
208
- assert_equal({"message" => msg}, emits[1][2])
209
- assert_equal(num_emits, d.emit_streams.size)
210
- end
211
-
212
- def test_emit_with_read_from_head
213
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
214
- f.puts "test1"
215
- f.puts "test2"
216
- }
217
-
218
- d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
219
-
220
- d.run do
221
- sleep 1
290
+ sub_test_case "rotate file" do
291
+ data(flat: SINGLE_LINE_CONFIG,
292
+ parse: PARSE_SINGLE_LINE_CONFIG)
293
+ def test_rotate_file(data)
294
+ config = data
295
+ events = sub_test_rotate_file(config, expect_emits: 2)
296
+ assert_equal(4, events.length)
297
+ assert_equal({"message" => "test3"}, events[0][2])
298
+ assert_equal({"message" => "test4"}, events[1][2])
299
+ assert_equal({"message" => "test5"}, events[2][2])
300
+ assert_equal({"message" => "test6"}, events[3][2])
301
+ end
302
+
303
+ data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
304
+ parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
305
+ def test_rotate_file_with_read_from_head(data)
306
+ config = data
307
+ events = sub_test_rotate_file(config, expect_records: 6)
308
+ assert_equal(6, events.length)
309
+ assert_equal({"message" => "test1"}, events[0][2])
310
+ assert_equal({"message" => "test2"}, events[1][2])
311
+ assert_equal({"message" => "test3"}, events[2][2])
312
+ assert_equal({"message" => "test4"}, events[3][2])
313
+ assert_equal({"message" => "test5"}, events[4][2])
314
+ assert_equal({"message" => "test6"}, events[5][2])
315
+ end
316
+
317
+ data(flat: SINGLE_LINE_CONFIG,
318
+ parse: PARSE_SINGLE_LINE_CONFIG)
319
+ def test_rotate_file_with_write_old(data)
320
+ config = data
321
+ events = sub_test_rotate_file(config, expect_emits: 3) { |rotated_file|
322
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
323
+ rotated_file.puts "test7"
324
+ rotated_file.puts "test8"
325
+ rotated_file.flush
222
326
 
223
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
224
- f.puts "test3"
225
- f.puts "test4"
327
+ sleep 1
328
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
329
+ f.puts "test5"
330
+ f.puts "test6"
331
+ }
226
332
  }
227
- sleep 1
228
- end
229
-
230
- emits = d.emits
231
- assert(emits.length > 0)
232
- assert_equal({"message" => "test1"}, emits[0][2])
233
- assert_equal({"message" => "test2"}, emits[1][2])
234
- assert_equal({"message" => "test3"}, emits[2][2])
235
- assert_equal({"message" => "test4"}, emits[3][2])
236
- end
237
-
238
- def test_emit_with_enable_watch_timer
239
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
240
- f.puts "test1"
241
- f.puts "test2"
242
- }
243
-
244
- d = create_driver(CONFIG_ENABLE_WATCH_TIMER + SINGLE_LINE_CONFIG)
245
-
246
- d.run do
247
- sleep 1
248
-
249
- File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
250
- f.puts "test3"
251
- f.puts "test4"
333
+ assert_equal(6, events.length)
334
+ assert_equal({"message" => "test3"}, events[0][2])
335
+ assert_equal({"message" => "test4"}, events[1][2])
336
+ assert_equal({"message" => "test7"}, events[2][2])
337
+ assert_equal({"message" => "test8"}, events[3][2])
338
+ assert_equal({"message" => "test5"}, events[4][2])
339
+ assert_equal({"message" => "test6"}, events[5][2])
340
+ end
341
+
342
+ data(flat: SINGLE_LINE_CONFIG,
343
+ parse: PARSE_SINGLE_LINE_CONFIG)
344
+ def test_rotate_file_with_write_old_and_no_new_file(data)
345
+ config = data
346
+ events = sub_test_rotate_file(config, expect_emits: 2) { |rotated_file|
347
+ rotated_file.puts "test7"
348
+ rotated_file.puts "test8"
349
+ rotated_file.flush
252
350
  }
253
- # according to cool.io's stat_watcher.c, systems without inotify will use
254
- # an "automatic" value, typically around 5 seconds
255
- sleep 10
351
+ assert_equal(4, events.length)
352
+ assert_equal({"message" => "test3"}, events[0][2])
353
+ assert_equal({"message" => "test4"}, events[1][2])
354
+ assert_equal({"message" => "test7"}, events[2][2])
355
+ assert_equal({"message" => "test8"}, events[3][2])
256
356
  end
257
357
 
258
- emits = d.emits
259
- assert(emits.length > 0)
260
- assert_equal({"message" => "test3"}, emits[0][2])
261
- assert_equal({"message" => "test4"}, emits[1][2])
262
- end
263
-
264
- def test_rotate_file
265
- emits = sub_test_rotate_file(SINGLE_LINE_CONFIG)
266
- assert_equal(4, emits.length)
267
- assert_equal({"message" => "test3"}, emits[0][2])
268
- assert_equal({"message" => "test4"}, emits[1][2])
269
- assert_equal({"message" => "test5"}, emits[2][2])
270
- assert_equal({"message" => "test6"}, emits[3][2])
271
- end
272
-
273
- def test_rotate_file_with_read_from_head
274
- emits = sub_test_rotate_file(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
275
- assert_equal(6, emits.length)
276
- assert_equal({"message" => "test1"}, emits[0][2])
277
- assert_equal({"message" => "test2"}, emits[1][2])
278
- assert_equal({"message" => "test3"}, emits[2][2])
279
- assert_equal({"message" => "test4"}, emits[3][2])
280
- assert_equal({"message" => "test5"}, emits[4][2])
281
- assert_equal({"message" => "test6"}, emits[5][2])
282
- end
283
-
284
- def test_rotate_file_with_write_old
285
- emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
286
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
287
- rotated_file.puts "test7"
288
- rotated_file.puts "test8"
289
- rotated_file.flush
290
-
291
- sleep 1
292
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
293
- f.puts "test5"
294
- f.puts "test6"
295
- }
296
- }
297
- assert_equal(6, emits.length)
298
- assert_equal({"message" => "test3"}, emits[0][2])
299
- assert_equal({"message" => "test4"}, emits[1][2])
300
- assert_equal({"message" => "test7"}, emits[2][2])
301
- assert_equal({"message" => "test8"}, emits[3][2])
302
- assert_equal({"message" => "test5"}, emits[4][2])
303
- assert_equal({"message" => "test6"}, emits[5][2])
304
- end
305
-
306
- def test_rotate_file_with_write_old_and_no_new_file
307
- emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
308
- rotated_file.puts "test7"
309
- rotated_file.puts "test8"
310
- rotated_file.flush
311
- }
312
- assert_equal(4, emits.length)
313
- assert_equal({"message" => "test3"}, emits[0][2])
314
- assert_equal({"message" => "test4"}, emits[1][2])
315
- assert_equal({"message" => "test7"}, emits[2][2])
316
- assert_equal({"message" => "test8"}, emits[3][2])
317
- end
318
-
319
- def sub_test_rotate_file(config = nil)
320
- file = Fluent::FileWrapper.open("#{TMP_DIR}/tail.txt", "wb")
321
- file.puts "test1"
322
- file.puts "test2"
323
- file.flush
324
-
325
- d = create_driver(config)
326
- d.run do
327
- sleep 1
328
-
329
- file.puts "test3"
330
- file.puts "test4"
358
+ def sub_test_rotate_file(config = nil, expect_emits: nil, expect_records: nil, timeout: nil)
359
+ file = Fluent::FileWrapper.open("#{TMP_DIR}/tail.txt", "wb")
360
+ file.puts "test1"
361
+ file.puts "test2"
331
362
  file.flush
332
- sleep 1
333
363
 
334
- FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
335
- if block_given?
336
- yield file
337
- sleep 1
338
- else
339
- sleep 1
340
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
341
- sleep 1
364
+ d = create_driver(config)
365
+ d.run(expect_emits: expect_emits, expect_records: expect_records, timeout: timeout) do
366
+ size = d.emit_count
367
+ file.puts "test3"
368
+ file.puts "test4"
369
+ file.flush
370
+ sleep(0.1) until d.emit_count >= size + 1
371
+ size = d.emit_count
372
+
373
+ if Fluent.windows?
374
+ file.close
375
+ FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt", force: true)
376
+ file = File.open("#{TMP_DIR}/tail.txt", "ab")
377
+ else
378
+ FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
379
+ end
380
+ if block_given?
381
+ yield file
382
+ else
383
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
384
+ sleep 1
342
385
 
343
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
344
- f.puts "test5"
345
- f.puts "test6"
346
- }
347
- sleep 1
386
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
387
+ f.puts "test5"
388
+ f.puts "test6"
389
+ }
390
+ end
348
391
  end
349
- end
350
392
 
351
- d.run do
352
- sleep 1
393
+ d.events
394
+ ensure
395
+ file.close if file && !file.closed?
353
396
  end
354
-
355
- d.emits
356
- ensure
357
- file.close if file
358
397
  end
359
398
 
360
399
  def test_lf
@@ -374,9 +413,9 @@ class TailInputTest < Test::Unit::TestCase
374
413
  sleep 1
375
414
  end
376
415
 
377
- emits = d.emits
378
- assert_equal(true, emits.length > 0)
379
- assert_equal({"message" => "test3test4"}, emits[0][2])
416
+ events = d.events
417
+ assert_equal(true, events.length > 0)
418
+ assert_equal({"message" => "test3test4"}, events[0][2])
380
419
  end
381
420
 
382
421
  def test_whitespace
@@ -384,9 +423,7 @@ class TailInputTest < Test::Unit::TestCase
384
423
 
385
424
  d = create_driver
386
425
 
387
- d.run do
388
- sleep 1
389
-
426
+ d.run(expect_emits: 1) do
390
427
  File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
391
428
  f.puts " " # 4 spaces
392
429
  f.puts " 4 spaces"
@@ -395,328 +432,346 @@ class TailInputTest < Test::Unit::TestCase
395
432
  f.puts " tab"
396
433
  f.puts "tab "
397
434
  }
398
- sleep 1
399
435
  end
400
436
 
401
- emits = d.emits
402
- assert_equal(true, emits.length > 0)
403
- assert_equal({"message" => " "}, emits[0][2])
404
- assert_equal({"message" => " 4 spaces"}, emits[1][2])
405
- assert_equal({"message" => "4 spaces "}, emits[2][2])
406
- assert_equal({"message" => " "}, emits[3][2])
407
- assert_equal({"message" => " tab"}, emits[4][2])
408
- assert_equal({"message" => "tab "}, emits[5][2])
437
+ events = d.events
438
+ assert_equal(true, events.length > 0)
439
+ assert_equal({"message" => " "}, events[0][2])
440
+ assert_equal({"message" => " 4 spaces"}, events[1][2])
441
+ assert_equal({"message" => "4 spaces "}, events[2][2])
442
+ assert_equal({"message" => " "}, events[3][2])
443
+ assert_equal({"message" => " tab"}, events[4][2])
444
+ assert_equal({"message" => "tab "}, events[5][2])
409
445
  end
410
446
 
411
447
  data(
412
- 'default encoding' => ['', Encoding::ASCII_8BIT],
413
- 'explicit encoding config' => ['encoding utf-8', Encoding::UTF_8])
448
+ 'flat default encoding' => [SINGLE_LINE_CONFIG, Encoding::ASCII_8BIT],
449
+ 'flat explicit encoding config' => [SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8],
450
+ 'parse default encoding' => [PARSE_SINGLE_LINE_CONFIG, Encoding::ASCII_8BIT],
451
+ 'parse explicit encoding config' => [PARSE_SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8])
414
452
  def test_encoding(data)
415
453
  encoding_config, encoding = data
416
454
 
417
- d = create_driver(SINGLE_LINE_CONFIG + CONFIG_READ_FROM_HEAD + encoding_config)
418
-
419
- d.run do
420
- sleep 1
455
+ d = create_driver(CONFIG_READ_FROM_HEAD + encoding_config)
421
456
 
457
+ d.run(expect_emits: 1) do
422
458
  File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
423
459
  f.puts "test"
424
460
  }
425
- sleep 1
426
461
  end
427
462
 
428
- emits = d.emits
429
- assert_equal(encoding, emits[0][2]['message'].encoding)
463
+ events = d.events
464
+ assert_equal(encoding, events[0][2]['message'].encoding)
430
465
  end
431
466
 
432
467
  def test_from_encoding
433
- d = create_driver %[
434
- format /(?<message>.*)/
435
- read_from_head true
436
- from_encoding cp932
437
- encoding utf-8
438
- ]
439
-
440
- d.run do
441
- sleep 1
442
-
468
+ conf = config_element(
469
+ "", "", {
470
+ "format" => "/(?<message>.*)/",
471
+ "read_from_head" => "true",
472
+ "from_encoding" => "cp932",
473
+ "encoding" => "utf-8"
474
+ })
475
+ d = create_driver(conf)
476
+ cp932_message = "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
477
+ utf8_message = cp932_message.encode(Encoding::UTF_8)
478
+
479
+ d.run(expect_emits: 1) do
443
480
  File.open("#{TMP_DIR}/tail.txt", "w:cp932") {|f|
444
- f.puts "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
445
- }
446
- sleep 1
447
- end
448
-
449
- emits = d.emits
450
- assert_equal("\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932).encode(Encoding::UTF_8), emits[0][2]['message'])
451
- assert_equal(Encoding::UTF_8, emits[0][2]['message'].encoding)
452
- end
453
-
454
- # multiline mode test
455
-
456
- def test_multiline
457
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
458
-
459
- d = create_driver %[
460
- format multiline
461
- format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
462
- format_firstline /^[s]/
463
- ]
464
- d.run do
465
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
466
- f.puts "f test1"
467
- f.puts "s test2"
468
- f.puts "f test3"
469
- f.puts "f test4"
470
- f.puts "s test5"
471
- f.puts "s test6"
472
- f.puts "f test7"
473
- f.puts "s test8"
481
+ f.puts cp932_message
474
482
  }
475
- sleep 1
476
-
477
- emits = d.emits
478
- assert(emits.length == 3)
479
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
480
- assert_equal({"message1" => "test5"}, emits[1][2])
481
- assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
482
-
483
- sleep 3
484
- emits = d.emits
485
- assert(emits.length == 3)
486
483
  end
487
484
 
488
- emits = d.emits
489
- assert(emits.length == 4)
490
- assert_equal({"message1" => "test8"}, emits[3][2])
485
+ events = d.events
486
+ assert_equal(utf8_message, events[0][2]['message'])
487
+ assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
491
488
  end
492
489
 
493
- def test_multiline_with_flush_interval
494
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
495
-
496
- d = create_driver %[
497
- format multiline
498
- format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
499
- format_firstline /^[s]/
500
- multiline_flush_interval 2s
501
- ]
502
-
503
- assert_equal 2, d.instance.multiline_flush_interval
504
-
505
- d.run do
506
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
507
- f.puts "f test1"
508
- f.puts "s test2"
509
- f.puts "f test3"
510
- f.puts "f test4"
511
- f.puts "s test5"
512
- f.puts "s test6"
513
- f.puts "f test7"
514
- f.puts "s test8"
515
- }
516
- sleep 1
490
+ sub_test_case "multiline" do
491
+ data(flat: MULTILINE_CONFIG,
492
+ parse: PARSE_MULTILINE_CONFIG)
493
+ def test_multiline(data)
494
+ config = data
495
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
517
496
 
518
- emits = d.emits
519
- assert(emits.length == 3)
520
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
521
- assert_equal({"message1" => "test5"}, emits[1][2])
522
- assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
497
+ d = create_driver(config)
498
+ d.run(expect_emits: 1) do
499
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
500
+ f.puts "f test1"
501
+ f.puts "s test2"
502
+ f.puts "f test3"
503
+ f.puts "f test4"
504
+ f.puts "s test5"
505
+ f.puts "s test6"
506
+ f.puts "f test7"
507
+ f.puts "s test8"
508
+ }
509
+ end
523
510
 
524
- sleep 3
525
- emits = d.emits
526
- assert(emits.length == 4)
527
- assert_equal({"message1" => "test8"}, emits[3][2])
511
+ events = d.events
512
+ assert_equal(4, events.length)
513
+ assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
514
+ assert_equal({"message1" => "test5"}, events[1][2])
515
+ assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
516
+ assert_equal({"message1" => "test8"}, events[3][2])
528
517
  end
529
- end
530
518
 
531
- data(
532
- 'default encoding' => ['', Encoding::ASCII_8BIT],
533
- 'explicit encoding config' => ['encoding utf-8', Encoding::UTF_8])
534
- def test_multiline_encoding_of_flushed_record(data)
535
- encoding_config, encoding = data
519
+ data(flat: MULTILINE_CONFIG,
520
+ parse: PARSE_MULTILINE_CONFIG)
521
+ def test_multiline_with_flush_interval(data)
522
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
536
523
 
537
- d = create_driver %[
538
- format multiline
539
- format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
540
- format_firstline /^[s]/
541
- multiline_flush_interval 2s
542
- read_from_head true
543
- #{encoding_config}
544
- ]
524
+ config = data + config_element("", "", { "multiline_flush_interval" => "2s" })
525
+ d = create_driver(config)
545
526
 
546
- d.run do
547
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f|
548
- f.puts "s test"
549
- }
527
+ assert_equal 2, d.instance.multiline_flush_interval
550
528
 
551
- sleep 4
552
- emits = d.emits
553
- assert_equal(1, emits.length)
554
- assert_equal(encoding, emits[0][2]['message1'].encoding)
555
- end
556
- end
529
+ d.run(expect_emits: 1) do
530
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
531
+ f.puts "f test1"
532
+ f.puts "s test2"
533
+ f.puts "f test3"
534
+ f.puts "f test4"
535
+ f.puts "s test5"
536
+ f.puts "s test6"
537
+ f.puts "f test7"
538
+ f.puts "s test8"
539
+ }
540
+ end
557
541
 
558
- def test_multiline_from_encoding_of_flushed_record
559
- d = create_driver %[
560
- format multiline
561
- format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
562
- format_firstline /^[s]/
563
- multiline_flush_interval 2s
564
- read_from_head true
565
- from_encoding cp932
566
- encoding utf-8
567
- ]
542
+ events = d.events
543
+ assert_equal(4, events.length)
544
+ assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
545
+ assert_equal({"message1" => "test5"}, events[1][2])
546
+ assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
547
+ assert_equal({"message1" => "test8"}, events[3][2])
548
+ end
549
+
550
+ data(
551
+ 'flat default encoding' => [MULTILINE_CONFIG, Encoding::ASCII_8BIT],
552
+ 'flat explicit encoding config' => [MULTILINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8],
553
+ 'parse default encoding' => [PARSE_MULTILINE_CONFIG, Encoding::ASCII_8BIT],
554
+ 'parse explicit encoding config' => [PARSE_MULTILINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8])
555
+ def test_multiline_encoding_of_flushed_record(data)
556
+ encoding_config, encoding = data
557
+
558
+ config = config_element("", "", {
559
+ "multiline_flush_interval" => "2s",
560
+ "read_from_head" => "true",
561
+ })
562
+ d = create_driver(config + encoding_config)
563
+
564
+ d.run(expect_emits: 1) do
565
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f|
566
+ f.puts "s test"
567
+ }
568
+ end
569
+ events = d.events
570
+ assert_equal(1, events.length)
571
+ assert_equal(encoding, events[0][2]['message1'].encoding)
572
+ end
573
+
574
+ def test_multiline_from_encoding_of_flushed_record
575
+ conf = MULTILINE_CONFIG + config_element(
576
+ "", "",
577
+ {
578
+ "multiline_flush_interval" => "1s",
579
+ "read_from_head" => "true",
580
+ "from_encoding" => "cp932",
581
+ "encoding" => "utf-8"
582
+ })
583
+ d = create_driver(conf)
584
+
585
+ cp932_message = "s \x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
586
+ utf8_message = "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".encode(Encoding::UTF_8, Encoding::CP932)
587
+ d.run(expect_emits: 1) do
588
+ File.open("#{TMP_DIR}/tail.txt", "w:cp932") { |f|
589
+ f.puts cp932_message
590
+ }
591
+ end
568
592
 
569
- d.run do
570
- sleep 1
571
- File.open("#{TMP_DIR}/tail.txt", "w:cp932") { |f|
572
- f.puts "s \x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
573
- }
593
+ events = d.events
594
+ assert_equal(1, events.length)
595
+ assert_equal(utf8_message, events[0][2]['message1'])
596
+ assert_equal(Encoding::UTF_8, events[0][2]['message1'].encoding)
597
+ end
598
+
599
+ data(flat: config_element(
600
+ "", "", {
601
+ "format" => "multiline",
602
+ "format1" => "/^s (?<message1>[^\\n]+)\\n?/",
603
+ "format2" => "/(f (?<message2>[^\\n]+)\\n?)?/",
604
+ "format3" => "/(f (?<message3>.*))?/",
605
+ "format_firstline" => "/^[s]/"
606
+ }),
607
+ parse: config_element(
608
+ "", "", {},
609
+ [config_element("parse", "", {
610
+ "@type" => "multiline",
611
+ "format1" => "/^s (?<message1>[^\\n]+)\\n?/",
612
+ "format2" => "/(f (?<message2>[^\\n]+)\\n?)?/",
613
+ "format3" => "/(f (?<message3>.*))?/",
614
+ "format_firstline" => "/^[s]/"
615
+ })
616
+ ])
617
+ )
618
+ def test_multiline_with_multiple_formats(data)
619
+ config = data
620
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
574
621
 
575
- sleep 4
576
- emits = d.emits
577
- assert_equal(1, emits.length)
578
- assert_equal("\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932).encode(Encoding::UTF_8), emits[0][2]['message1'])
579
- assert_equal(Encoding::UTF_8, emits[0][2]['message1'].encoding)
580
- end
581
- end
622
+ d = create_driver(config)
623
+ d.run(expect_emits: 1) do
624
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
625
+ f.puts "f test1"
626
+ f.puts "s test2"
627
+ f.puts "f test3"
628
+ f.puts "f test4"
629
+ f.puts "s test5"
630
+ f.puts "s test6"
631
+ f.puts "f test7"
632
+ f.puts "s test8"
633
+ }
634
+ end
582
635
 
583
- def test_multiline_with_multiple_formats
584
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
636
+ events = d.events
637
+ assert(events.length > 0)
638
+ assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
639
+ assert_equal({"message1" => "test5"}, events[1][2])
640
+ assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
641
+ assert_equal({"message1" => "test8"}, events[3][2])
642
+ end
643
+
644
+ data(flat: config_element(
645
+ "", "", {
646
+ "format" => "multiline",
647
+ "format1" => "/^[s|f] (?<message>.*)/",
648
+ "format_firstline" => "/^[s]/"
649
+ }),
650
+ parse: config_element(
651
+ "", "", {},
652
+ [config_element("parse", "", {
653
+ "@type" => "multiline",
654
+ "format1" => "/^[s|f] (?<message>.*)/",
655
+ "format_firstline" => "/^[s]/"
656
+ })
657
+ ])
658
+ )
659
+ def test_multilinelog_with_multiple_paths(data)
660
+ files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
661
+ files.each { |file| File.open(file, "wb") { |f| } }
585
662
 
586
- d = create_driver %[
587
- format multiline
588
- format1 /^s (?<message1>[^\\n]+)\\n?/
589
- format2 /(f (?<message2>[^\\n]+)\\n?)?/
590
- format3 /(f (?<message3>.*))?/
591
- format_firstline /^[s]/
592
- ]
593
- d.run do
594
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
595
- f.puts "f test1"
596
- f.puts "s test2"
597
- f.puts "f test3"
598
- f.puts "f test4"
599
- f.puts "s test5"
600
- f.puts "s test6"
601
- f.puts "f test7"
602
- f.puts "s test8"
603
- }
604
- sleep 1
605
- end
663
+ config = data + config_element("", "", {
664
+ "path" => "#{files[0]},#{files[1]}",
665
+ "tag" => "t1",
666
+ })
667
+ d = create_driver(config, false)
668
+ d.run(expect_emits: 2) do
669
+ files.each do |file|
670
+ File.open(file, 'ab') { |f|
671
+ f.puts "f #{file} line should be ignored"
672
+ f.puts "s test1"
673
+ f.puts "f test2"
674
+ f.puts "f test3"
675
+ f.puts "s test4"
676
+ }
677
+ end
678
+ end
606
679
 
607
- emits = d.emits
608
- assert(emits.length > 0)
609
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
610
- assert_equal({"message1" => "test5"}, emits[1][2])
611
- assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
612
- assert_equal({"message1" => "test8"}, emits[3][2])
613
- end
680
+ events = d.events
681
+ assert_equal({"message" => "test1\nf test2\nf test3"}, events[0][2])
682
+ assert_equal({"message" => "test1\nf test2\nf test3"}, events[1][2])
683
+ # "test4" events are here because these events are flushed at shutdown phase
684
+ assert_equal({"message" => "test4"}, events[2][2])
685
+ assert_equal({"message" => "test4"}, events[3][2])
686
+ end
687
+
688
+ data(flat: config_element("", "", {
689
+ "format" => "multiline",
690
+ "format1" => "/(?<var1>foo \\d)\\n/",
691
+ "format2" => "/(?<var2>bar \\d)\\n/",
692
+ "format3" => "/(?<var3>baz \\d)/"
693
+ }),
694
+ parse: config_element(
695
+ "", "", {},
696
+ [config_element("parse", "", {
697
+ "@type" => "multiline",
698
+ "format1" => "/(?<var1>foo \\d)\\n/",
699
+ "format2" => "/(?<var2>bar \\d)\\n/",
700
+ "format3" => "/(?<var3>baz \\d)/"
701
+ })
702
+ ])
703
+ )
704
+ def test_multiline_without_firstline(data)
705
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
614
706
 
615
- def test_multilinelog_with_multiple_paths
616
- files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
617
- files.each { |file| File.open(file, "wb") { |f| } }
618
-
619
- d = create_driver(%[
620
- path #{files[0]},#{files[1]}
621
- tag t1
622
- format multiline
623
- format1 /^[s|f] (?<message>.*)/
624
- format_firstline /^[s]/
625
- ], false)
626
- d.run do
627
- files.each do |file|
628
- File.open(file, 'ab') { |f|
629
- f.puts "f #{file} line should be ignored"
630
- f.puts "s test1"
631
- f.puts "f test2"
632
- f.puts "f test3"
633
- f.puts "s test4"
707
+ config = data
708
+ d = create_driver(config)
709
+ d.run(expect_emits: 1) do
710
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
711
+ f.puts "foo 1"
712
+ f.puts "bar 1"
713
+ f.puts "baz 1"
714
+ f.puts "foo 2"
715
+ f.puts "bar 2"
716
+ f.puts "baz 2"
634
717
  }
635
718
  end
636
- sleep 1
637
- end
638
719
 
639
- emits = d.emits
640
- assert_equal({"message" => "test1\nf test2\nf test3"}, emits[0][2])
641
- assert_equal({"message" => "test1\nf test2\nf test3"}, emits[1][2])
642
- # "test4" events are here because these events are flushed at shutdown phase
643
- assert_equal({"message" => "test4"}, emits[2][2])
644
- assert_equal({"message" => "test4"}, emits[3][2])
645
- end
646
-
647
- def test_multiline_without_firstline
648
- File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
649
-
650
- d = create_driver %[
651
- format multiline
652
- format1 /(?<var1>foo \\d)\\n/
653
- format2 /(?<var2>bar \\d)\\n/
654
- format3 /(?<var3>baz \\d)/
720
+ events = d.events
721
+ assert_equal(2, events.length)
722
+ assert_equal({"var1" => "foo 1", "var2" => "bar 1", "var3" => "baz 1"}, events[0][2])
723
+ assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, events[1][2])
724
+ end
725
+ end
726
+
727
+ sub_test_case "path" do
728
+ # * path test
729
+ # TODO: Clean up tests
730
+ EX_RORATE_WAIT = 0
731
+
732
+ EX_CONFIG = config_element("", "", {
733
+ "tag" => "tail",
734
+ "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
735
+ "format" => "none",
736
+ "pos_file" => "#{TMP_DIR}/tail.pos",
737
+ "read_from_head" => true,
738
+ "refresh_interval" => 30,
739
+ "rotate_wait" => "#{EX_RORATE_WAIT}s",
740
+ })
741
+ EX_PATHS = [
742
+ 'test/plugin/data/2010/01/20100102-030405.log',
743
+ 'test/plugin/data/log/foo/bar.log',
744
+ 'test/plugin/data/log/test.log'
655
745
  ]
656
- d.run do
657
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
658
- f.puts "foo 1"
659
- f.puts "bar 1"
660
- f.puts "baz 1"
661
- f.puts "foo 2"
662
- f.puts "bar 2"
663
- f.puts "baz 2"
664
- }
665
- sleep 1
666
- end
667
746
 
668
- emits = d.emits
669
- assert_equal(2, emits.length)
670
- assert_equal({"var1" => "foo 1", "var2" => "bar 1", "var3" => "baz 1"}, emits[0][2])
671
- assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, emits[1][2])
672
- end
747
+ def test_expand_paths
748
+ plugin = create_driver(EX_CONFIG, false).instance
749
+ flexstub(Time) do |timeclass|
750
+ timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
751
+ assert_equal EX_PATHS, plugin.expand_paths.sort
752
+ end
673
753
 
674
- # * path test
675
- # TODO: Clean up tests
676
- EX_RORATE_WAIT = 0
677
-
678
- EX_CONFIG = %[
679
- tag tail
680
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
681
- format none
682
- pos_file #{TMP_DIR}/tail.pos
683
- read_from_head true
684
- refresh_interval 30
685
- rotate_wait #{EX_RORATE_WAIT}s
686
- ]
687
- EX_PATHS = [
688
- 'test/plugin/data/2010/01/20100102-030405.log',
689
- 'test/plugin/data/log/foo/bar.log',
690
- 'test/plugin/data/log/test.log'
691
- ]
692
-
693
- def test_expand_paths
694
- plugin = create_driver(EX_CONFIG, false).instance
695
- flexstub(Time) do |timeclass|
696
- timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
697
- assert_equal EX_PATHS, plugin.expand_paths.sort
754
+ # Test exclusion
755
+ exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{EX_PATHS.last}"]) })
756
+ plugin = create_driver(exclude_config, false).instance
757
+ assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
698
758
  end
699
-
700
- # Test exclusion
701
- exclude_config = EX_CONFIG + " exclude_path [\"#{EX_PATHS.last}\"]"
702
- plugin = create_driver(exclude_config, false).instance
703
- assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
704
759
  end
705
760
 
706
761
  def test_z_refresh_watchers
707
762
  plugin = create_driver(EX_CONFIG, false).instance
708
763
  sio = StringIO.new
709
764
  plugin.instance_eval do
710
- @pf = Fluent::NewTailInput::PositionFile.parse(sio)
765
+ @pf = Fluent::Plugin::TailInput::PositionFile.parse(sio)
711
766
  @loop = Coolio::Loop.new
712
767
  end
713
768
 
714
769
  flexstub(Time) do |timeclass|
715
770
  timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5), Time.new(2010, 1, 2, 3, 4, 6), Time.new(2010, 1, 2, 3, 4, 7))
716
771
 
717
- flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
772
+ flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
718
773
  EX_PATHS.each do |path|
719
- watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
774
+ watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
720
775
  flexmock('TailWatcher') { |watcher|
721
776
  watcher.should_receive(:attach).once
722
777
  watcher.should_receive(:unwatched=).zero_or_more_times
@@ -731,8 +786,8 @@ class TailInputTest < Test::Unit::TestCase
731
786
  @tails['test/plugin/data/2010/01/20100102-030405.log'].should_receive(:close).zero_or_more_times
732
787
  end
733
788
 
734
- flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
735
- watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
789
+ flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
790
+ watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
736
791
  flexmock('TailWatcher') do |watcher|
737
792
  watcher.should_receive(:attach).once
738
793
  watcher.should_receive(:unwatched=).zero_or_more_times
@@ -742,67 +797,77 @@ class TailInputTest < Test::Unit::TestCase
742
797
  plugin.refresh_watchers
743
798
  end
744
799
 
745
- flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
800
+ flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
746
801
  watcherclass.should_receive(:new).never
747
802
  plugin.refresh_watchers
748
803
  end
749
804
  end
750
805
  end
751
806
 
752
- DummyWatcher = Struct.new("DummyWatcher", :tag)
807
+ sub_test_case "receive_lines" do
808
+ DummyWatcher = Struct.new("DummyWatcher", :tag)
753
809
 
754
- def test_receive_lines
755
- plugin = create_driver(EX_CONFIG, false).instance
756
- flexstub(plugin.router) do |engineclass|
757
- engineclass.should_receive(:emit_stream).with('tail', any).once
810
+ def test_tag
811
+ d = create_driver(EX_CONFIG, false)
812
+ d.run {}
813
+ plugin = d.instance
814
+ mock(plugin.router).emit_stream('tail', anything).once
758
815
  plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
759
816
  end
760
817
 
761
- config = %[
762
- tag pre.*
763
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
764
- format none
765
- read_from_head true
766
- ]
767
- plugin = create_driver(config, false).instance
768
- flexstub(plugin.router) do |engineclass|
769
- engineclass.should_receive(:emit_stream).with('pre.foo.bar.log', any).once
818
+ def test_tag_prefix
819
+ config = config_element("", "", {
820
+ "tag" => "pre.*",
821
+ "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
822
+ "format" => "none",
823
+ "read_from_head" => true
824
+ })
825
+ d = create_driver(config, false)
826
+ d.run {}
827
+ plugin = d.instance
828
+ mock(plugin.router).emit_stream('pre.foo.bar.log', anything).once
770
829
  plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
771
830
  end
772
831
 
773
- config = %[
774
- tag *.post
775
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
776
- format none
777
- read_from_head true
778
- ]
779
- plugin = create_driver(config, false).instance
780
- flexstub(plugin.router) do |engineclass|
781
- engineclass.should_receive(:emit_stream).with('foo.bar.log.post', any).once
832
+ def test_tag_suffix
833
+ config = config_element("", "", {
834
+ "tag" => "*.post",
835
+ "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
836
+ "format" => "none",
837
+ "read_from_head" => true
838
+ })
839
+ d = create_driver(config, false)
840
+ d.run {}
841
+ plugin = d.instance
842
+ mock(plugin.router).emit_stream('foo.bar.log.post', anything).once
782
843
  plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
783
844
  end
784
845
 
785
- config = %[
786
- tag pre.*.post
787
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
788
- format none
789
- read_from_head true
790
- ]
791
- plugin = create_driver(config, false).instance
792
- flexstub(plugin.router) do |engineclass|
793
- engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
846
+ def test_tag_prefix_and_suffix
847
+ config = config_element("", "", {
848
+ "tag" => "pre.*.post",
849
+ "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
850
+ "format" => "none",
851
+ "read_from_head" => true
852
+ })
853
+ d = create_driver(config, false)
854
+ d.run {}
855
+ plugin = d.instance
856
+ mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once
794
857
  plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
795
858
  end
796
859
 
797
- config = %[
798
- tag pre.*.post*ignore
799
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
800
- format none
801
- read_from_head true
802
- ]
803
- plugin = create_driver(config, false).instance
804
- flexstub(plugin.router) do |engineclass|
805
- engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
860
+ def test_tag_prefix_and_suffix_ignore
861
+ config = config_element("", "", {
862
+ "tag" => "pre.*.post*ignore",
863
+ "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
864
+ "format" => "none",
865
+ "read_from_head" => true
866
+ })
867
+ d = create_driver(config, false)
868
+ d.run {}
869
+ plugin = d.instance
870
+ mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once
806
871
  plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
807
872
  end
808
873
  end
@@ -817,70 +882,26 @@ class TailInputTest < Test::Unit::TestCase
817
882
 
818
883
  # Try two different configs - one with read_from_head and one without,
819
884
  # since their interactions with the filesystem differ.
820
- config1 = %[
821
- tag t1
822
- path #{TMP_DIR}/non_existent_file.txt,#{TMP_DIR}/tail.txt
823
- format none
824
- rotate_wait 2s
825
- pos_file #{TMP_DIR}/tail.pos
826
- ]
827
- config2 = config1 + ' read_from_head true'
885
+ config1 = config_element("", "", {
886
+ "tag" => "t1",
887
+ "path" => "#{TMP_DIR}/non_existent_file.txt,#{TMP_DIR}/tail.txt",
888
+ "format" => "none",
889
+ "rotate_wait" => "2s",
890
+ "pos_file" => "#{TMP_DIR}/tail.pos"
891
+ })
892
+ config2 = config1 + config_element("", "", { "read_from_head" => true })
828
893
  [config1, config2].each do |config|
829
894
  d = create_driver(config, false)
830
- d.run do
831
- sleep 1
895
+ d.run(expect_emits: 1) do
832
896
  File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
833
897
  f.puts "test3"
834
898
  f.puts "test4"
835
899
  }
836
- sleep 1
837
900
  end
838
- emits = d.emits
839
- assert_equal(2, emits.length)
840
- assert_equal({"message" => "test3"}, emits[0][2])
841
- assert_equal({"message" => "test4"}, emits[1][2])
842
- end
843
- end
844
-
845
- sub_test_case 'emit error cases' do
846
- def test_emit_error_with_buffer_queue_limit_error
847
- emits = execute_test(Fluent::Plugin::Buffer::BufferOverflowError, "buffer space has too many data")
848
- assert_equal(10, emits.length)
849
- 10.times { |i|
850
- assert_equal({"message" => "test#{i}"}, emits[i][2])
851
- }
852
- end
853
-
854
- def test_emit_error_with_non_buffer_queue_limit_error
855
- emits = execute_test(StandardError, "non BufferQueueLimitError error")
856
- assert_true(emits.size > 0 && emits.size != 10)
857
- emits.size.times { |i|
858
- assert_equal({"message" => "test#{10 - emits.size + i}"}, emits[i][2])
859
- }
860
- end
861
-
862
- def execute_test(error_class, error_message)
863
- d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
864
- # Use define_singleton_method instead of d.emit_stream to capture local variable
865
- d.define_singleton_method(:emit_stream) do |tag, es|
866
- @test_num_errors ||= 0
867
- if @test_num_errors < 5
868
- @test_num_errors += 1
869
- raise error_class, error_message
870
- else
871
- @emit_streams << [tag, es.to_a]
872
- end
873
- end
874
-
875
- d.run do
876
- 10.times { |i|
877
- File.open("#{TMP_DIR}/tail.txt", "ab") { |f| f.puts "test#{i}" }
878
- sleep 0.5
879
- }
880
- sleep 1
881
- end
882
-
883
- d.emits
901
+ events = d.events
902
+ assert_equal(2, events.length)
903
+ assert_equal({"message" => "test3"}, events[0][2])
904
+ assert_equal({"message" => "test4"}, events[1][2])
884
905
  end
885
906
  end
886
907
 
@@ -891,21 +912,18 @@ class TailInputTest < Test::Unit::TestCase
891
912
  f.puts "test2"
892
913
  }
893
914
 
894
- d = create_driver(%[path_key path] + SINGLE_LINE_CONFIG)
895
-
896
- d.run do
897
- sleep 1
915
+ d = create_driver(SINGLE_LINE_CONFIG + config_element("", "", { "path_key" => "path" }))
898
916
 
917
+ d.run(expect_emits: 1) do
899
918
  File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
900
919
  f.puts "test3"
901
920
  f.puts "test4"
902
921
  }
903
- sleep 1
904
922
  end
905
923
 
906
- emits = d.emits
907
- assert_equal(true, emits.length > 0)
908
- emits.each do |emit|
924
+ events = d.events
925
+ assert_equal(true, events.length > 0)
926
+ events.each do |emit|
909
927
  assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
910
928
  end
911
929
  end
@@ -913,13 +931,14 @@ class TailInputTest < Test::Unit::TestCase
913
931
  def test_tail_path_with_multiline_with_firstline
914
932
  File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
915
933
 
916
- d = create_driver %[
917
- path_key path
918
- format multiline
919
- format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
920
- format_firstline /^[s]/
921
- ]
922
- d.run do
934
+ config = config_element("", "", {
935
+ "path_key" => "path",
936
+ "format" => "multiline",
937
+ "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
938
+ "format_firstline" => "/^[s]/"
939
+ })
940
+ d = create_driver(config)
941
+ d.run(expect_emits: 1) do
923
942
  File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
924
943
  f.puts "f test1"
925
944
  f.puts "s test2"
@@ -930,12 +949,11 @@ class TailInputTest < Test::Unit::TestCase
930
949
  f.puts "f test7"
931
950
  f.puts "s test8"
932
951
  }
933
- sleep 1
934
952
  end
935
953
 
936
- emits = d.emits
937
- assert(emits.length == 4)
938
- emits.each do |emit|
954
+ events = d.events
955
+ assert_equal(4, events.length)
956
+ events.each do |emit|
939
957
  assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
940
958
  end
941
959
  end
@@ -943,25 +961,25 @@ class TailInputTest < Test::Unit::TestCase
943
961
  def test_tail_path_with_multiline_without_firstline
944
962
  File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
945
963
 
946
- d = create_driver %[
947
- path_key path
948
- format multiline
949
- format1 /(?<var1>foo \\d)\\n/
950
- format2 /(?<var2>bar \\d)\\n/
951
- format3 /(?<var3>baz \\d)/
952
- ]
953
- d.run do
964
+ config = config_element("", "", {
965
+ "path_key" => "path",
966
+ "format" => "multiline",
967
+ "format1" => "/(?<var1>foo \\d)\\n/",
968
+ "format2" => "/(?<var2>bar \\d)\\n/",
969
+ "format3" => "/(?<var3>baz \\d)/",
970
+ })
971
+ d = create_driver(config)
972
+ d.run(expect_emits: 1) do
954
973
  File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
955
974
  f.puts "foo 1"
956
975
  f.puts "bar 1"
957
976
  f.puts "baz 1"
958
977
  }
959
- sleep 1
960
978
  end
961
979
 
962
- emits = d.emits
963
- assert(emits.length > 0)
964
- emits.each do |emit|
980
+ events = d.events
981
+ assert(events.length > 0)
982
+ events.each do |emit|
965
983
  assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
966
984
  end
967
985
  end
@@ -970,15 +988,16 @@ class TailInputTest < Test::Unit::TestCase
970
988
  files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
971
989
  files.each { |file| File.open(file, "wb") { |f| } }
972
990
 
973
- d = create_driver(%[
974
- path #{files[0]},#{files[1]}
975
- path_key path
976
- tag t1
977
- format multiline
978
- format1 /^[s|f] (?<message>.*)/
979
- format_firstline /^[s]/
980
- ], false)
981
- d.run do
991
+ config = config_element("", "", {
992
+ "path" => "#{files[0]},#{files[1]}",
993
+ "path_key" => "path",
994
+ "tag" => "t1",
995
+ "format" => "multiline",
996
+ "format1" => "/^[s|f] (?<message>.*)/",
997
+ "format_firstline" => "/^[s]/"
998
+ })
999
+ d = create_driver(config, false)
1000
+ d.run(expect_emits: 2) do
982
1001
  files.each do |file|
983
1002
  File.open(file, 'ab') { |f|
984
1003
  f.puts "f #{file} line should be ignored"
@@ -988,14 +1007,13 @@ class TailInputTest < Test::Unit::TestCase
988
1007
  f.puts "s test4"
989
1008
  }
990
1009
  end
991
- sleep 1
992
1010
  end
993
1011
 
994
- emits = d.emits
995
- assert(emits.length == 4)
996
- assert_equal(files, [emits[0][2]["path"], emits[1][2]["path"]].sort)
1012
+ events = d.events
1013
+ assert_equal(4, events.length)
1014
+ assert_equal(files, [events[0][2]["path"], events[1][2]["path"]].sort)
997
1015
  # "test4" events are here because these events are flushed at shutdown phase
998
- assert_equal(files, [emits[2][2]["path"], emits[3][2]["path"]].sort)
1016
+ assert_equal(files, [events[2][2]["path"], events[3][2]["path"]].sort)
999
1017
  end
1000
1018
  end
1001
1019
  end