fluent-plugin-tail-ex-rotate 0.0.4 → 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 41366d67f88d73008c13f8ccaba644c8a0275015
4
- data.tar.gz: fe145dc47313f4198d4d454570f2a94c99f49484
3
+ metadata.gz: d39dbd1566b0cf365977a8c7ddd278fae06d2613
4
+ data.tar.gz: '0459bc883918a34886c77d9e336b04d3baf06c6c'
5
5
  SHA512:
6
- metadata.gz: e9b342459226d0778124e4c027f982b4ee7b915f8d986f76f94c2d9eb292e275b8b667fd7bcce8d421a62c87c5b9250d5c1d3cdb56becc0ceec20e81118ac489
7
- data.tar.gz: 1e5855ef0238b44210cb1c20ee24fe0ead7653bde7052a0f06c770e462e432ed110b2562d9ce320471013586b7480c7b2df78d1f99a0d3f8dbb5e28b7e467e45
6
+ metadata.gz: 41109bee6e6ee72fe72080fe9746d386cb9d5d050395659ed6882b006b937e8c6dc26bf5333bf965ebf80c5afc6184224276b886da2e1a095556929b9706ae9c
7
+ data.tar.gz: d712a9cefcda6f269c86047afcd0f71e641f4ebd403c141594d33b7ce4ef886875cab1945cd8b29728805c06e7364abca334a5acd7d886994acf9038735151dd
@@ -0,0 +1 @@
1
+ test/tmp
@@ -1,3 +1,3 @@
1
1
  language: ruby
2
2
  rvm:
3
- - 1.9.3
3
+ - 2.4
data/COPYING CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (C) 2015 DWANGO Co., Ltd.
1
+ Copyright Yuta Mizushima.
2
2
 
3
3
  Licensed under the Apache License, Version 2.0 (the "License");
4
4
  you may not use this file except in compliance with the License.
data/Gemfile CHANGED
@@ -1,3 +1,4 @@
1
1
  source 'https://rubygems.org/'
2
-
2
+ gem "test-unit", "3.2.5"
3
+ gem "timecop"
3
4
  gemspec
@@ -0,0 +1,57 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ fluent-plugin-tail-ex-rotate (0.1.0)
5
+ fluentd (>= 0.14.19)
6
+
7
+ GEM
8
+ remote: https://rubygems.org/
9
+ specs:
10
+ cool.io (1.5.0)
11
+ flexmock (2.3.5)
12
+ fluentd (0.14.19)
13
+ cool.io (>= 1.4.5, < 2.0.0)
14
+ http_parser.rb (>= 0.5.1, < 0.7.0)
15
+ msgpack (>= 0.7.0, < 2.0.0)
16
+ serverengine (>= 2.0.4, < 3.0.0)
17
+ sigdump (~> 0.2.2)
18
+ strptime (~> 0.1.7)
19
+ tzinfo (~> 1.0)
20
+ tzinfo-data (~> 1.0)
21
+ yajl-ruby (~> 1.0)
22
+ http_parser.rb (0.6.0)
23
+ msgpack (1.1.0)
24
+ power_assert (1.0.2)
25
+ rake (12.0.0)
26
+ rr (1.2.0)
27
+ serverengine (2.0.5)
28
+ sigdump (~> 0.2.2)
29
+ sigdump (0.2.4)
30
+ strptime (0.1.9)
31
+ test-unit (3.2.5)
32
+ power_assert
33
+ test-unit-rr (1.0.5)
34
+ rr (>= 1.1.1)
35
+ test-unit (>= 2.5.2)
36
+ thread_safe (0.3.6)
37
+ timecop (0.9.1)
38
+ tzinfo (1.2.3)
39
+ thread_safe (~> 0.1)
40
+ tzinfo-data (1.2017.2)
41
+ tzinfo (>= 1.0.0)
42
+ yajl-ruby (1.3.0)
43
+
44
+ PLATFORMS
45
+ ruby
46
+
47
+ DEPENDENCIES
48
+ bundler (~> 1.3)
49
+ flexmock
50
+ fluent-plugin-tail-ex-rotate!
51
+ rake (>= 0.9.2)
52
+ test-unit (= 3.2.5)
53
+ test-unit-rr
54
+ timecop
55
+
56
+ BUNDLED WITH
57
+ 1.14.6
data/README.md CHANGED
@@ -1,5 +1,5 @@
1
1
  # fluent-plugin-tail-ex-rotate
2
- [![Build Status](https://travis-ci.org/dwango/fluent-plugin-tail-ex-rotate.png?branch=master)](https://travis-ci.org/dwango/fluent-plugin-tail-ex-rotate)
2
+ [![Build Status](https://travis-ci.org/ymizushi/fluent-plugin-tail-ex-rotate.png?branch=master)](https://travis-ci.org/ymizushi/fluent-plugin-tail-ex-rotate)
3
3
 
4
4
  fluent-plugin-tail-ex-rotate is a fluentd plugin to delay file lotation time.
5
5
 
@@ -2,13 +2,14 @@ require File.expand_path('../lib/fluent/version', __FILE__)
2
2
 
3
3
  Gem::Specification.new do |gem|
4
4
  gem.name = "fluent-plugin-tail-ex-rotate"
5
- gem.version = "0.0.4"
5
+ gem.version = "0.1.0"
6
6
 
7
7
  gem.authors = ["Yuta Mizushima"]
8
- gem.email = ["yuta_mizushima@dwango.co.jp"]
8
+ gem.licenses = ["MIT"]
9
+ gem.email = ["mizushima@gmail.com"]
9
10
  gem.description = %q{Extension of in_tail plugin to customize log rotate timing.}
10
11
  gem.summary = %q{customize log rotate timing}
11
- gem.homepage = "https://github.com/dwango/fluent-plugin-tail-ex-rotate"
12
+ gem.homepage = "https://github.com/ymizushi/fluent-plugin-tail-ex-rotate"
12
13
 
13
14
  gem.files = `git ls-files`.split($\)
14
15
  gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -16,12 +17,12 @@ Gem::Specification.new do |gem|
16
17
  gem.require_paths = ["lib"]
17
18
  gem.has_rdoc = false
18
19
 
19
- gem.required_ruby_version = '>= 1.9.3'
20
+ gem.required_ruby_version = '~> 2.1'
20
21
 
21
- gem.add_runtime_dependency("fluentd", [">= 0.12.16"])
22
+ gem.add_runtime_dependency("fluentd", ["~> 0.14"])
22
23
 
23
24
  gem.add_development_dependency("bundler", "~> 1.3")
24
- gem.add_development_dependency("test-unit-rr")
25
- gem.add_development_dependency("flexmock")
26
- gem.add_development_dependency("rake", [">= 0.9.2"])
25
+ gem.add_development_dependency("test-unit-rr", "~> 0")
26
+ gem.add_development_dependency("flexmock", "~> 0")
27
+ gem.add_development_dependency("rake", ["~> 0.9"])
27
28
  end
@@ -1,7 +1,19 @@
1
- require 'fluent/plugin/in_tail'
1
+ require 'cool.io'
2
2
 
3
- module Fluent
4
- class TailExRotateInput < Fluent::NewTailInput
3
+ require 'fluent/plugin/input'
4
+ require 'fluent/config/error'
5
+ require 'fluent/event'
6
+ require 'fluent/plugin/buffer'
7
+ require 'fluent/plugin/parser_multiline'
8
+
9
+ if Fluent.windows?
10
+ require_relative 'file_wrapper'
11
+ else
12
+ Fluent::FileWrapper = File
13
+ end
14
+
15
+ module Fluent::Plugin
16
+ class TailExRotateInput < Fluent::Plugin::TailInput
5
17
  Fluent::Plugin.register_input('tail_ex_rotate', self)
6
18
  config_param :expand_rotate_time, :time, :default => 0
7
19
 
@@ -9,15 +21,24 @@ module Fluent
9
21
  date = Time.now - @expand_rotate_time
10
22
  paths = []
11
23
 
12
- excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
13
24
  @paths.each { |path|
14
25
  path = date.strftime(path)
15
26
  if path.include?('*')
16
27
  paths += Dir.glob(path).select { |p|
17
- if File.readable?(p)
18
- true
28
+ is_file = !File.directory?(p)
29
+ if File.readable?(p) && is_file
30
+ if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
31
+ false
32
+ else
33
+ true
34
+ end
19
35
  else
20
- log.warn "#{p} unreadable. It is excluded and would be examined next time."
36
+ if is_file
37
+ unless @ignore_list.include?(path)
38
+ log.warn "#{p} unreadable. It is excluded and would be examined next time."
39
+ @ignore_list << path if @ignore_repeated_permission_error
40
+ end
41
+ end
21
42
  false
22
43
  end
23
44
  }
@@ -26,6 +47,7 @@ module Fluent
26
47
  paths << path
27
48
  end
28
49
  }
50
+ excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
29
51
  paths - excluded
30
52
  end
31
53
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '0.12.2'
19
+ VERSION = '0.14.19'
20
20
 
21
21
  end
@@ -1,4 +1,4 @@
1
- # simplecov must be loaded before any of target code is loaded
1
+ # simplecov must be loaded before any of target code
2
2
  if ENV['SIMPLE_COV']
3
3
  require 'simplecov'
4
4
  if defined?(SimpleCov::SourceFile)
@@ -23,23 +23,116 @@ if ENV['SIMPLE_COV']
23
23
  end
24
24
  end
25
25
 
26
+ # Some tests use Hash instead of Element for configure.
27
+ # We should rewrite these tests in the future and remove this ad-hoc code
28
+ class Hash
29
+ def corresponding_proxies
30
+ @corresponding_proxies ||= []
31
+ end
32
+
33
+ def to_masked_element
34
+ self
35
+ end
36
+ end
37
+
26
38
  require 'rr'
27
39
  require 'test/unit'
28
40
  require 'test/unit/rr'
29
41
  require 'fileutils'
42
+ require 'fluent/config/element'
30
43
  require 'fluent/log'
31
44
  require 'fluent/test'
45
+ require 'fluent/test/helpers'
46
+ require 'fluent/plugin/base'
47
+ require 'fluent/plugin_id'
48
+ require 'fluent/plugin_helper'
49
+ require 'fluent/msgpack_factory'
50
+ require 'fluent/time'
51
+ require 'serverengine'
52
+
53
+ module Fluent
54
+ module Plugin
55
+ class TestBase < Base
56
+ # a base plugin class, but not input nor output
57
+ # mainly for helpers and owned plugins
58
+ include PluginId
59
+ include PluginLoggerMixin
60
+ include PluginHelper::Mixin
61
+ end
62
+ end
63
+ end
32
64
 
33
65
  unless defined?(Test::Unit::AssertionFailedError)
34
66
  class Test::Unit::AssertionFailedError < StandardError
35
67
  end
36
68
  end
37
69
 
38
- def unused_port
39
- s = TCPServer.open(0)
40
- port = s.addr[1]
41
- s.close
42
- port
70
+ include Fluent::Test::Helpers
71
+
72
+ def unused_port(num = 1, protocol: :tcp, bind: "0.0.0.0")
73
+ case protocol
74
+ when :tcp
75
+ unused_port_tcp(num)
76
+ when :udp
77
+ unused_port_udp(num, bind: bind)
78
+ else
79
+ raise ArgumentError, "unknown protocol: #{protocol}"
80
+ end
81
+ end
82
+
83
+ def unused_port_tcp(num = 1)
84
+ ports = []
85
+ sockets = []
86
+ num.times do
87
+ s = TCPServer.open(0)
88
+ sockets << s
89
+ ports << s.addr[1]
90
+ end
91
+ sockets.each{|s| s.close }
92
+ if num == 1
93
+ return ports.first
94
+ else
95
+ return *ports
96
+ end
97
+ end
98
+
99
+ PORT_RANGE_AVAILABLE = (1024...65535)
100
+
101
+ def unused_port_udp(num = 1, bind: "0.0.0.0")
102
+ family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
103
+ ports = []
104
+ sockets = []
105
+ while ports.size < num
106
+ port = rand(PORT_RANGE_AVAILABLE)
107
+ u = UDPSocket.new(family)
108
+ if (u.bind(bind, port) rescue nil)
109
+ ports << port
110
+ sockets << u
111
+ else
112
+ u.close
113
+ end
114
+ end
115
+ sockets.each{|s| s.close }
116
+ if num == 1
117
+ return ports.first
118
+ else
119
+ return *ports
120
+ end
121
+ end
122
+
123
+ def waiting(seconds, logs: nil, plugin: nil)
124
+ begin
125
+ Timeout.timeout(seconds) do
126
+ yield
127
+ end
128
+ rescue Timeout::Error
129
+ if logs
130
+ STDERR.print(*logs)
131
+ elsif plugin
132
+ STDERR.print(*plugin.log.out.logs)
133
+ end
134
+ raise
135
+ end
43
136
  end
44
137
 
45
138
  def ipv6_enabled?
@@ -53,4 +146,8 @@ def ipv6_enabled?
53
146
  end
54
147
  end
55
148
 
56
- $log = Fluent::Log.new(Fluent::Test::DummyLogDevice.new, Fluent::Log::LEVEL_WARN)
149
+ dl_opts = {}
150
+ dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
151
+ logdev = Fluent::Test::DummyLogDevice.new
152
+ logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
153
+ $log ||= Fluent::Log.new(logger)
File without changes
@@ -1,291 +1,479 @@
1
1
  require_relative '../helper'
2
- require 'fluent/test'
2
+ require 'fluent/test/driver/input'
3
+ require 'fluent/plugin/in_tail'
4
+ require 'fluent/plugin/in_tail_ex_rotate'
5
+ require 'fluent/plugin/buffer'
6
+ require 'fluent/system_config'
3
7
  require 'net/http'
4
- require 'flexmock'
5
8
  require 'flexmock/test_unit'
6
- require 'fluent/plugin/in_tail_ex_rotate'
9
+ require 'timecop'
7
10
 
8
11
  class TailExRotateInputTest < Test::Unit::TestCase
9
12
  include FlexMock::TestCase
10
13
 
11
14
  def setup
12
15
  Fluent::Test.setup
13
- FileUtils.rm_rf(TMP_DIR)
14
- FileUtils.mkdir_p(TMP_DIR)
16
+ cleanup_directory(TMP_DIR)
17
+ end
18
+
19
+ def teardown
20
+ super
21
+ Fluent::Engine.stop
22
+ end
23
+
24
+ def cleanup_directory(path)
25
+ FileUtils.rm_rf(path, secure: true)
26
+ if File.exist?(path)
27
+ # ensure files are closed for Windows, on which deleted files
28
+ # are still visible from filesystem
29
+ GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
30
+ FileUtils.remove_entry_secure(path, true)
31
+ end
32
+ FileUtils.mkdir_p(path)
15
33
  end
16
34
 
17
35
  TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
18
36
 
19
- CONFIG = %[
20
- path #{TMP_DIR}/tail.txt
21
- tag t1
22
- rotate_wait 2s
23
- expand_rotate_time 18000s
24
- ]
25
- COMMON_CONFIG = CONFIG + %[
26
- pos_file #{TMP_DIR}/tail.pos
27
- ]
28
- CONFIG_READ_FROM_HEAD = %[
29
- read_from_head true
30
- ]
31
- SINGLE_LINE_CONFIG = %[
32
- format /(?<message>.*)/
33
- ]
34
-
35
- # * path test
36
- # TODO: Clean up tests
37
- EX_RORATE_WAIT = 0
38
-
39
- EX_CONFIG = %[
40
- tag tail
41
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
42
- format none
43
- pos_file #{TMP_DIR}/tail.pos
44
- read_from_head true
45
- refresh_interval 30
46
- rotate_wait #{EX_RORATE_WAIT}s
47
- ]
48
- EX_PATHS = [
49
- 'test/plugin/data/2010/01/20100102-030405.log',
50
- 'test/plugin/data/log/foo/bar.log',
51
- 'test/plugin/data/log/test.log'
52
- ]
53
-
54
-
55
- EX_CONFIG_FOR_ROTATE = %[
56
- tag tail
57
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
58
- format none
59
- pos_file #{TMP_DIR}/tail.pos
60
- read_from_head true
61
- refresh_interval 30
62
- rotate_wait #{EX_RORATE_WAIT}s
63
- expand_rotate_time 18000s
64
- ]
65
-
66
- EX_PATHS_FOR_ROATE = [
67
- 'test/plugin/data/2014/12/20141224-000000.log',
68
- 'test/plugin/data/log/foo/bar.log',
69
- 'test/plugin/data/log/test.log'
70
- ]
37
+ CONFIG = config_element("ROOT", "", {
38
+ "path" => "#{TMP_DIR}/tail.txt",
39
+ "tag" => "t1",
40
+ "rotate_wait" => "2s",
41
+ "expand_rotate_time" => "18000s",
42
+ })
43
+ COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
44
+ CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
45
+ CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
46
+ CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
47
+ EX_CONFIG_FOR_ROTATE = CONFIG + config_element("", "", { "expand_rotate_time" => "18000s" })
48
+
49
+
50
+
51
+ SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
52
+ PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
53
+ MULTILINE_CONFIG = config_element(
54
+ "", "", {
55
+ "format" => "multiline",
56
+ "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
57
+ "format_firstline" => "/^[s]/"
58
+ })
59
+ PARSE_MULTILINE_CONFIG = config_element(
60
+ "", "", {},
61
+ [config_element("parse", "", {
62
+ "@type" => "multiline",
63
+ "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
64
+ "format_firstline" => "/^[s]/"
65
+ })
66
+ ])
71
67
 
72
68
  def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
73
69
  config = use_common_conf ? COMMON_CONFIG + conf : conf
74
- Fluent::Test::InputTestDriver.new(Fluent::TailExRotateInput).configure(config)
70
+ Fluent::Test::Driver::Input.new(Fluent::Plugin::TailInput).configure(config)
75
71
  end
76
72
 
77
- def test_configure
78
- d = create_driver
79
- assert_equal ["#{TMP_DIR}/tail.txt"], d.instance.paths
80
- assert_equal "t1", d.instance.tag
81
- assert_equal 2, d.instance.rotate_wait
82
- assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
83
- assert_equal 1000, d.instance.read_lines_limit
84
- end
73
+ sub_test_case "configure" do
74
+ test "plain single line" do
75
+ d = create_driver
76
+ assert_equal ["#{TMP_DIR}/tail.txt"], d.instance.paths
77
+ assert_equal "t1", d.instance.tag
78
+ assert_equal 2, d.instance.rotate_wait
79
+ assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
80
+ assert_equal 1000, d.instance.read_lines_limit
81
+ assert_equal false, d.instance.ignore_repeated_permission_error
82
+ end
85
83
 
86
- # TODO: Should using more better approach instead of sleep wait
84
+ data("empty" => config_element,
85
+ "w/o @type" => config_element("", "", {}, [config_element("parse", "", {})]))
86
+ test "w/o parse section" do |conf|
87
+ assert_raise(Fluent::ConfigError) do
88
+ create_driver(conf)
89
+ end
90
+ end
87
91
 
88
- def test_emit
89
- File.open("#{TMP_DIR}/tail.txt", "w") {|f|
90
- f.puts "test1"
91
- f.puts "test2"
92
- }
92
+ sub_test_case "encoding" do
93
+ test "valid" do
94
+ conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" })
95
+ d = create_driver(conf)
96
+ assert_equal Encoding::UTF_8, d.instance.encoding
97
+ end
93
98
 
94
- d = create_driver
99
+ test "invalid" do
100
+ conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "no-such-encoding" })
101
+ assert_raise(Fluent::ConfigError) do
102
+ create_driver(conf)
103
+ end
104
+ end
105
+ end
95
106
 
96
- d.run do
97
- sleep 1
107
+ sub_test_case "from_encoding" do
108
+ test "only specified from_encoding raise ConfigError" do
109
+ conf = SINGLE_LINE_CONFIG + config_element("", "", { "from_encoding" => "utf-8" })
110
+ assert_raise(Fluent::ConfigError) do
111
+ create_driver(conf)
112
+ end
113
+ end
98
114
 
99
- File.open("#{TMP_DIR}/tail.txt", "a") {|f|
100
- f.puts "test3"
101
- f.puts "test4"
102
- }
103
- sleep 1
104
- end
115
+ test "valid" do
116
+ conf = SINGLE_LINE_CONFIG + config_element("", "", {
117
+ "from_encoding" => "utf-8",
118
+ "encoding" => "utf-8"
119
+ })
120
+ d = create_driver(conf)
121
+ assert_equal(Encoding::UTF_8, d.instance.from_encoding)
122
+ end
105
123
 
106
- emits = d.emits
107
- assert_equal(true, emits.length > 0)
108
- assert_equal({"message" => "test3"}, emits[0][2])
109
- assert_equal({"message" => "test4"}, emits[1][2])
110
- assert_equal(1, d.emit_streams.size)
124
+ test "invalid" do
125
+ conf = SINGLE_LINE_CONFIG + config_element("", "", {
126
+ "from_encoding" => "no-such-encoding",
127
+ "encoding" => "utf-8"
128
+ })
129
+ assert_raise(Fluent::ConfigError) do
130
+ create_driver(conf)
131
+ end
132
+ end
133
+ end
111
134
  end
112
135
 
113
- data('1' => [1, 2], '10' => [10, 1])
114
- def test_emit_with_read_lines_limit(data)
115
- limit, num_emits = data
116
- d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + "read_lines_limit #{limit}")
117
- msg = 'test' * 500 # in_tail reads 2048 bytes at once.
136
+ sub_test_case "singleline" do
137
+ data(flat: SINGLE_LINE_CONFIG,
138
+ parse: PARSE_SINGLE_LINE_CONFIG)
139
+ def test_emit(data)
140
+ config = data
141
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
142
+ f.puts "test1"
143
+ f.puts "test2"
144
+ }
118
145
 
119
- d.run do
120
- sleep 1
146
+ d = create_driver(config)
121
147
 
122
- File.open("#{TMP_DIR}/tail.txt", "a") {|f|
123
- f.puts msg
124
- f.puts msg
125
- }
126
- sleep 1
148
+ d.run(expect_emits: 1) do
149
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
150
+ f.puts "test3\ntest4"
151
+ }
152
+ end
153
+
154
+ events = d.events
155
+ assert_equal(true, events.length > 0)
156
+ assert_equal({"message" => "test3"}, events[0][2])
157
+ assert_equal({"message" => "test4"}, events[1][2])
158
+ assert(events[0][1].is_a?(Fluent::EventTime))
159
+ assert(events[1][1].is_a?(Fluent::EventTime))
160
+ assert_equal(1, d.emit_count)
127
161
  end
128
162
 
129
- emits = d.emits
130
- assert_equal(true, emits.length > 0)
131
- assert_equal({"message" => msg}, emits[0][2])
132
- assert_equal({"message" => msg}, emits[1][2])
133
- assert_equal(num_emits, d.emit_streams.size)
134
- end
163
+ def test_emit_with_emit_unmatched_lines_true
164
+ config = config_element("", "", { "format" => "/^(?<message>test.*)/", "emit_unmatched_lines" => true })
165
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
166
+
167
+ d = create_driver(config)
168
+ d.run(expect_emits: 1) do
169
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
170
+ f.puts "test line 1"
171
+ f.puts "test line 2"
172
+ f.puts "bad line 1"
173
+ f.puts "test line 3"
174
+ }
175
+ end
135
176
 
136
- def test_emit_with_read_from_head
137
- File.open("#{TMP_DIR}/tail.txt", "w") {|f|
138
- f.puts "test1"
139
- f.puts "test2"
140
- }
177
+ events = d.events
178
+ assert_equal(4, events.length)
179
+ assert_equal({"message" => "test line 1"}, events[0][2])
180
+ assert_equal({"message" => "test line 2"}, events[1][2])
181
+ assert_equal({"unmatched_line" => "bad line 1"}, events[2][2])
182
+ assert_equal({"message" => "test line 3"}, events[3][2])
183
+ end
184
+
185
+ data('flat 1' => [:flat, 1, 2],
186
+ 'flat 10' => [:flat, 10, 1],
187
+ 'parse 1' => [:parse, 1, 2],
188
+ 'parse 10' => [:parse, 10, 1])
189
+ def test_emit_with_read_lines_limit(data)
190
+ config_style, limit, num_events = data
191
+ case config_style
192
+ when :flat
193
+ config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit })
194
+ when :parse
195
+ config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit }) + PARSE_SINGLE_LINE_CONFIG
196
+ end
197
+ d = create_driver(config)
198
+ msg = 'test' * 500 # in_tail reads 2048 bytes at once.
141
199
 
142
- d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
200
+ d.run(expect_emits: 1) do
201
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
202
+ f.puts msg
203
+ f.puts msg
204
+ }
205
+ end
143
206
 
144
- d.run do
145
- sleep 1
207
+ events = d.events
208
+ assert_equal(true, events.length > 0)
209
+ assert_equal({"message" => msg}, events[0][2])
210
+ assert_equal({"message" => msg}, events[1][2])
211
+ assert_equal(num_events, d.emit_count)
212
+ end
146
213
 
147
- File.open("#{TMP_DIR}/tail.txt", "a") {|f|
148
- f.puts "test3"
149
- f.puts "test4"
214
+ data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
215
+ parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
216
+ def test_emit_with_read_from_head(data)
217
+ config = data
218
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
219
+ f.puts "test1"
220
+ f.puts "test2"
150
221
  }
151
- sleep 1
222
+
223
+ d = create_driver(config)
224
+
225
+ d.run(expect_emits: 2) do
226
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
227
+ f.puts "test3"
228
+ f.puts "test4"
229
+ }
230
+ end
231
+
232
+ events = d.events
233
+ assert(events.length > 0)
234
+ assert_equal({"message" => "test1"}, events[0][2])
235
+ assert_equal({"message" => "test2"}, events[1][2])
236
+ assert_equal({"message" => "test3"}, events[2][2])
237
+ assert_equal({"message" => "test4"}, events[3][2])
152
238
  end
153
239
 
154
- emits = d.emits
155
- assert(emits.length > 0)
156
- assert_equal({"message" => "test1"}, emits[0][2])
157
- assert_equal({"message" => "test2"}, emits[1][2])
158
- assert_equal({"message" => "test3"}, emits[2][2])
159
- assert_equal({"message" => "test4"}, emits[3][2])
160
- end
240
+ data(flat: CONFIG_ENABLE_WATCH_TIMER + SINGLE_LINE_CONFIG,
241
+ parse: CONFIG_ENABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG)
242
+ def test_emit_with_enable_watch_timer(data)
243
+ config = data
244
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
245
+ f.puts "test1"
246
+ f.puts "test2"
247
+ }
161
248
 
162
- def test_rotate_file
163
- emits = sub_test_rotate_file(SINGLE_LINE_CONFIG)
164
- assert_equal(4, emits.length)
165
- assert_equal({"message" => "test3"}, emits[0][2])
166
- assert_equal({"message" => "test4"}, emits[1][2])
167
- assert_equal({"message" => "test5"}, emits[2][2])
168
- assert_equal({"message" => "test6"}, emits[3][2])
169
- end
249
+ d = create_driver(config)
250
+
251
+ d.run(expect_emits: 1) do
252
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
253
+ f.puts "test3"
254
+ f.puts "test4"
255
+ }
256
+ # according to cool.io's stat_watcher.c, systems without inotify will use
257
+ # an "automatic" value, typically around 5 seconds
258
+ end
170
259
 
171
- def test_rotate_file_with_read_from_head
172
- emits = sub_test_rotate_file(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
173
- assert_equal(6, emits.length)
174
- assert_equal({"message" => "test1"}, emits[0][2])
175
- assert_equal({"message" => "test2"}, emits[1][2])
176
- assert_equal({"message" => "test3"}, emits[2][2])
177
- assert_equal({"message" => "test4"}, emits[3][2])
178
- assert_equal({"message" => "test5"}, emits[4][2])
179
- assert_equal({"message" => "test6"}, emits[5][2])
260
+ events = d.events
261
+ assert(events.length > 0)
262
+ assert_equal({"message" => "test3"}, events[0][2])
263
+ assert_equal({"message" => "test4"}, events[1][2])
264
+ end
180
265
  end
181
266
 
182
- def test_rotate_file_with_write_old
183
- emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
184
- File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
185
- rotated_file.puts "test7"
186
- rotated_file.puts "test8"
187
- rotated_file.flush
267
+ class TestWithSystem < self
268
+ include Fluent::SystemConfig::Mixin
188
269
 
189
- sleep 1
190
- File.open("#{TMP_DIR}/tail.txt", "a") { |f|
191
- f.puts "test5"
192
- f.puts "test6"
270
+ OVERRIDE_FILE_PERMISSION = 0620
271
+ CONFIG_SYSTEM = %[
272
+ <system>
273
+ file_permission #{OVERRIDE_FILE_PERMISSION}
274
+ </system>
275
+ ]
276
+
277
+ def setup
278
+ omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
279
+ # Store default permission
280
+ @default_permission = system_config.instance_variable_get(:@file_permission)
281
+ end
282
+
283
+ def teardown
284
+ # Restore default permission
285
+ system_config.instance_variable_set(:@file_permission, @default_permission)
286
+ end
287
+
288
+ def parse_system(text)
289
+ basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
290
+ Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
291
+ end
292
+
293
+ def test_emit_with_system
294
+ system_conf = parse_system(CONFIG_SYSTEM)
295
+ sc = Fluent::SystemConfig.new(system_conf)
296
+ Fluent::Engine.init(sc)
297
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
298
+ f.puts "test1"
299
+ f.puts "test2"
193
300
  }
194
- }
195
- assert_equal(6, emits.length)
196
- assert_equal({"message" => "test3"}, emits[0][2])
197
- assert_equal({"message" => "test4"}, emits[1][2])
198
- assert_equal({"message" => "test7"}, emits[2][2])
199
- assert_equal({"message" => "test8"}, emits[3][2])
200
- assert_equal({"message" => "test5"}, emits[4][2])
201
- assert_equal({"message" => "test6"}, emits[5][2])
202
- end
203
301
 
204
- def test_rotate_file_with_write_old_and_no_new_file
205
- emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
206
- rotated_file.puts "test7"
207
- rotated_file.puts "test8"
208
- rotated_file.flush
209
- }
210
- assert_equal(4, emits.length)
211
- assert_equal({"message" => "test3"}, emits[0][2])
212
- assert_equal({"message" => "test4"}, emits[1][2])
213
- assert_equal({"message" => "test7"}, emits[2][2])
214
- assert_equal({"message" => "test8"}, emits[3][2])
302
+ d = create_driver
303
+
304
+ d.run(expect_emits: 1) do
305
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
306
+ f.puts "test3"
307
+ f.puts "test4"
308
+ }
309
+ end
310
+
311
+ events = d.events
312
+ assert_equal(true, events.length > 0)
313
+ assert_equal({"message" => "test3"}, events[0][2])
314
+ assert_equal({"message" => "test4"}, events[1][2])
315
+ assert(events[0][1].is_a?(Fluent::EventTime))
316
+ assert(events[1][1].is_a?(Fluent::EventTime))
317
+ assert_equal(1, d.emit_count)
318
+ pos = d.instance.instance_variable_get(:@pf_file)
319
+ mode = "%o" % File.stat(pos).mode
320
+ assert_equal OVERRIDE_FILE_PERMISSION, mode[-3, 3].to_i
321
+ end
215
322
  end
216
323
 
217
- def sub_test_rotate_file(config = nil)
218
- file = File.open("#{TMP_DIR}/tail.txt", "w")
219
- file.puts "test1"
220
- file.puts "test2"
221
- file.flush
324
+ sub_test_case "rotate file" do
325
+ data(flat: SINGLE_LINE_CONFIG,
326
+ parse: PARSE_SINGLE_LINE_CONFIG)
327
+ def test_rotate_file(data)
328
+ config = data
329
+ events = sub_test_rotate_file(config, expect_emits: 2)
330
+ assert_equal(4, events.length)
331
+ assert_equal({"message" => "test3"}, events[0][2])
332
+ assert_equal({"message" => "test4"}, events[1][2])
333
+ assert_equal({"message" => "test5"}, events[2][2])
334
+ assert_equal({"message" => "test6"}, events[3][2])
335
+ end
222
336
 
223
- d = create_driver(config)
224
- d.run do
225
- sleep 1
337
+ data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
338
+ parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
339
+ def test_rotate_file_with_read_from_head(data)
340
+ config = data
341
+ events = sub_test_rotate_file(config, expect_records: 6)
342
+ assert_equal(6, events.length)
343
+ assert_equal({"message" => "test1"}, events[0][2])
344
+ assert_equal({"message" => "test2"}, events[1][2])
345
+ assert_equal({"message" => "test3"}, events[2][2])
346
+ assert_equal({"message" => "test4"}, events[3][2])
347
+ assert_equal({"message" => "test5"}, events[4][2])
348
+ assert_equal({"message" => "test6"}, events[5][2])
349
+ end
226
350
 
227
- file.puts "test3"
228
- file.puts "test4"
229
- file.flush
230
- sleep 1
351
+ data(flat: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
352
+ parse: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
353
+ def test_rotate_file_with_open_on_every_update(data)
354
+ config = data
355
+ events = sub_test_rotate_file(config, expect_records: 6)
356
+ assert_equal(6, events.length)
357
+ assert_equal({"message" => "test1"}, events[0][2])
358
+ assert_equal({"message" => "test2"}, events[1][2])
359
+ assert_equal({"message" => "test3"}, events[2][2])
360
+ assert_equal({"message" => "test4"}, events[3][2])
361
+ assert_equal({"message" => "test5"}, events[4][2])
362
+ assert_equal({"message" => "test6"}, events[5][2])
363
+ end
231
364
 
232
- FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
233
- if block_given?
234
- yield file
235
- sleep 1
236
- else
237
- sleep 1
238
- File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
239
- sleep 1
365
+ data(flat: SINGLE_LINE_CONFIG,
366
+ parse: PARSE_SINGLE_LINE_CONFIG)
367
+ def test_rotate_file_with_write_old(data)
368
+ config = data
369
+ events = sub_test_rotate_file(config, expect_emits: 3) { |rotated_file|
370
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
371
+ rotated_file.puts "test7"
372
+ rotated_file.puts "test8"
373
+ rotated_file.flush
240
374
 
241
- File.open("#{TMP_DIR}/tail.txt", "a") { |f|
375
+ sleep 1
376
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
242
377
  f.puts "test5"
243
378
  f.puts "test6"
244
379
  }
245
- sleep 1
246
- end
380
+ }
381
+ # This test sometimes fails and it shows a potential bug of in_tail
382
+ # https://github.com/fluent/fluentd/issues/1434
383
+ assert_equal(6, events.length)
384
+ assert_equal({"message" => "test3"}, events[0][2])
385
+ assert_equal({"message" => "test4"}, events[1][2])
386
+ assert_equal({"message" => "test7"}, events[2][2])
387
+ assert_equal({"message" => "test8"}, events[3][2])
388
+ assert_equal({"message" => "test5"}, events[4][2])
389
+ assert_equal({"message" => "test6"}, events[5][2])
247
390
  end
248
391
 
249
- d.run do
250
- sleep 1
392
+ data(flat: SINGLE_LINE_CONFIG,
393
+ parse: PARSE_SINGLE_LINE_CONFIG)
394
+ def test_rotate_file_with_write_old_and_no_new_file(data)
395
+ config = data
396
+ events = sub_test_rotate_file(config, expect_emits: 2) { |rotated_file|
397
+ rotated_file.puts "test7"
398
+ rotated_file.puts "test8"
399
+ rotated_file.flush
400
+ }
401
+ assert_equal(4, events.length)
402
+ assert_equal({"message" => "test3"}, events[0][2])
403
+ assert_equal({"message" => "test4"}, events[1][2])
404
+ assert_equal({"message" => "test7"}, events[2][2])
405
+ assert_equal({"message" => "test8"}, events[3][2])
251
406
  end
252
407
 
253
- d.emits
254
- ensure
255
- file.close
408
+ def sub_test_rotate_file(config = nil, expect_emits: nil, expect_records: nil, timeout: nil)
409
+ file = Fluent::FileWrapper.open("#{TMP_DIR}/tail.txt", "wb")
410
+ file.puts "test1"
411
+ file.puts "test2"
412
+ file.flush
413
+
414
+ d = create_driver(config)
415
+ d.run(expect_emits: expect_emits, expect_records: expect_records, timeout: timeout) do
416
+ size = d.emit_count
417
+ file.puts "test3"
418
+ file.puts "test4"
419
+ file.flush
420
+ sleep(0.1) until d.emit_count >= size + 1
421
+ size = d.emit_count
422
+
423
+ if Fluent.windows?
424
+ file.close
425
+ FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt", force: true)
426
+ file = File.open("#{TMP_DIR}/tail.txt", "ab")
427
+ else
428
+ FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
429
+ end
430
+ if block_given?
431
+ yield file
432
+ else
433
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
434
+ sleep 1
435
+
436
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
437
+ f.puts "test5"
438
+ f.puts "test6"
439
+ }
440
+ end
441
+ end
442
+
443
+ d.events
444
+ ensure
445
+ file.close if file && !file.closed?
446
+ end
256
447
  end
257
448
 
258
449
  def test_lf
259
- File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
450
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f| }
260
451
 
261
452
  d = create_driver
262
453
 
263
- d.run do
264
- File.open("#{TMP_DIR}/tail.txt", "a") {|f|
454
+ d.run(expect_emits: 1) do
455
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
265
456
  f.print "test3"
266
457
  }
267
458
  sleep 1
268
459
 
269
- File.open("#{TMP_DIR}/tail.txt", "a") {|f|
460
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
270
461
  f.puts "test4"
271
462
  }
272
- sleep 1
273
463
  end
274
464
 
275
- emits = d.emits
276
- assert_equal(true, emits.length > 0)
277
- assert_equal({"message" => "test3test4"}, emits[0][2])
465
+ events = d.events
466
+ assert_equal(true, events.length > 0)
467
+ assert_equal({"message" => "test3test4"}, events[0][2])
278
468
  end
279
469
 
280
470
  def test_whitespace
281
- File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
471
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f| }
282
472
 
283
473
  d = create_driver
284
474
 
285
- d.run do
286
- sleep 1
287
-
288
- File.open("#{TMP_DIR}/tail.txt", "a") {|f|
475
+ d.run(expect_emits: 1) do
476
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
289
477
  f.puts " " # 4 spaces
290
478
  f.puts " 4 spaces"
291
479
  f.puts "4 spaces "
@@ -293,169 +481,464 @@ class TailExRotateInputTest < Test::Unit::TestCase
293
481
  f.puts " tab"
294
482
  f.puts "tab "
295
483
  }
296
- sleep 1
297
484
  end
298
485
 
299
- emits = d.emits
300
- assert_equal(true, emits.length > 0)
301
- assert_equal({"message" => " "}, emits[0][2])
302
- assert_equal({"message" => " 4 spaces"}, emits[1][2])
303
- assert_equal({"message" => "4 spaces "}, emits[2][2])
304
- assert_equal({"message" => " "}, emits[3][2])
305
- assert_equal({"message" => " tab"}, emits[4][2])
306
- assert_equal({"message" => "tab "}, emits[5][2])
486
+ events = d.events
487
+ assert_equal(true, events.length > 0)
488
+ assert_equal({"message" => " "}, events[0][2])
489
+ assert_equal({"message" => " 4 spaces"}, events[1][2])
490
+ assert_equal({"message" => "4 spaces "}, events[2][2])
491
+ assert_equal({"message" => " "}, events[3][2])
492
+ assert_equal({"message" => " tab"}, events[4][2])
493
+ assert_equal({"message" => "tab "}, events[5][2])
307
494
  end
308
495
 
309
- # multiline mode test
496
+ data(
497
+ 'flat default encoding' => [SINGLE_LINE_CONFIG, Encoding::ASCII_8BIT],
498
+ 'flat explicit encoding config' => [SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8],
499
+ 'parse default encoding' => [PARSE_SINGLE_LINE_CONFIG, Encoding::ASCII_8BIT],
500
+ 'parse explicit encoding config' => [PARSE_SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8])
501
+ def test_encoding(data)
502
+ encoding_config, encoding = data
310
503
 
311
- def test_multiline
312
- File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
504
+ d = create_driver(CONFIG_READ_FROM_HEAD + encoding_config)
313
505
 
314
- d = create_driver %[
315
- format multiline
316
- format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
317
- format_firstline /^[s]/
318
- ]
319
- d.run do
320
- File.open("#{TMP_DIR}/tail.txt", "a") { |f|
321
- f.puts "f test1"
322
- f.puts "s test2"
323
- f.puts "f test3"
324
- f.puts "f test4"
325
- f.puts "s test5"
326
- f.puts "s test6"
327
- f.puts "f test7"
328
- f.puts "s test8"
506
+ d.run(expect_emits: 1) do
507
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
508
+ f.puts "test"
329
509
  }
330
- sleep 1
331
510
  end
332
511
 
333
- emits = d.emits
334
- assert(emits.length > 0)
335
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
336
- assert_equal({"message1" => "test5"}, emits[1][2])
337
- assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
338
- assert_equal({"message1" => "test8"}, emits[3][2])
512
+ events = d.events
513
+ assert_equal(encoding, events[0][2]['message'].encoding)
339
514
  end
340
515
 
341
- def test_multiline_with_multiple_formats
342
- File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
516
+ def test_from_encoding
517
+ conf = config_element(
518
+ "", "", {
519
+ "format" => "/(?<message>.*)/",
520
+ "read_from_head" => "true",
521
+ "from_encoding" => "cp932",
522
+ "encoding" => "utf-8"
523
+ })
524
+ d = create_driver(conf)
525
+ cp932_message = "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
526
+ utf8_message = cp932_message.encode(Encoding::UTF_8)
527
+
528
+ d.run(expect_emits: 1) do
529
+ File.open("#{TMP_DIR}/tail.txt", "w:cp932") {|f|
530
+ f.puts cp932_message
531
+ }
532
+ end
343
533
 
344
- d = create_driver %[
345
- format multiline
346
- format1 /^s (?<message1>[^\\n]+)\\n?/
347
- format2 /(f (?<message2>[^\\n]+)\\n?)?/
348
- format3 /(f (?<message3>.*))?/
349
- format_firstline /^[s]/
350
- ]
351
- d.run do
352
- File.open("#{TMP_DIR}/tail.txt", "a") { |f|
353
- f.puts "f test1"
354
- f.puts "s test2"
355
- f.puts "f test3"
356
- f.puts "f test4"
357
- f.puts "s test5"
358
- f.puts "s test6"
359
- f.puts "f test7"
360
- f.puts "s test8"
534
+ events = d.events
535
+ assert_equal(utf8_message, events[0][2]['message'])
536
+ assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
537
+ end
538
+
539
+ def test_from_encoding_utf16
540
+ conf = config_element(
541
+ "", "", {
542
+ "format" => "/(?<message>.*)/",
543
+ "read_from_head" => "true",
544
+ "from_encoding" => "utf-16le",
545
+ "encoding" => "utf-8"
546
+ })
547
+ d = create_driver(conf)
548
+ utf16_message = "\u306F\u308D\u30FC\u308F\u30FC\u308B\u3069\n".encode(Encoding::UTF_16LE)
549
+ utf8_message = utf16_message.encode(Encoding::UTF_8).strip
550
+
551
+ d.run(expect_emits: 1) do
552
+ File.open("#{TMP_DIR}/tail.txt", "w:utf-16le") { |f|
553
+ f.write utf16_message
361
554
  }
362
- sleep 1
363
555
  end
364
556
 
365
- emits = d.emits
366
- assert(emits.length > 0)
367
- assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
368
- assert_equal({"message1" => "test5"}, emits[1][2])
369
- assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
370
- assert_equal({"message1" => "test8"}, emits[3][2])
557
+ events = d.events
558
+ assert_equal(utf8_message, events[0][2]['message'])
559
+ assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
371
560
  end
372
561
 
373
- def test_multilinelog_with_multiple_paths
374
- files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
375
- files.each { |file| File.open(file, "w") { |f| } }
376
-
377
- d = create_driver(%[
378
- path #{files[0]},#{files[1]}
379
- tag t1
380
- format multiline
381
- format1 /^[s|f] (?<message>.*)/
382
- format_firstline /^[s]/
383
- ], false)
384
- d.run do
385
- files.each do |file|
386
- File.open(file, 'a') { |f|
387
- f.puts "f #{file} line should be ignored"
388
- f.puts "s test1"
389
- f.puts "f test2"
562
+
563
+ sub_test_case "multiline" do
564
+ data(flat: MULTILINE_CONFIG,
565
+ parse: PARSE_MULTILINE_CONFIG)
566
+ def test_multiline(data)
567
+ config = data
568
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
569
+
570
+ d = create_driver(config)
571
+ d.run(expect_emits: 1) do
572
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
573
+ f.puts "f test1"
574
+ f.puts "s test2"
390
575
  f.puts "f test3"
391
- f.puts "s test4"
576
+ f.puts "f test4"
577
+ f.puts "s test5"
578
+ f.puts "s test6"
579
+ f.puts "f test7"
580
+ f.puts "s test8"
392
581
  }
393
582
  end
394
- sleep 1
583
+
584
+ events = d.events
585
+ assert_equal(4, events.length)
586
+ assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
587
+ assert_equal({"message1" => "test5"}, events[1][2])
588
+ assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
589
+ assert_equal({"message1" => "test8"}, events[3][2])
590
+ end
591
+
592
+ data(flat: MULTILINE_CONFIG,
593
+ parse: PARSE_MULTILINE_CONFIG)
594
+ def test_multiline_with_emit_unmatched_lines_true(data)
595
+ config = data + config_element("", "", { "emit_unmatched_lines" => true })
596
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
597
+
598
+ d = create_driver(config)
599
+ d.run(expect_emits: 1) do
600
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
601
+ f.puts "f test1"
602
+ f.puts "s test2"
603
+ f.puts "f test3"
604
+ f.puts "f test4"
605
+ f.puts "s test5"
606
+ f.puts "s test6"
607
+ f.puts "f test7"
608
+ f.puts "s test8"
609
+ }
610
+ end
611
+
612
+ events = d.events
613
+ assert_equal(5, events.length)
614
+ assert_equal({"unmatched_line" => "f test1"}, events[0][2])
615
+ assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[1][2])
616
+ assert_equal({"message1" => "test5"}, events[2][2])
617
+ assert_equal({"message1" => "test6", "message2" => "test7"}, events[3][2])
618
+ assert_equal({"message1" => "test8"}, events[4][2])
619
+ end
620
+
621
+ data(flat: MULTILINE_CONFIG,
622
+ parse: PARSE_MULTILINE_CONFIG)
623
+ def test_multiline_with_flush_interval(data)
624
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
625
+
626
+ config = data + config_element("", "", { "multiline_flush_interval" => "2s" })
627
+ d = create_driver(config)
628
+
629
+ assert_equal 2, d.instance.multiline_flush_interval
630
+
631
+ d.run(expect_emits: 1) do
632
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
633
+ f.puts "f test1"
634
+ f.puts "s test2"
635
+ f.puts "f test3"
636
+ f.puts "f test4"
637
+ f.puts "s test5"
638
+ f.puts "s test6"
639
+ f.puts "f test7"
640
+ f.puts "s test8"
641
+ }
642
+ end
643
+
644
+ events = d.events
645
+ assert_equal(4, events.length)
646
+ assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
647
+ assert_equal({"message1" => "test5"}, events[1][2])
648
+ assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
649
+ assert_equal({"message1" => "test8"}, events[3][2])
650
+ end
651
+
652
+ data(
653
+ 'flat default encoding' => [MULTILINE_CONFIG, Encoding::ASCII_8BIT],
654
+ 'flat explicit encoding config' => [MULTILINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8],
655
+ 'parse default encoding' => [PARSE_MULTILINE_CONFIG, Encoding::ASCII_8BIT],
656
+ 'parse explicit encoding config' => [PARSE_MULTILINE_CONFIG + config_element("", "", { "encoding" => "utf-8" }), Encoding::UTF_8])
657
+ def test_multiline_encoding_of_flushed_record(data)
658
+ encoding_config, encoding = data
659
+
660
+ config = config_element("", "", {
661
+ "multiline_flush_interval" => "2s",
662
+ "read_from_head" => "true",
663
+ })
664
+ d = create_driver(config + encoding_config)
665
+
666
+ d.run(expect_emits: 1) do
667
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f|
668
+ f.puts "s test"
669
+ }
670
+ end
671
+ events = d.events
672
+ assert_equal(1, events.length)
673
+ assert_equal(encoding, events[0][2]['message1'].encoding)
674
+ end
675
+
676
+ def test_multiline_from_encoding_of_flushed_record
677
+ conf = MULTILINE_CONFIG + config_element(
678
+ "", "",
679
+ {
680
+ "multiline_flush_interval" => "1s",
681
+ "read_from_head" => "true",
682
+ "from_encoding" => "cp932",
683
+ "encoding" => "utf-8"
684
+ })
685
+ d = create_driver(conf)
686
+
687
+ cp932_message = "s \x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
688
+ utf8_message = "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".encode(Encoding::UTF_8, Encoding::CP932)
689
+ d.run(expect_emits: 1) do
690
+ File.open("#{TMP_DIR}/tail.txt", "w:cp932") { |f|
691
+ f.puts cp932_message
692
+ }
693
+ end
694
+
695
+ events = d.events
696
+ assert_equal(1, events.length)
697
+ assert_equal(utf8_message, events[0][2]['message1'])
698
+ assert_equal(Encoding::UTF_8, events[0][2]['message1'].encoding)
395
699
  end
396
700
 
397
- emits = d.emits
398
- assert_equal({"message" => "test1\nf test2\nf test3"}, emits[0][2])
399
- assert_equal({"message" => "test1\nf test2\nf test3"}, emits[1][2])
400
- # "test4" events are here because these events are flushed at shutdown phase
401
- assert_equal({"message" => "test4"}, emits[2][2])
402
- assert_equal({"message" => "test4"}, emits[3][2])
701
+ data(flat: config_element(
702
+ "", "", {
703
+ "format" => "multiline",
704
+ "format1" => "/^s (?<message1>[^\\n]+)\\n?/",
705
+ "format2" => "/(f (?<message2>[^\\n]+)\\n?)?/",
706
+ "format3" => "/(f (?<message3>.*))?/",
707
+ "format_firstline" => "/^[s]/"
708
+ }),
709
+ parse: config_element(
710
+ "", "", {},
711
+ [config_element("parse", "", {
712
+ "@type" => "multiline",
713
+ "format1" => "/^s (?<message1>[^\\n]+)\\n?/",
714
+ "format2" => "/(f (?<message2>[^\\n]+)\\n?)?/",
715
+ "format3" => "/(f (?<message3>.*))?/",
716
+ "format_firstline" => "/^[s]/"
717
+ })
718
+ ])
719
+ )
720
+ def test_multiline_with_multiple_formats(data)
721
+ config = data
722
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
723
+
724
+ d = create_driver(config)
725
+ d.run(expect_emits: 1) do
726
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
727
+ f.puts "f test1"
728
+ f.puts "s test2"
729
+ f.puts "f test3"
730
+ f.puts "f test4"
731
+ f.puts "s test5"
732
+ f.puts "s test6"
733
+ f.puts "f test7"
734
+ f.puts "s test8"
735
+ }
736
+ end
737
+
738
+ events = d.events
739
+ assert(events.length > 0)
740
+ assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[0][2])
741
+ assert_equal({"message1" => "test5"}, events[1][2])
742
+ assert_equal({"message1" => "test6", "message2" => "test7"}, events[2][2])
743
+ assert_equal({"message1" => "test8"}, events[3][2])
744
+ end
745
+
746
+ data(flat: config_element(
747
+ "", "", {
748
+ "format" => "multiline",
749
+ "format1" => "/^[s|f] (?<message>.*)/",
750
+ "format_firstline" => "/^[s]/"
751
+ }),
752
+ parse: config_element(
753
+ "", "", {},
754
+ [config_element("parse", "", {
755
+ "@type" => "multiline",
756
+ "format1" => "/^[s|f] (?<message>.*)/",
757
+ "format_firstline" => "/^[s]/"
758
+ })
759
+ ])
760
+ )
761
+ def test_multilinelog_with_multiple_paths(data)
762
+ files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
763
+ files.each { |file| File.open(file, "wb") { |f| } }
764
+
765
+ config = data + config_element("", "", {
766
+ "path" => "#{files[0]},#{files[1]}",
767
+ "tag" => "t1",
768
+ })
769
+ d = create_driver(config, false)
770
+ d.run(expect_emits: 2) do
771
+ files.each do |file|
772
+ File.open(file, 'ab') { |f|
773
+ f.puts "f #{file} line should be ignored"
774
+ f.puts "s test1"
775
+ f.puts "f test2"
776
+ f.puts "f test3"
777
+ f.puts "s test4"
778
+ }
779
+ end
780
+ end
781
+
782
+ events = d.events
783
+ assert_equal({"message" => "test1\nf test2\nf test3"}, events[0][2])
784
+ assert_equal({"message" => "test1\nf test2\nf test3"}, events[1][2])
785
+ # "test4" events are here because these events are flushed at shutdown phase
786
+ assert_equal({"message" => "test4"}, events[2][2])
787
+ assert_equal({"message" => "test4"}, events[3][2])
788
+ end
789
+
790
+ data(flat: config_element("", "", {
791
+ "format" => "multiline",
792
+ "format1" => "/(?<var1>foo \\d)\\n/",
793
+ "format2" => "/(?<var2>bar \\d)\\n/",
794
+ "format3" => "/(?<var3>baz \\d)/"
795
+ }),
796
+ parse: config_element(
797
+ "", "", {},
798
+ [config_element("parse", "", {
799
+ "@type" => "multiline",
800
+ "format1" => "/(?<var1>foo \\d)\\n/",
801
+ "format2" => "/(?<var2>bar \\d)\\n/",
802
+ "format3" => "/(?<var3>baz \\d)/"
803
+ })
804
+ ])
805
+ )
806
+ def test_multiline_without_firstline(data)
807
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
808
+
809
+ config = data
810
+ d = create_driver(config)
811
+ d.run(expect_emits: 1) do
812
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
813
+ f.puts "foo 1"
814
+ f.puts "bar 1"
815
+ f.puts "baz 1"
816
+ f.puts "foo 2"
817
+ f.puts "bar 2"
818
+ f.puts "baz 2"
819
+ }
820
+ end
821
+
822
+ events = d.events
823
+ assert_equal(2, events.length)
824
+ assert_equal({"var1" => "foo 1", "var2" => "bar 1", "var3" => "baz 1"}, events[0][2])
825
+ assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, events[1][2])
826
+ end
403
827
  end
404
828
 
405
- def test_multiline_without_firstline
406
- File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
829
+ sub_test_case "path" do
830
+ # * path test
831
+ # TODO: Clean up tests
832
+ EX_ROTATE_WAIT = 0
833
+
834
+ EX_CONFIG = config_element("", "", {
835
+ "tag" => "tail",
836
+ "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
837
+ "format" => "none",
838
+ "pos_file" => "#{TMP_DIR}/tail.pos",
839
+ "read_from_head" => true,
840
+ "refresh_interval" => 30,
841
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
842
+ })
843
+ EX_PATHS = [
844
+ 'test/plugin/data/2010/01/20100102-030405.log',
845
+ 'test/plugin/data/log/foo/bar.log',
846
+ 'test/plugin/data/log/test.log'
847
+ ]
407
848
 
408
- d = create_driver %[
409
- format multiline
410
- format1 /(?<var1>foo \\d)\\n/
411
- format2 /(?<var2>bar \\d)\\n/
412
- format3 /(?<var3>baz \\d)/
849
+ EX_PATHS_FOR_ROATE = [
850
+ 'test/plugin/data/2014/12/20141224-000000.log',
851
+ 'test/plugin/data/log/foo/bar.log',
852
+ 'test/plugin/data/log/test.log'
413
853
  ]
414
- d.run do
415
- File.open("#{TMP_DIR}/tail.txt", "a") { |f|
416
- f.puts "foo 1"
417
- f.puts "bar 1"
418
- f.puts "baz 1"
419
- f.puts "foo 2"
420
- f.puts "bar 2"
421
- f.puts "baz 2"
422
- }
423
- sleep 1
854
+
855
+ def test_expand_paths
856
+ plugin = create_driver(EX_CONFIG, false).instance
857
+ flexstub(Time) do |timeclass|
858
+ timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
859
+ assert_equal EX_PATHS, plugin.expand_paths.sort
860
+ end
861
+
862
+ # Test exclusion
863
+ exclude_config = EX_CONFIG + config_element("", "", { "exclude_path" => %Q(["#{EX_PATHS.last}"]) })
864
+ plugin = create_driver(exclude_config, false).instance
865
+ assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
424
866
  end
425
867
 
426
- emits = d.emits
427
- assert_equal(2, emits.length)
428
- assert_equal({"var1" => "foo 1", "var2" => "bar 1", "var3" => "baz 1"}, emits[0][2])
429
- assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, emits[1][2])
430
- end
868
+ def test_log_file_without_extension
869
+ expected_files = [
870
+ 'test/plugin/data/log/bar',
871
+ 'test/plugin/data/log/foo/bar.log',
872
+ 'test/plugin/data/log/foo/bar2',
873
+ 'test/plugin/data/log/test.log'
874
+ ]
875
+
876
+ config = config_element("", "", {
877
+ "tag" => "tail",
878
+ "path" => "test/plugin/data/log/**/*",
879
+ "format" => "none",
880
+ "pos_file" => "#{TMP_DIR}/tail.pos"
881
+ })
882
+
883
+ plugin = create_driver(config, false).instance
884
+ assert_equal expected_files, plugin.expand_paths.sort
885
+ end
431
886
 
432
- def test_expand_paths
433
- plugin = create_driver(EX_CONFIG, false).instance
434
- flexstub(Time) do |timeclass|
435
- timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
436
- assert_equal EX_PATHS, plugin.expand_paths.sort
887
+ # For https://github.com/fluent/fluentd/issues/1455
888
+ # This test is fragile because test content depends on internal implementaion.
889
+ # So if you modify in_tail internal, this test may break.
890
+ def test_unwatched_files_should_be_removed
891
+ config = config_element("", "", {
892
+ "tag" => "tail",
893
+ "path" => "#{TMP_DIR}/*.txt",
894
+ "format" => "none",
895
+ "pos_file" => "#{TMP_DIR}/tail.pos",
896
+ "read_from_head" => true,
897
+ "refresh_interval" => 1,
898
+ })
899
+ d = create_driver(config, false)
900
+ d.run(expect_emits: 1, shutdown: false) do
901
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f| f.puts "test3\n" }
902
+ end
903
+
904
+ assert_equal 1, d.instance.instance_variable_get(:@tails).keys.size
905
+ cleanup_directory(TMP_DIR)
906
+ waiting(20) { sleep 0.1 until Dir.glob("#{TMP_DIR}/*.txt").size == 0 } # Ensure file is deleted on Windows
907
+ waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 0 }
908
+
909
+ # Previous implementaion has an infinite watcher creation bug.
910
+ # Following code checks such unexpected bug by couting actual object allocation.
911
+ base_num = count_timer_object
912
+ 2.times {
913
+ sleep 1
914
+ num = count_timer_object
915
+ assert_equal base_num, num
916
+ }
917
+
918
+ d.instance_shutdown
437
919
  end
438
920
 
439
- # Test exclusion
440
- exclude_config = EX_CONFIG + " exclude_path [\"#{EX_PATHS.last}\"]"
441
- plugin = create_driver(exclude_config, false).instance
442
- assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
921
+ def count_timer_object
922
+ num = 0
923
+ ObjectSpace.each_object(Fluent::PluginHelper::Timer::TimerWatcher) { |obj|
924
+ num += 1
925
+ }
926
+ num
927
+ end
443
928
  end
444
929
 
445
- def test_refresh_watchers
930
+ def test_z_refresh_watchers
446
931
  plugin = create_driver(EX_CONFIG, false).instance
447
932
  sio = StringIO.new
448
933
  plugin.instance_eval do
449
- @pf = Fluent::TailExRotateInput::PositionFile.parse(sio)
934
+ @pf = Fluent::Plugin::TailExRotateInput::PositionFile.parse(sio)
450
935
  @loop = Coolio::Loop.new
451
- end
452
-
453
- flexstub(Time) do |timeclass|
454
- timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5), Time.new(2010, 1, 2, 3, 4, 6), Time.new(2010, 1, 2, 3, 4, 7))
936
+ end
455
937
 
456
- flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
938
+ Timecop.freeze(2010, 1, 2, 3, 4, 5) do
939
+ flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
457
940
  EX_PATHS.each do |path|
458
- watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
941
+ watcherclass.should_receive(:new).with(path, EX_ROTATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
459
942
  flexmock('TailWatcher') { |watcher|
460
943
  watcher.should_receive(:attach).once
461
944
  watcher.should_receive(:unwatched=).zero_or_more_times
@@ -465,13 +948,15 @@ class TailExRotateInputTest < Test::Unit::TestCase
465
948
  end
466
949
  plugin.refresh_watchers
467
950
  end
951
+ end
468
952
 
469
- plugin.instance_eval do
470
- @tails['test/plugin/data/2010/01/20100102-030405.log'].should_receive(:close).zero_or_more_times
471
- end
953
+ plugin.instance_eval do
954
+ @tails['test/plugin/data/2010/01/20100102-030405.log'].should_receive(:close).zero_or_more_times
955
+ end
472
956
 
473
- flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
474
- watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
957
+ Timecop.freeze(2010, 1, 2, 3, 4, 6) do
958
+ flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
959
+ watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_ROTATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
475
960
  flexmock('TailWatcher') do |watcher|
476
961
  watcher.should_receive(:attach).once
477
962
  watcher.should_receive(:unwatched=).zero_or_more_times
@@ -481,67 +966,77 @@ class TailExRotateInputTest < Test::Unit::TestCase
481
966
  plugin.refresh_watchers
482
967
  end
483
968
 
484
- flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
969
+ flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
485
970
  watcherclass.should_receive(:new).never
486
971
  plugin.refresh_watchers
487
972
  end
488
973
  end
489
974
  end
490
975
 
491
- DummyWatcher = Struct.new("DummyWatcher", :tag)
976
+ sub_test_case "receive_lines" do
977
+ DummyWatcher = Struct.new("DummyWatcher", :tag)
492
978
 
493
- def test_receive_lines
494
- plugin = create_driver(EX_CONFIG, false).instance
495
- flexstub(plugin.router) do |engineclass|
496
- engineclass.should_receive(:emit_stream).with('tail', any).once
979
+ def test_tag
980
+ d = create_driver(EX_CONFIG, false)
981
+ d.run {}
982
+ plugin = d.instance
983
+ mock(plugin.router).emit_stream('tail', anything).once
497
984
  plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
498
985
  end
499
986
 
500
- config = %[
501
- tag pre.*
502
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
503
- format none
504
- read_from_head true
505
- ]
506
- plugin = create_driver(config, false).instance
507
- flexstub(plugin.router) do |engineclass|
508
- engineclass.should_receive(:emit_stream).with('pre.foo.bar.log', any).once
987
+ def test_tag_prefix
988
+ config = config_element("", "", {
989
+ "tag" => "pre.*",
990
+ "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
991
+ "format" => "none",
992
+ "read_from_head" => true
993
+ })
994
+ d = create_driver(config, false)
995
+ d.run {}
996
+ plugin = d.instance
997
+ mock(plugin.router).emit_stream('pre.foo.bar.log', anything).once
509
998
  plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
510
999
  end
511
1000
 
512
- config = %[
513
- tag *.post
514
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
515
- format none
516
- read_from_head true
517
- ]
518
- plugin = create_driver(config, false).instance
519
- flexstub(plugin.router) do |engineclass|
520
- engineclass.should_receive(:emit_stream).with('foo.bar.log.post', any).once
1001
+ def test_tag_suffix
1002
+ config = config_element("", "", {
1003
+ "tag" => "*.post",
1004
+ "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1005
+ "format" => "none",
1006
+ "read_from_head" => true
1007
+ })
1008
+ d = create_driver(config, false)
1009
+ d.run {}
1010
+ plugin = d.instance
1011
+ mock(plugin.router).emit_stream('foo.bar.log.post', anything).once
521
1012
  plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
522
1013
  end
523
1014
 
524
- config = %[
525
- tag pre.*.post
526
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
527
- format none
528
- read_from_head true
529
- ]
530
- plugin = create_driver(config, false).instance
531
- flexstub(plugin.router) do |engineclass|
532
- engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
1015
+ def test_tag_prefix_and_suffix
1016
+ config = config_element("", "", {
1017
+ "tag" => "pre.*.post",
1018
+ "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1019
+ "format" => "none",
1020
+ "read_from_head" => true
1021
+ })
1022
+ d = create_driver(config, false)
1023
+ d.run {}
1024
+ plugin = d.instance
1025
+ mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once
533
1026
  plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
534
1027
  end
535
1028
 
536
- config = %[
537
- tag pre.*.post*ignore
538
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
539
- format none
540
- read_from_head true
541
- ]
542
- plugin = create_driver(config, false).instance
543
- flexstub(plugin.router) do |engineclass|
544
- engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
1029
+ def test_tag_prefix_and_suffix_ignore
1030
+ config = config_element("", "", {
1031
+ "tag" => "pre.*.post*ignore",
1032
+ "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1033
+ "format" => "none",
1034
+ "read_from_head" => true
1035
+ })
1036
+ d = create_driver(config, false)
1037
+ d.run {}
1038
+ plugin = d.instance
1039
+ mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once
545
1040
  plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
546
1041
  end
547
1042
  end
@@ -549,43 +1044,188 @@ class TailExRotateInputTest < Test::Unit::TestCase
549
1044
  # Ensure that no fatal exception is raised when a file is missing and that
550
1045
  # files that do exist are still tailed as expected.
551
1046
  def test_missing_file
552
- File.open("#{TMP_DIR}/tail.txt", "w") {|f|
1047
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
553
1048
  f.puts "test1"
554
1049
  f.puts "test2"
555
1050
  }
556
1051
 
557
1052
  # Try two different configs - one with read_from_head and one without,
558
1053
  # since their interactions with the filesystem differ.
559
- config1 = %[
560
- tag t1
561
- path #{TMP_DIR}/non_existent_file.txt,#{TMP_DIR}/tail.txt
562
- format none
563
- rotate_wait 2s
564
- pos_file #{TMP_DIR}/tail.pos
565
- ]
566
- config2 = config1 + ' read_from_head true'
1054
+ config1 = config_element("", "", {
1055
+ "tag" => "t1",
1056
+ "path" => "#{TMP_DIR}/non_existent_file.txt,#{TMP_DIR}/tail.txt",
1057
+ "format" => "none",
1058
+ "rotate_wait" => "2s",
1059
+ "pos_file" => "#{TMP_DIR}/tail.pos"
1060
+ })
1061
+ config2 = config1 + config_element("", "", { "read_from_head" => true })
567
1062
  [config1, config2].each do |config|
568
1063
  d = create_driver(config, false)
569
- d.run do
570
- sleep 1
571
- File.open("#{TMP_DIR}/tail.txt", "a") {|f|
1064
+ d.run(expect_emits: 1) do
1065
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
572
1066
  f.puts "test3"
573
1067
  f.puts "test4"
574
1068
  }
575
- sleep 1
576
1069
  end
577
- emits = d.emits
578
- assert_equal(2, emits.length)
579
- assert_equal({"message" => "test3"}, emits[0][2])
580
- assert_equal({"message" => "test4"}, emits[1][2])
1070
+ # This test sometimes fails and it shows a potential bug of in_tail
1071
+ # https://github.com/fluent/fluentd/issues/1434
1072
+ events = d.events
1073
+ assert_equal(2, events.length)
1074
+ assert_equal({"message" => "test3"}, events[0][2])
1075
+ assert_equal({"message" => "test4"}, events[1][2])
1076
+ end
1077
+ end
1078
+
1079
+ sub_test_case "tail_path" do
1080
+ def test_tail_path_with_singleline
1081
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1082
+ f.puts "test1"
1083
+ f.puts "test2"
1084
+ }
1085
+
1086
+ d = create_driver(SINGLE_LINE_CONFIG + config_element("", "", { "path_key" => "path" }))
1087
+
1088
+ d.run(expect_emits: 1) do
1089
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
1090
+ f.puts "test3"
1091
+ f.puts "test4"
1092
+ }
1093
+ end
1094
+
1095
+ events = d.events
1096
+ assert_equal(true, events.length > 0)
1097
+ events.each do |emit|
1098
+ assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
1099
+ end
1100
+ end
1101
+
1102
+ def test_tail_path_with_multiline_with_firstline
1103
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
1104
+
1105
+ config = config_element("", "", {
1106
+ "path_key" => "path",
1107
+ "format" => "multiline",
1108
+ "format1" => "/^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/",
1109
+ "format_firstline" => "/^[s]/"
1110
+ })
1111
+ d = create_driver(config)
1112
+ d.run(expect_emits: 1) do
1113
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
1114
+ f.puts "f test1"
1115
+ f.puts "s test2"
1116
+ f.puts "f test3"
1117
+ f.puts "f test4"
1118
+ f.puts "s test5"
1119
+ f.puts "s test6"
1120
+ f.puts "f test7"
1121
+ f.puts "s test8"
1122
+ }
1123
+ end
1124
+
1125
+ events = d.events
1126
+ assert_equal(4, events.length)
1127
+ events.each do |emit|
1128
+ assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
1129
+ end
1130
+ end
1131
+
1132
+ def test_tail_path_with_multiline_without_firstline
1133
+ File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
1134
+
1135
+ config = config_element("", "", {
1136
+ "path_key" => "path",
1137
+ "format" => "multiline",
1138
+ "format1" => "/(?<var1>foo \\d)\\n/",
1139
+ "format2" => "/(?<var2>bar \\d)\\n/",
1140
+ "format3" => "/(?<var3>baz \\d)/",
1141
+ })
1142
+ d = create_driver(config)
1143
+ d.run(expect_emits: 1) do
1144
+ File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
1145
+ f.puts "foo 1"
1146
+ f.puts "bar 1"
1147
+ f.puts "baz 1"
1148
+ }
1149
+ end
1150
+
1151
+ events = d.events
1152
+ assert(events.length > 0)
1153
+ events.each do |emit|
1154
+ assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
1155
+ end
1156
+ end
1157
+
1158
+ def test_tail_path_with_multiline_with_multiple_paths
1159
+ files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
1160
+ files.each { |file| File.open(file, "wb") { |f| } }
1161
+
1162
+ config = config_element("", "", {
1163
+ "path" => "#{files[0]},#{files[1]}",
1164
+ "path_key" => "path",
1165
+ "tag" => "t1",
1166
+ "format" => "multiline",
1167
+ "format1" => "/^[s|f] (?<message>.*)/",
1168
+ "format_firstline" => "/^[s]/"
1169
+ })
1170
+ d = create_driver(config, false)
1171
+ d.run(expect_emits: 2) do
1172
+ files.each do |file|
1173
+ File.open(file, 'ab') { |f|
1174
+ f.puts "f #{file} line should be ignored"
1175
+ f.puts "s test1"
1176
+ f.puts "f test2"
1177
+ f.puts "f test3"
1178
+ f.puts "s test4"
1179
+ }
1180
+ end
1181
+ end
1182
+
1183
+ events = d.events
1184
+ assert_equal(4, events.length)
1185
+ assert_equal(files, [events[0][2]["path"], events[1][2]["path"]].sort)
1186
+ # "test4" events are here because these events are flushed at shutdown phase
1187
+ assert_equal(files, [events[2][2]["path"], events[3][2]["path"]].sort)
581
1188
  end
582
1189
  end
583
1190
 
584
- def test_expand_paths_for_ex_rotate
585
- plugin = create_driver(EX_CONFIG_FOR_ROTATE, false).instance
586
- flexstub(Time) do |timeclass|
587
- timeclass.should_receive(:now).with_no_args.and_return(Time.new(2014, 12, 24, 5, 0, 0))
588
- assert_equal EX_PATHS_FOR_ROATE, plugin.expand_paths.sort
1191
+ def test_limit_recently_modified
1192
+ now = Time.new(2010, 1, 2, 3, 4, 5)
1193
+ FileUtils.touch("#{TMP_DIR}/tail_unwatch.txt", mtime: (now - 3601))
1194
+ FileUtils.touch("#{TMP_DIR}/tail_watch1.txt", mtime: (now - 3600))
1195
+ FileUtils.touch("#{TMP_DIR}/tail_watch2.txt", mtime: now)
1196
+
1197
+ config = config_element('', '', {
1198
+ 'tag' => 'tail',
1199
+ 'path' => "#{TMP_DIR}/*.txt",
1200
+ 'format' => 'none',
1201
+ 'limit_recently_modified' => '3600s'
1202
+ })
1203
+
1204
+ expected_files = [
1205
+ "#{TMP_DIR}/tail_watch1.txt",
1206
+ "#{TMP_DIR}/tail_watch2.txt"
1207
+ ]
1208
+
1209
+ Timecop.freeze(now) do
1210
+ plugin = create_driver(config, false).instance
1211
+ assert_equal expected_files, plugin.expand_paths.sort
589
1212
  end
590
1213
  end
1214
+
1215
+ def test_skip_refresh_on_startup
1216
+ FileUtils.touch("#{TMP_DIR}/tail.txt")
1217
+ config = config_element('', '', {
1218
+ 'format' => 'none',
1219
+ 'refresh_interval' => 1,
1220
+ 'skip_refresh_on_startup' => true
1221
+ })
1222
+ d = create_driver(config)
1223
+ d.run(shutdown: false) {}
1224
+ assert_equal 0, d.instance.instance_variable_get(:@tails).keys.size
1225
+ # detect a file at first execution of in_tail_refresh_watchers timer
1226
+ waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1 }
1227
+ d.instance_shutdown
1228
+ end
1229
+
1230
+
591
1231
  end