fluent-plugin-tail_path 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/.travis.yml +6 -0
- data/CHANGELOG.md +3 -0
- data/Gemfile +3 -0
- data/LICENSE +1 -0
- data/README.md +33 -0
- data/Rakefile +16 -0
- data/fluent-plugin-tail_path.gemspec +24 -0
- data/lib/fluent/plugin/in_tail_path.rb +59 -0
- data/test/helper.rb +18 -0
- data/test/plugin/in_tail_path.rb +52 -0
- metadata +113 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2b4639f29d0249ebd21ba0e5691498bf4b78e301
|
4
|
+
data.tar.gz: 2e6c03cb4576e8454348da3785b76edd5425f908
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 4e6b5b7ac271210134c713f97ca1eca279fccf46e278db730e8d34d841aa7f5b6ac40f18c480acebff9d53231c28ede20c87333cc1ee223ce8ef8b0aa431869d
|
7
|
+
data.tar.gz: 1d02771648360cddea2c64a2d2d0d36eb9b409ecd687ce9da2eebd39c8bbed28ee06ec08ee133121c375e508c1a9db1f973dcaca8898598da11b415905325e75
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
Same with Fluentd
|
data/README.md
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
# fluent-plugin-tail_path
|
2
|
+
|
3
|
+
## About
|
4
|
+
|
5
|
+
This is an extension of fluentd in\_tail plugin to add `path` field which tells the log path being tailed
|
6
|
+
|
7
|
+
## Parameters
|
8
|
+
|
9
|
+
Basically same with in\_tail plugin. See http://docs.fluentd.org/articles/in_tail
|
10
|
+
|
11
|
+
Following parameters are additionally available:
|
12
|
+
|
13
|
+
- path_key
|
14
|
+
|
15
|
+
Add `path` field which tells the log path being tailed. Specify the field name.
|
16
|
+
|
17
|
+
## Contributing
|
18
|
+
|
19
|
+
1. Fork it
|
20
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
21
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
22
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
23
|
+
5. Create new [Pull Request](../../pull/new/master)
|
24
|
+
|
25
|
+
## ChangeLog
|
26
|
+
|
27
|
+
See [CHANGELOG.md](CHANGELOG.md) for details.
|
28
|
+
|
29
|
+
## Copyright
|
30
|
+
|
31
|
+
* Copyright (c) 2014- @szhem
|
32
|
+
* Copyright (c) 2014- Naotoshi Seo
|
33
|
+
* See [LICENSE](LICENSE) for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
|
4
|
+
require 'rake/testtask'
|
5
|
+
Rake::TestTask.new(:test) do |test|
|
6
|
+
test.libs << 'lib' << 'test'
|
7
|
+
test.pattern = 'test/**/*.rb'
|
8
|
+
test.verbose = true
|
9
|
+
end
|
10
|
+
task :default => :test
|
11
|
+
|
12
|
+
desc 'Open an irb session preloaded with the gem library'
|
13
|
+
task :console do
|
14
|
+
sh 'irb -rubygems -I lib'
|
15
|
+
end
|
16
|
+
task :c => :console
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.name = "fluent-plugin-tail_path"
|
6
|
+
s.version = "0.0.1"
|
7
|
+
s.authors = ["szhem", "Naotoshi Seo"]
|
8
|
+
s.email = ["sonots@gmail.com"]
|
9
|
+
s.homepage = "https://github.com/sonots/fluent-plugin-tail_path"
|
10
|
+
s.summary = "Fluentd in_tail extension to add `path` field"
|
11
|
+
s.description = s.summary
|
12
|
+
|
13
|
+
s.rubyforge_project = "fluent-plugin-tail_path"
|
14
|
+
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
s.require_paths = ["lib"]
|
19
|
+
|
20
|
+
s.add_runtime_dependency "fluentd", ">= 0.10.45"
|
21
|
+
s.add_development_dependency "rake"
|
22
|
+
s.add_development_dependency "pry"
|
23
|
+
s.add_development_dependency "pry-nav"
|
24
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require 'fluent/plugin/in_tail'
|
2
|
+
|
3
|
+
class Fluent::NewTailPathInput < Fluent::NewTailInput
|
4
|
+
Fluent::Plugin.register_output('in_taiL_path', self)
|
5
|
+
|
6
|
+
config_param :path_key, :string, :default => nil
|
7
|
+
|
8
|
+
def configure(conf)
|
9
|
+
super
|
10
|
+
end
|
11
|
+
|
12
|
+
# Override to add path field
|
13
|
+
def convert_line_to_event(line, es, tail_watcher)
|
14
|
+
begin
|
15
|
+
line.chomp! # remove \n
|
16
|
+
time, record = parse_line(line)
|
17
|
+
if time && record
|
18
|
+
record[@path_key] ||= tail_watcher.path unless @path_key.nil?
|
19
|
+
es.add(time, record)
|
20
|
+
else
|
21
|
+
log.warn "pattern not match: #{line.inspect}"
|
22
|
+
end
|
23
|
+
rescue => e
|
24
|
+
log.warn line.dump, :error => e.to_s
|
25
|
+
log.debug_backtrace(e)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# Override to pass tail_watcher to convert_line_to_event
|
30
|
+
def parse_singleline(lines, tail_watcher)
|
31
|
+
es = ::Fluent::MultiEventStream.new
|
32
|
+
lines.each { |line|
|
33
|
+
convert_line_to_event(line, es, tail_watcher)
|
34
|
+
}
|
35
|
+
es
|
36
|
+
end
|
37
|
+
|
38
|
+
# Override to pass tail_watcher to convert_line_to_event
|
39
|
+
def parse_multilines(lines, tail_watcher)
|
40
|
+
lb = tail_watcher.line_buffer
|
41
|
+
es = ::Fluent::MultiEventStream.new
|
42
|
+
lines.each { |line|
|
43
|
+
if @parser.parser.firstline?(line)
|
44
|
+
if lb
|
45
|
+
convert_line_to_event(lb, es, tail_watcher)
|
46
|
+
end
|
47
|
+
lb = line
|
48
|
+
else
|
49
|
+
if lb.nil?
|
50
|
+
log.warn "got incomplete line before first line from #{tail_watcher.path}: #{lb.inspect}"
|
51
|
+
else
|
52
|
+
lb << line
|
53
|
+
end
|
54
|
+
end
|
55
|
+
}
|
56
|
+
tail_watcher.line_buffer = lb
|
57
|
+
es
|
58
|
+
end
|
59
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
require 'fluent/plugin/in_tail_path'
|
16
|
+
|
17
|
+
class Test::Unit::TestCase
|
18
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
|
3
|
+
class TailPathInputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
FileUtils.rm_rf(TMP_DIR)
|
7
|
+
FileUtils.mkdir_p(TMP_DIR)
|
8
|
+
end
|
9
|
+
|
10
|
+
TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
|
11
|
+
|
12
|
+
COMMON_CONFIG = %[
|
13
|
+
path #{TMP_DIR}/tail.txt
|
14
|
+
tag t1
|
15
|
+
rotate_wait 2s
|
16
|
+
pos_file #{TMP_DIR}/tail.pos
|
17
|
+
]
|
18
|
+
SINGLE_LINE_CONFIG = %[
|
19
|
+
format /(?<message>.*)/
|
20
|
+
]
|
21
|
+
|
22
|
+
def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
|
23
|
+
config = use_common_conf ? COMMON_CONFIG + conf : conf
|
24
|
+
Fluent::Test::InputTestDriver.new(Fluent::NewTailPathInput).configure(config)
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_path_key
|
28
|
+
File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
|
29
|
+
|
30
|
+
d = create_driver(%[
|
31
|
+
path #{TMP_DIR}/tail.txt
|
32
|
+
tag t1
|
33
|
+
format /(?<message>.*)/
|
34
|
+
path_key foobar
|
35
|
+
], false)
|
36
|
+
|
37
|
+
d.run do
|
38
|
+
sleep 1
|
39
|
+
|
40
|
+
File.open("#{TMP_DIR}/tail.txt", "a") {|f|
|
41
|
+
f.puts "test1"
|
42
|
+
f.puts "test2"
|
43
|
+
}
|
44
|
+
sleep 1
|
45
|
+
end
|
46
|
+
|
47
|
+
emits = d.emits
|
48
|
+
assert_equal(true, emits.length > 0)
|
49
|
+
assert_equal({"message"=>"test1", "foobar"=>"#{TMP_DIR}/tail.txt"}, emits[0][2])
|
50
|
+
assert_equal({"message"=>"test2", "foobar"=>"#{TMP_DIR}/tail.txt"}, emits[1][2])
|
51
|
+
end
|
52
|
+
end
|
metadata
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-tail_path
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- szhem
|
8
|
+
- Naotoshi Seo
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2014-03-28 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fluentd
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - '>='
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: 0.10.45
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - '>='
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: 0.10.45
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: rake
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - '>='
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0'
|
35
|
+
type: :development
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - '>='
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: pry
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - '>='
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0'
|
49
|
+
type: :development
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - '>='
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: pry-nav
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - '>='
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
description: Fluentd in_tail extension to add `path` field
|
71
|
+
email:
|
72
|
+
- sonots@gmail.com
|
73
|
+
executables: []
|
74
|
+
extensions: []
|
75
|
+
extra_rdoc_files: []
|
76
|
+
files:
|
77
|
+
- .gitignore
|
78
|
+
- .travis.yml
|
79
|
+
- CHANGELOG.md
|
80
|
+
- Gemfile
|
81
|
+
- LICENSE
|
82
|
+
- README.md
|
83
|
+
- Rakefile
|
84
|
+
- fluent-plugin-tail_path.gemspec
|
85
|
+
- lib/fluent/plugin/in_tail_path.rb
|
86
|
+
- test/helper.rb
|
87
|
+
- test/plugin/in_tail_path.rb
|
88
|
+
homepage: https://github.com/sonots/fluent-plugin-tail_path
|
89
|
+
licenses: []
|
90
|
+
metadata: {}
|
91
|
+
post_install_message:
|
92
|
+
rdoc_options: []
|
93
|
+
require_paths:
|
94
|
+
- lib
|
95
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
96
|
+
requirements:
|
97
|
+
- - '>='
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: '0'
|
100
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - '>='
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
requirements: []
|
106
|
+
rubyforge_project: fluent-plugin-tail_path
|
107
|
+
rubygems_version: 2.0.3
|
108
|
+
signing_key:
|
109
|
+
specification_version: 4
|
110
|
+
summary: Fluentd in_tail extension to add `path` field
|
111
|
+
test_files:
|
112
|
+
- test/helper.rb
|
113
|
+
- test/plugin/in_tail_path.rb
|