fluent-plugin-elb-log 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +9 -0
- data/Rakefile +10 -0
- data/fluent-plugin-elb-log.gemspec +2 -1
- data/lib/fluent/plugin/in_elb_log.rb +30 -8
- data/test/helper.rb +19 -0
- data/test/plugin/in_elb_log.rb +102 -0
- metadata +34 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 758e11bd1e39f44c8f3fa111370b4a6c98f47234
|
4
|
+
data.tar.gz: 1d175b06fabd4016957b5093627c948174b7d1e2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7d603a8276abf6dae8990dd30becde23eb2e1749e621e04d0699cb3ba199c14d62026547855744770c51cbf475cc21f9a3a7ac4cc2bc83fcdf581baee93574a3
|
7
|
+
data.tar.gz: 87062acf6b2809effaa29f8df2db6d26c987511a9452b625b662ed4ee2ee6504a920c06b698f203b744d3653dd4b9bc9cd61c74c00acb2f70b2ddcf1b89d77d3
|
data/.travis.yml
ADDED
data/Rakefile
CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-elb-log"
|
7
|
-
spec.version = "0.0.
|
7
|
+
spec.version = "0.0.6"
|
8
8
|
spec.authors = ["shinsaka"]
|
9
9
|
spec.email = ["shinx1265@gmail.com"]
|
10
10
|
spec.summary = "Amazon ELB log input plugin"
|
@@ -22,4 +22,5 @@ Gem::Specification.new do |spec|
|
|
22
22
|
|
23
23
|
spec.add_development_dependency "bundler", "~> 1.5"
|
24
24
|
spec.add_development_dependency "rake"
|
25
|
+
spec.add_development_dependency "webmock"
|
25
26
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
class Fluent::Elb_LogInput < Fluent::Input
|
2
2
|
Fluent::Plugin.register_input('elb_log', self)
|
3
|
-
|
3
|
+
|
4
4
|
LOGFILE_REGEXP = /^((?<prefix>.+?)\/|)AWSLogs\/(?<account_id>[0-9]{12})\/elasticloadbalancing\/(?<region>.+?)\/(?<logfile_date>[0-9]{4}\/[0-9]{2}\/[0-9]{2})\/[0-9]{12}_elasticloadbalancing_.+?_(?<logfile_elb_name>[^_]+)_(?<elb_timestamp>[0-9]{8}T[0-9]{4}Z)_(?<elb_ip_address>.+?)_(?<logfile_hash>.+)\.log$/
|
5
5
|
ACCESSLOG_REGEXP = /^(?<time>.+?) (?<elb>.+?) (?<client>.+)\:(?<client_port>.+) (?<backend>.+)\:(?<backend_port>.+) (?<request_processing_time>.+?) (?<backend_processing_time>.+?) (?<response_processing_time>.+?) (?<elb_status_code>.+?) (?<backend_status_code>.+?) (?<received_bytes>.+?) (?<sent_bytes>.+?) \"(?<request_method>.+?) (?<request_uri>.+?) (?<request_protocol>.+?)\"$/
|
6
6
|
|
@@ -15,12 +15,14 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
15
15
|
def configure(conf)
|
16
16
|
super
|
17
17
|
require 'aws-sdk'
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
18
|
+
|
19
|
+
if @access_key_id.nil? and has_not_iam_role?
|
20
|
+
raise Fluent::ConfigError.new("access_key_id is required")
|
21
|
+
end
|
22
|
+
if @secret_access_key.nil? and has_not_iam_role?
|
23
|
+
raise Fluent::ConfigError.new("secret_access_key is required")
|
23
24
|
end
|
25
|
+
|
24
26
|
raise Fluent::ConfigError.new("s3_bucketname is required") unless @s3_bucketname
|
25
27
|
raise Fluent::ConfigError.new("timestamp_file is required") unless @timestamp_file
|
26
28
|
end
|
@@ -65,7 +67,7 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
65
67
|
|
66
68
|
def input
|
67
69
|
$log.info "fluent-plugin-elb-log: input start"
|
68
|
-
|
70
|
+
|
69
71
|
# get timestamp last proc
|
70
72
|
@timestamp_file.rewind
|
71
73
|
timestamp = @timestamp_file.read.to_i
|
@@ -120,7 +122,7 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
120
122
|
request_uri: line_match[:request_uri],
|
121
123
|
request_protocol: line_match[:request_protocol],
|
122
124
|
}
|
123
|
-
|
125
|
+
|
124
126
|
Fluent::Engine.emit("elb.access", Fluent::Engine.now, record_common.merge(record))
|
125
127
|
end
|
126
128
|
# timestamp save
|
@@ -131,6 +133,26 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
131
133
|
end
|
132
134
|
end
|
133
135
|
|
136
|
+
def has_iam_role?
|
137
|
+
return @has_iam_role unless @has_iam_role.nil?
|
138
|
+
|
139
|
+
require 'net/http'
|
140
|
+
@has_iam_role = false
|
141
|
+
begin
|
142
|
+
http = Net::HTTP.new('169.254.169.254', '80')
|
143
|
+
http.open_timeout = 5 # sec
|
144
|
+
response = http.request(Net::HTTP::Get.new('/latest/meta-data/iam/info'))
|
145
|
+
@has_iam_role = true if response.code == '200'
|
146
|
+
rescue => e
|
147
|
+
$log.warn "fluent-plugin-elb-log: #{e.message}"
|
148
|
+
end
|
149
|
+
@has_iam_role
|
150
|
+
end
|
151
|
+
|
152
|
+
def has_not_iam_role?
|
153
|
+
!has_iam_role?
|
154
|
+
end
|
155
|
+
|
134
156
|
class TimerWatcher < Coolio::TimerWatcher
|
135
157
|
def initialize(interval, repeat, &callback)
|
136
158
|
@callback = callback
|
data/test/helper.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
require 'webmock/test_unit'
|
12
|
+
|
13
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
14
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
15
|
+
require 'fluent/test'
|
16
|
+
require 'fluent/plugin/in_elb_log'
|
17
|
+
|
18
|
+
class Test::Unit::TestCase
|
19
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
|
3
|
+
class Elb_LogInputTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def setup
|
6
|
+
Fluent::Test.setup
|
7
|
+
end
|
8
|
+
|
9
|
+
DEFAULT_CONFIG = {
|
10
|
+
:access_key_id => 'dummy_access_key_id',
|
11
|
+
:secret_access_key => 'dummy_secret_access_key',
|
12
|
+
:s3_endpoint => 's3-ap-northeast-1.amazonaws.com',
|
13
|
+
:s3_bucketname => 'bummy_bucket',
|
14
|
+
:s3_prefix => 'test',
|
15
|
+
:timestamp_file => 'elb_last_at.dat',
|
16
|
+
:refresh_interval => 300
|
17
|
+
}
|
18
|
+
|
19
|
+
def parse_config(conf = {})
|
20
|
+
''.tap{|s| conf.each { |k, v| s << "#{k} #{v}\n" } }
|
21
|
+
end
|
22
|
+
|
23
|
+
def create_driver(conf = DEFAULT_CONFIG)
|
24
|
+
Fluent::Test::InputTestDriver.new(Fluent::Elb_LogInput).configure(parse_config conf)
|
25
|
+
end
|
26
|
+
|
27
|
+
def iam_info
|
28
|
+
stub_request(:get, "http://169.254.169.254/latest/meta-data/iam/info")
|
29
|
+
end
|
30
|
+
|
31
|
+
def use_iam_role
|
32
|
+
iam_info.to_return(:status => [200, 'OK'])
|
33
|
+
end
|
34
|
+
|
35
|
+
def not_use_iam_role
|
36
|
+
iam_info.to_return(:status => [404, 'Not Found'])
|
37
|
+
end
|
38
|
+
|
39
|
+
def test_confiture_default
|
40
|
+
use_iam_role
|
41
|
+
assert_nothing_raised { driver = create_driver }
|
42
|
+
|
43
|
+
exception = assert_raise(Fluent::ConfigError) {
|
44
|
+
conf = DEFAULT_CONFIG.clone
|
45
|
+
conf.delete(:s3_bucketname)
|
46
|
+
driver = create_driver(conf)
|
47
|
+
}
|
48
|
+
assert_equal('s3_bucketname is required', exception.message)
|
49
|
+
|
50
|
+
exception = assert_raise(Fluent::ConfigError) {
|
51
|
+
conf = DEFAULT_CONFIG.clone
|
52
|
+
conf.delete(:timestamp_file)
|
53
|
+
driver = create_driver(conf)
|
54
|
+
}
|
55
|
+
assert_equal('timestamp_file is required', exception.message)
|
56
|
+
end
|
57
|
+
|
58
|
+
def test_configure_in_EC2_with_IAM_role
|
59
|
+
use_iam_role
|
60
|
+
conf = DEFAULT_CONFIG.clone
|
61
|
+
conf.delete(:access_key_id)
|
62
|
+
conf.delete(:secret_access_key)
|
63
|
+
assert_nothing_raised { driver = create_driver(conf) }
|
64
|
+
end
|
65
|
+
|
66
|
+
def test_configure_in_EC2_without_IAM_role
|
67
|
+
not_use_iam_role
|
68
|
+
exception = assert_raise(Fluent::ConfigError) {
|
69
|
+
conf = DEFAULT_CONFIG.clone
|
70
|
+
conf.delete(:access_key_id)
|
71
|
+
driver = create_driver(conf)
|
72
|
+
}
|
73
|
+
assert_equal('access_key_id is required', exception.message)
|
74
|
+
|
75
|
+
exception = assert_raise(Fluent::ConfigError) {
|
76
|
+
conf = DEFAULT_CONFIG.clone
|
77
|
+
conf.delete(:secret_access_key)
|
78
|
+
driver = create_driver(conf)
|
79
|
+
}
|
80
|
+
assert_equal('secret_access_key is required', exception.message)
|
81
|
+
end
|
82
|
+
|
83
|
+
def test_configure_outside_EC2
|
84
|
+
iam_info.to_timeout
|
85
|
+
|
86
|
+
assert_nothing_raised { driver = create_driver }
|
87
|
+
exception = assert_raise(Fluent::ConfigError) {
|
88
|
+
conf = DEFAULT_CONFIG.clone
|
89
|
+
conf.delete(:access_key_id)
|
90
|
+
driver = create_driver(conf)
|
91
|
+
}
|
92
|
+
assert_equal('access_key_id is required', exception.message)
|
93
|
+
|
94
|
+
exception = assert_raise(Fluent::ConfigError) {
|
95
|
+
conf = DEFAULT_CONFIG.clone
|
96
|
+
conf.delete(:secret_access_key)
|
97
|
+
driver = create_driver(conf)
|
98
|
+
}
|
99
|
+
assert_equal('secret_access_key is required', exception.message)
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
metadata
CHANGED
@@ -1,69 +1,83 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elb-log
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- shinsaka
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-05-
|
11
|
+
date: 2014-05-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - '>='
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: aws-sdk
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - '>='
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - '>='
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: bundler
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - ~>
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '1.5'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ~>
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '1.5'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: rake
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - '>='
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: webmock
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '>='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - '>='
|
67
81
|
- !ruby/object:Gem::Version
|
68
82
|
version: '0'
|
69
83
|
description: Amazon ELB log input plugin for fluentd
|
@@ -73,7 +87,8 @@ executables: []
|
|
73
87
|
extensions: []
|
74
88
|
extra_rdoc_files: []
|
75
89
|
files:
|
76
|
-
-
|
90
|
+
- .gitignore
|
91
|
+
- .travis.yml
|
77
92
|
- Gemfile
|
78
93
|
- LICENSE.txt
|
79
94
|
- README.md
|
@@ -81,6 +96,8 @@ files:
|
|
81
96
|
- fluent-plugin-elb-log.gemspec
|
82
97
|
- fluent.conf.sample
|
83
98
|
- lib/fluent/plugin/in_elb_log.rb
|
99
|
+
- test/helper.rb
|
100
|
+
- test/plugin/in_elb_log.rb
|
84
101
|
homepage: https://github.com/shinsaka/fluent-plugin-elb-log
|
85
102
|
licenses:
|
86
103
|
- MIT
|
@@ -91,18 +108,20 @@ require_paths:
|
|
91
108
|
- lib
|
92
109
|
required_ruby_version: !ruby/object:Gem::Requirement
|
93
110
|
requirements:
|
94
|
-
- -
|
111
|
+
- - '>='
|
95
112
|
- !ruby/object:Gem::Version
|
96
113
|
version: '0'
|
97
114
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
98
115
|
requirements:
|
99
|
-
- -
|
116
|
+
- - '>='
|
100
117
|
- !ruby/object:Gem::Version
|
101
118
|
version: '0'
|
102
119
|
requirements: []
|
103
120
|
rubyforge_project:
|
104
|
-
rubygems_version: 2.0.
|
121
|
+
rubygems_version: 2.0.14
|
105
122
|
signing_key:
|
106
123
|
specification_version: 4
|
107
124
|
summary: Amazon ELB log input plugin
|
108
|
-
test_files:
|
125
|
+
test_files:
|
126
|
+
- test/helper.rb
|
127
|
+
- test/plugin/in_elb_log.rb
|