fluent-plugin-elb-log 0.1.4 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +66 -30
- data/fluent-plugin-elb-log.gemspec +4 -6
- data/lib/fluent/plugin/in_elb_log.rb +189 -108
- metadata +19 -59
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 966e83a0c3cce1f14ff40f3a54a4e6c5cc3c88fd
|
4
|
+
data.tar.gz: 0c6319930698bc00048dfd3ebe50f7c0904086b1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 595c8db95152913481d55f0edd42730dade97d9f5d68c1943ffda3d9d6677360aafff78548c680d0c794e0137013d76a9d1a26ff4d8bfa07162013c0369624e2
|
7
|
+
data.tar.gz: 157c1301e07758ae19abbf742a32c3350a0d39d3c390fb5c7b195330e41665d26d3fbecc63f72b83bcbfd0ee3c5bcc781262fe20b060446915e3ea47cfdfa825
|
data/README.md
CHANGED
@@ -11,6 +11,32 @@
|
|
11
11
|
- settings see: [Elastic Load Balancing](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-access-logs.html)
|
12
12
|
- developer guide: [](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html)
|
13
13
|
|
14
|
+
## Different from version 0.1.x
|
15
|
+
- Using version 2 of the AWS SDK for Ruby.
|
16
|
+
- add parameter
|
17
|
+
- region (required. see:http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region)
|
18
|
+
- tag (optional)
|
19
|
+
- remove parameter
|
20
|
+
- s3_endpoint
|
21
|
+
- output data change
|
22
|
+
- add items
|
23
|
+
- region (AWS Region name)
|
24
|
+
- key (source S3 object name)
|
25
|
+
- prefix
|
26
|
+
- elb_timestamp_unixtime (converted to unix timestamp)
|
27
|
+
- user_agent
|
28
|
+
- option1, option2, option3
|
29
|
+
|
30
|
+
## When SSL certification error
|
31
|
+
log:
|
32
|
+
```
|
33
|
+
SSL_connect returned=1 errno=0 state=SSLv3 read server certificate B: certificate verify failed
|
34
|
+
```
|
35
|
+
Do env setting follows:
|
36
|
+
```
|
37
|
+
SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
|
38
|
+
```
|
39
|
+
|
14
40
|
## Configuration
|
15
41
|
|
16
42
|
```config
|
@@ -18,12 +44,13 @@
|
|
18
44
|
type elb_log
|
19
45
|
|
20
46
|
# following attibutes are required
|
21
|
-
|
47
|
+
region <region name>
|
22
48
|
s3_bucketname <bucketname>
|
23
49
|
s3_prefix <elb log's prefix>
|
24
50
|
timestamp_file <proc last file timestamp record filename>
|
25
|
-
refresh_interval <interval number by second>
|
26
51
|
buf_file <buffer file path>
|
52
|
+
refresh_interval <interval number by second>
|
53
|
+
tag <tag name(default: elb.access)>
|
27
54
|
|
28
55
|
# following attibutes are required if you don't use IAM Role
|
29
56
|
access_key_id <access_key>
|
@@ -35,14 +62,15 @@
|
|
35
62
|
```config
|
36
63
|
<source>
|
37
64
|
type elb_log
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
s3_bucketname bucketname
|
42
|
-
s3_prefix prefix
|
65
|
+
region us-east-1
|
66
|
+
s3_bucketname my-elblog-bucket
|
67
|
+
s3_prefix prodcution/web
|
43
68
|
timestamp_file elb_last_at.dat
|
44
|
-
refresh_interval 300
|
45
69
|
buf_file /tmp/fluentd-elblog.tmpfile
|
70
|
+
refresh_interval 300
|
71
|
+
tag elb.access
|
72
|
+
access_key_id XXXXXXXXXXXXXXXXXXXX
|
73
|
+
secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
46
74
|
</source>
|
47
75
|
|
48
76
|
<match **>
|
@@ -53,28 +81,36 @@
|
|
53
81
|
### json output example
|
54
82
|
```
|
55
83
|
{
|
56
|
-
"account_id":
|
57
|
-
"
|
58
|
-
"
|
59
|
-
"
|
60
|
-
"
|
61
|
-
"
|
62
|
-
"
|
63
|
-
"
|
64
|
-
"
|
65
|
-
"
|
66
|
-
"
|
67
|
-
"
|
68
|
-
"
|
69
|
-
"
|
70
|
-
"
|
71
|
-
"
|
72
|
-
"request_processing_time":
|
73
|
-
"
|
74
|
-
"
|
75
|
-
"
|
76
|
-
"
|
77
|
-
"
|
84
|
+
"account_id":"123456789012",
|
85
|
+
"region":"ap-northeast-1",
|
86
|
+
"logfile_date":"2015/06/15",
|
87
|
+
"logfile_elb_name":"my-elb-name",
|
88
|
+
"elb_ip_address":"52.0.0.0",
|
89
|
+
"logfile_hash":"12squv5w",
|
90
|
+
"elb_timestamp":"20150615T0400Z",
|
91
|
+
"key":"TEST/AWSLogs/123456789012/elasticloadbalancing/ap-northeast-1/2015/06/15/123456789012_elasticloadbalancing_ap-northeast-1_my-elb-name_20150615T0400Z_52.68.215.138_69squv5w.log",
|
92
|
+
"prefix":"TEST",
|
93
|
+
"elb_timestamp_unixtime":1434340800,
|
94
|
+
"time":"2015-06-15T03:47:12.728427+0000",
|
95
|
+
"elb":"my-elb-name",
|
96
|
+
"client":"54.1.1.1",
|
97
|
+
"client_port":"43759",
|
98
|
+
"backend":"10.0.0.1",
|
99
|
+
"backend_port":"80",
|
100
|
+
"request_processing_time":4.0e-05,
|
101
|
+
"backend_processing_time":0.105048,
|
102
|
+
"response_processing_time":2.4e-05,
|
103
|
+
"elb_status_code":"200",
|
104
|
+
"backend_status_code":"200",
|
105
|
+
"received_bytes":0,
|
106
|
+
"sent_bytes":4622,
|
107
|
+
"request_method":"GET",
|
108
|
+
"request_uri":"https://my-elb-test.example.com/",
|
109
|
+
"request_protocol":"HTTP/1.1",
|
110
|
+
"user_agent":"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
|
111
|
+
"option1":"DHE-RSA-AES128-SHA",
|
112
|
+
"option2":"TLSv1.2",
|
113
|
+
"option3":null
|
78
114
|
}
|
79
115
|
```
|
80
116
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-elb-log"
|
7
|
-
spec.version = "0.
|
7
|
+
spec.version = "0.2.0"
|
8
8
|
spec.authors = ["shinsaka"]
|
9
9
|
spec.email = ["shinx1265@gmail.com"]
|
10
10
|
spec.summary = "Amazon ELB log input plugin"
|
@@ -17,10 +17,8 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
18
|
spec.require_paths = ["lib"]
|
19
19
|
|
20
|
-
spec.add_dependency "fluentd"
|
21
|
-
spec.add_dependency "aws-sdk", "~>
|
20
|
+
spec.add_dependency "fluentd", "~> 0"
|
21
|
+
spec.add_dependency "aws-sdk", "~> 2"
|
22
22
|
|
23
|
-
spec.add_development_dependency "bundler", "~> 1.
|
24
|
-
spec.add_development_dependency "rake"
|
25
|
-
spec.add_development_dependency "webmock"
|
23
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
26
24
|
end
|
@@ -2,13 +2,14 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
2
2
|
Fluent::Plugin.register_input('elb_log', self)
|
3
3
|
|
4
4
|
LOGFILE_REGEXP = /^((?<prefix>.+?)\/|)AWSLogs\/(?<account_id>[0-9]{12})\/elasticloadbalancing\/(?<region>.+?)\/(?<logfile_date>[0-9]{4}\/[0-9]{2}\/[0-9]{2})\/[0-9]{12}_elasticloadbalancing_.+?_(?<logfile_elb_name>[^_]+)_(?<elb_timestamp>[0-9]{8}T[0-9]{4}Z)_(?<elb_ip_address>.+?)_(?<logfile_hash>.+)\.log$/
|
5
|
-
ACCESSLOG_REGEXP = /^(?<time>\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z) (?<elb>.+?) (?<client
|
5
|
+
ACCESSLOG_REGEXP = /^(?<time>\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z) (?<elb>.+?) (?<client>.+?)\:(?<client_port>.+?) (?<backend>.+?)\:(?<backend_port>.+?) (?<request_processing_time>.+?) (?<backend_processing_time>.+?) (?<response_processing_time>.+?) (?<elb_status_code>.+?) (?<backend_status_code>.+?) (?<received_bytes>.+?) (?<sent_bytes>.+?) \"(?<request_method>.+?) (?<request_uri>.+?) (?<request_protocol>.+?)\" \"(?<user_agent>.*?)\" (?<option1>.+?) (?<option2>.+)(| (?<option3>.*))/
|
6
6
|
|
7
7
|
config_param :access_key_id, :string, :default => nil
|
8
8
|
config_param :secret_access_key, :string, :default => nil
|
9
|
+
config_param :region, :string, :default => nil
|
9
10
|
config_param :s3_bucketname, :string, :default => nil
|
10
11
|
config_param :s3_prefix, :string, :default => nil
|
11
|
-
config_param :
|
12
|
+
config_param :tag, :string, :default => 'elb.access'
|
12
13
|
config_param :timestamp_file, :string, :default => nil
|
13
14
|
config_param :refresh_interval, :integer, :default => 300
|
14
15
|
config_param :buf_file, :string, :default => './fluentd_elb_log_buf_file'
|
@@ -18,24 +19,23 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
18
19
|
super
|
19
20
|
require 'aws-sdk'
|
20
21
|
|
21
|
-
|
22
|
-
|
22
|
+
raise Fluent::ConfigError.new("region is required") unless @region
|
23
|
+
if !has_iam_role?
|
24
|
+
raise Fluent::ConfigError.new("access_key_id is required") if @access_key_id.nil?
|
25
|
+
raise Fluent::ConfigError.new("secret_access_key is required") if @secret_access_key.nil?
|
23
26
|
end
|
24
|
-
if @secret_access_key.nil? and has_not_iam_role?
|
25
|
-
raise Fluent::ConfigError.new("secret_access_key is required")
|
26
|
-
end
|
27
|
-
|
28
27
|
raise Fluent::ConfigError.new("s3_bucketname is required") unless @s3_bucketname
|
29
28
|
raise Fluent::ConfigError.new("timestamp_file is required") unless @timestamp_file
|
30
|
-
raise Fluent::ConfigError.new("s3 bucket fetch error #{@s3_bucketname}") if init_s3bucket.nil?
|
31
|
-
FileUtils.touch(@buf_file)
|
32
29
|
end
|
33
30
|
|
34
31
|
def start
|
35
32
|
super
|
36
33
|
|
37
|
-
|
38
|
-
@timestamp_file.
|
34
|
+
# files touch
|
35
|
+
File.open(@timestamp_file, File::RDWR|File::CREAT).close
|
36
|
+
File.open(@buf_file, File::RDWR|File::CREAT).close
|
37
|
+
|
38
|
+
raise StandardError.new("s3 bucket not found #{@s3_bucketname}") unless s3bucket_is_ok()
|
39
39
|
|
40
40
|
@loop = Coolio::Loop.new
|
41
41
|
timer_trigger = TimerWatcher.new(@refresh_interval, true, &method(:input))
|
@@ -47,142 +47,223 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
47
47
|
super
|
48
48
|
@loop.stop
|
49
49
|
@thread.join
|
50
|
-
@timestamp_file.close
|
51
50
|
end
|
52
51
|
|
53
52
|
private
|
54
53
|
|
55
|
-
def
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
options[:s3_endpoint] = @s3_endpoint if @s3_endpoint
|
62
|
-
if @proxy_uri
|
63
|
-
options[:proxy_uri] = @proxy_uri
|
54
|
+
def has_iam_role?
|
55
|
+
begin
|
56
|
+
ec2 = Aws::EC2::Client.new(region: @region)
|
57
|
+
!ec2.config.credentials.nil?
|
58
|
+
rescue => e
|
59
|
+
$log.warn "EC2 Client error occurred: #{e.message}"
|
64
60
|
end
|
61
|
+
end
|
65
62
|
|
63
|
+
def get_timestamp_file
|
66
64
|
begin
|
67
|
-
|
68
|
-
|
65
|
+
# get timestamp last proc
|
66
|
+
timestamp = 0
|
67
|
+
$log.debug "timestamp file #{@timestamp_file} read"
|
68
|
+
File.open(@timestamp_file, File::RDONLY) do |file|
|
69
|
+
timestamp = file.read.to_i
|
70
|
+
end
|
71
|
+
$log.debug "timestamp start at:" + Time.at(timestamp).to_s
|
72
|
+
return timestamp
|
69
73
|
rescue => e
|
70
|
-
$log.warn "
|
71
|
-
nil
|
74
|
+
$log.warn "timestamp file get and parse error occurred: #{e.message}"
|
72
75
|
end
|
73
76
|
end
|
74
77
|
|
75
|
-
def
|
76
|
-
|
78
|
+
def put_timestamp_file(timestamp)
|
79
|
+
begin
|
80
|
+
$log.debug "timestamp file #{@timestamp_file} write"
|
81
|
+
File.open(@timestamp_file, File::WRONLY|File::CREAT|File::TRUNC) do |file|
|
82
|
+
file.puts timestamp.to_s
|
83
|
+
end
|
84
|
+
rescue => e
|
85
|
+
$log.warn "timestamp file get and parse error occurred: #{e.message}"
|
86
|
+
end
|
77
87
|
end
|
78
88
|
|
79
|
-
def
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
next unless matches
|
93
|
-
next if !@s3_prefix.nil? && matches[:prefix] != @s3_prefix
|
94
|
-
log_objects.push obj
|
89
|
+
def s3_client
|
90
|
+
begin
|
91
|
+
options = {
|
92
|
+
:region => @region,
|
93
|
+
}
|
94
|
+
if @access_key_id && @secret_access_key
|
95
|
+
options[:access_key_id] = @access_key_id
|
96
|
+
options[:secret_access_key] = @secret_access_key
|
97
|
+
end
|
98
|
+
$log.debug "S3 client connect"
|
99
|
+
Aws::S3::Client.new(options)
|
100
|
+
rescue => e
|
101
|
+
$log.warn "S3 Client error occurred: #{e.message}"
|
95
102
|
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def s3bucket_is_ok
|
106
|
+
begin
|
107
|
+
$log.debug "search bucket #{@s3_bucketname}"
|
96
108
|
|
97
|
-
|
98
|
-
|
99
|
-
|
109
|
+
s3_client.list_buckets.buckets.any? do |bucket|
|
110
|
+
bucket.name == @s3_bucketname
|
111
|
+
end
|
112
|
+
rescue => e
|
113
|
+
$log.warn "S3 Client error occurred: #{e.message}"
|
100
114
|
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def input
|
118
|
+
$log.debug "start"
|
119
|
+
|
120
|
+
timestamp = get_timestamp_file()
|
121
|
+
|
122
|
+
object_keys = get_object_keys(timestamp)
|
123
|
+
object_keys = sort_object_key(object_keys)
|
124
|
+
|
125
|
+
$log.info "processing #{object_keys.count} object(s)."
|
101
126
|
|
102
|
-
|
103
|
-
matches = LOGFILE_REGEXP.match(obj.key)
|
104
|
-
timestamp = matches[:elb_timestamp].to_i
|
127
|
+
object_keys.each do |object_key|
|
105
128
|
record_common = {
|
106
|
-
"account_id" =>
|
107
|
-
"region" =>
|
108
|
-
"logfile_date" =>
|
109
|
-
"logfile_elb_name" =>
|
110
|
-
"elb_ip_address" =>
|
111
|
-
"logfile_hash" =>
|
112
|
-
"elb_timestamp" =>
|
129
|
+
"account_id" => object_key[:account_id],
|
130
|
+
"region" => object_key[:region],
|
131
|
+
"logfile_date" => object_key[:logfile_date],
|
132
|
+
"logfile_elb_name" => object_key[:logfile_elb_name],
|
133
|
+
"elb_ip_address" => object_key[:elb_ip_address],
|
134
|
+
"logfile_hash" => object_key[:logfile_hash],
|
135
|
+
"elb_timestamp" => object_key[:elb_timestamp],
|
136
|
+
"key" => object_key[:key],
|
137
|
+
"prefix" => object_key[:prefix],
|
138
|
+
"elb_timestamp_unixtime" => object_key[:elb_timestamp_unixtime],
|
113
139
|
}
|
114
140
|
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
141
|
+
get_file_from_s3(object_key[:key])
|
142
|
+
emit_lines_from_buffer_file(record_common)
|
143
|
+
|
144
|
+
put_timestamp_file(object_key[:elb_timestamp_unixtime])
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
def get_object_keys(timestamp)
|
149
|
+
# get values from object file name
|
150
|
+
begin
|
151
|
+
object_keys = []
|
152
|
+
|
153
|
+
objects = s3_client.list_objects(
|
154
|
+
bucket: @s3_bucketname,
|
155
|
+
max_keys: 100,
|
156
|
+
prefix: @s3_prefix,
|
157
|
+
)
|
158
|
+
|
159
|
+
objects.each do |object|
|
160
|
+
object.contents.each do |content|
|
161
|
+
matches = LOGFILE_REGEXP.match(content.key)
|
162
|
+
next unless matches
|
121
163
|
|
122
|
-
|
164
|
+
# snip old items
|
165
|
+
elb_timestamp_unixtime = Time.parse(matches[:elb_timestamp]).to_i
|
166
|
+
next if elb_timestamp_unixtime <= timestamp
|
123
167
|
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
168
|
+
$log.debug content.key
|
169
|
+
object_keys << {
|
170
|
+
key: content.key,
|
171
|
+
prefix: matches[:prefix],
|
172
|
+
account_id: matches[:account_id],
|
173
|
+
region: matches[:region],
|
174
|
+
logfile_date: matches[:logfile_date],
|
175
|
+
logfile_elb_name: matches[:logfile_elb_name],
|
176
|
+
elb_timestamp: matches[:elb_timestamp],
|
177
|
+
elb_ip_address: matches[:elb_ip_address],
|
178
|
+
logfile_hash: matches[:logfile_hash],
|
179
|
+
elb_timestamp_unixtime: elb_timestamp_unixtime,
|
180
|
+
}
|
181
|
+
end
|
182
|
+
end
|
183
|
+
return object_keys
|
184
|
+
rescue => e
|
185
|
+
$log.warn "error occurred: #{e.message}"
|
129
186
|
end
|
130
187
|
end
|
131
188
|
|
132
|
-
def
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
line_match = ACCESSLOG_REGEXP.match(line)
|
137
|
-
next unless line_match
|
138
|
-
|
139
|
-
record = {
|
140
|
-
"time" => line_match[:time].gsub(/Z/, "+0000"),
|
141
|
-
"elb" => line_match[:elb],
|
142
|
-
"client" => line_match[:client],
|
143
|
-
"client_port" => line_match[:client_port],
|
144
|
-
"backend" => line_match[:backend],
|
145
|
-
"backend_port" => line_match[:backend_port],
|
146
|
-
"request_processing_time" => line_match[:request_processing_time].to_f,
|
147
|
-
"backend_processing_time" => line_match[:backend_processing_time].to_f,
|
148
|
-
"response_processing_time" => line_match[:response_processing_time].to_f,
|
149
|
-
"elb_status_code" => line_match[:elb_status_code],
|
150
|
-
"backend_status_code" => line_match[:backend_status_code],
|
151
|
-
"received_bytes" => line_match[:received_bytes].to_i,
|
152
|
-
"sent_bytes" => line_match[:sent_bytes].to_i,
|
153
|
-
"request_method" => line_match[:request_method],
|
154
|
-
"request_uri" => line_match[:request_uri],
|
155
|
-
"request_protocol" => line_match[:request_protocol],
|
156
|
-
}
|
157
|
-
|
158
|
-
Fluent::Engine.emit("elb.access", Fluent::Engine.now, record_common.merge(record))
|
189
|
+
def sort_object_key(src_object_keys)
|
190
|
+
begin
|
191
|
+
src_object_keys.sort do |a, b|
|
192
|
+
a[:elb_timestamp_unixtime] <=> b[:elb_timestamp_unixtime]
|
159
193
|
end
|
194
|
+
rescue => e
|
195
|
+
$log.warn "error occurred: #{e.message}"
|
160
196
|
end
|
161
197
|
end
|
162
198
|
|
163
|
-
def
|
164
|
-
|
199
|
+
def get_file_from_s3(object_name)
|
200
|
+
begin
|
201
|
+
$log.debug "getting object from s3 name is #{object_name}"
|
165
202
|
|
166
|
-
|
167
|
-
|
203
|
+
# read an object from S3 to a file and write buffer file
|
204
|
+
File.open(@buf_file, File::WRONLY|File::CREAT|File::TRUNC) do |file|
|
205
|
+
s3_client.get_object(
|
206
|
+
bucket: @s3_bucketname,
|
207
|
+
key: object_name
|
208
|
+
) do |chunk|
|
209
|
+
file.write(chunk)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
rescue => e
|
213
|
+
$log.warn "error occurred: #{e.message}"
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def emit_lines_from_buffer_file(record_common)
|
168
218
|
begin
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
219
|
+
# emit per line
|
220
|
+
File.open(@buf_file, File::RDONLY) do |file|
|
221
|
+
file.each_line do |line|
|
222
|
+
line_match = ACCESSLOG_REGEXP.match(line)
|
223
|
+
unless line_match
|
224
|
+
$log.info "nomatch log found: #{line} in #{record_common['key']}"
|
225
|
+
next
|
226
|
+
end
|
227
|
+
|
228
|
+
record = {
|
229
|
+
"time" => line_match[:time].gsub(/Z/, "+0000"),
|
230
|
+
"elb" => line_match[:elb],
|
231
|
+
"client" => line_match[:client],
|
232
|
+
"client_port" => line_match[:client_port],
|
233
|
+
"backend" => line_match[:backend],
|
234
|
+
"backend_port" => line_match[:backend_port],
|
235
|
+
"request_processing_time" => line_match[:request_processing_time].to_f,
|
236
|
+
"backend_processing_time" => line_match[:backend_processing_time].to_f,
|
237
|
+
"response_processing_time" => line_match[:response_processing_time].to_f,
|
238
|
+
"elb_status_code" => line_match[:elb_status_code],
|
239
|
+
"backend_status_code" => line_match[:backend_status_code],
|
240
|
+
"received_bytes" => line_match[:received_bytes].to_i,
|
241
|
+
"sent_bytes" => line_match[:sent_bytes].to_i,
|
242
|
+
"request_method" => line_match[:request_method],
|
243
|
+
"request_uri" => line_match[:request_uri],
|
244
|
+
"request_protocol" => line_match[:request_protocol],
|
245
|
+
"user_agent" => line_match[:user_agent],
|
246
|
+
"option1" => line_match[:option1],
|
247
|
+
"option2" => line_match[:option2],
|
248
|
+
"option3" => line_match[:option3],
|
249
|
+
}
|
250
|
+
|
251
|
+
Fluent::Engine.emit(@tag, Fluent::Engine.now, record_common.merge(record))
|
252
|
+
end
|
253
|
+
end
|
173
254
|
rescue => e
|
174
|
-
$log.warn "
|
255
|
+
$log.warn "error occurred: #{e.message}"
|
175
256
|
end
|
176
|
-
@has_iam_role
|
177
257
|
end
|
178
258
|
|
179
|
-
def
|
180
|
-
|
259
|
+
def run
|
260
|
+
@loop.run
|
181
261
|
end
|
182
262
|
|
183
263
|
class TimerWatcher < Coolio::TimerWatcher
|
184
264
|
def initialize(interval, repeat, &callback)
|
185
265
|
@callback = callback
|
266
|
+
on_timer # first call
|
186
267
|
super(interval, repeat)
|
187
268
|
end
|
188
269
|
|
metadata
CHANGED
@@ -1,96 +1,57 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elb-log
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.2.0
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- shinsaka
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2015-
|
11
|
+
date: 2015-06-16 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: fluentd
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - "~>"
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: '0'
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - "~>"
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: '0'
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: aws-sdk
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- - ~>
|
31
|
+
- - "~>"
|
36
32
|
- !ruby/object:Gem::Version
|
37
|
-
version: '
|
33
|
+
version: '2'
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- - ~>
|
38
|
+
- - "~>"
|
44
39
|
- !ruby/object:Gem::Version
|
45
|
-
version: '
|
40
|
+
version: '2'
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: bundler
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- - ~>
|
45
|
+
- - "~>"
|
52
46
|
- !ruby/object:Gem::Version
|
53
|
-
version: '1.
|
47
|
+
version: '1.7'
|
54
48
|
type: :development
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- - ~>
|
52
|
+
- - "~>"
|
60
53
|
- !ruby/object:Gem::Version
|
61
|
-
version: '1.
|
62
|
-
- !ruby/object:Gem::Dependency
|
63
|
-
name: rake
|
64
|
-
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
|
-
requirements:
|
67
|
-
- - ! '>='
|
68
|
-
- !ruby/object:Gem::Version
|
69
|
-
version: '0'
|
70
|
-
type: :development
|
71
|
-
prerelease: false
|
72
|
-
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
|
-
requirements:
|
75
|
-
- - ! '>='
|
76
|
-
- !ruby/object:Gem::Version
|
77
|
-
version: '0'
|
78
|
-
- !ruby/object:Gem::Dependency
|
79
|
-
name: webmock
|
80
|
-
requirement: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
|
-
requirements:
|
83
|
-
- - ! '>='
|
84
|
-
- !ruby/object:Gem::Version
|
85
|
-
version: '0'
|
86
|
-
type: :development
|
87
|
-
prerelease: false
|
88
|
-
version_requirements: !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
|
-
requirements:
|
91
|
-
- - ! '>='
|
92
|
-
- !ruby/object:Gem::Version
|
93
|
-
version: '0'
|
54
|
+
version: '1.7'
|
94
55
|
description: Amazon ELB log input plugin for fluentd
|
95
56
|
email:
|
96
57
|
- shinx1265@gmail.com
|
@@ -98,8 +59,8 @@ executables: []
|
|
98
59
|
extensions: []
|
99
60
|
extra_rdoc_files: []
|
100
61
|
files:
|
101
|
-
- .gitignore
|
102
|
-
- .travis.yml
|
62
|
+
- ".gitignore"
|
63
|
+
- ".travis.yml"
|
103
64
|
- Gemfile
|
104
65
|
- LICENSE.txt
|
105
66
|
- README.md
|
@@ -112,27 +73,26 @@ files:
|
|
112
73
|
homepage: https://github.com/shinsaka/fluent-plugin-elb-log
|
113
74
|
licenses:
|
114
75
|
- MIT
|
76
|
+
metadata: {}
|
115
77
|
post_install_message:
|
116
78
|
rdoc_options: []
|
117
79
|
require_paths:
|
118
80
|
- lib
|
119
81
|
required_ruby_version: !ruby/object:Gem::Requirement
|
120
|
-
none: false
|
121
82
|
requirements:
|
122
|
-
- -
|
83
|
+
- - ">="
|
123
84
|
- !ruby/object:Gem::Version
|
124
85
|
version: '0'
|
125
86
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
126
|
-
none: false
|
127
87
|
requirements:
|
128
|
-
- -
|
88
|
+
- - ">="
|
129
89
|
- !ruby/object:Gem::Version
|
130
90
|
version: '0'
|
131
91
|
requirements: []
|
132
92
|
rubyforge_project:
|
133
|
-
rubygems_version:
|
93
|
+
rubygems_version: 2.2.1
|
134
94
|
signing_key:
|
135
|
-
specification_version:
|
95
|
+
specification_version: 4
|
136
96
|
summary: Amazon ELB log input plugin
|
137
97
|
test_files:
|
138
98
|
- test/helper.rb
|