fluent-plugin-elb-log 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +66 -30
- data/fluent-plugin-elb-log.gemspec +4 -6
- data/lib/fluent/plugin/in_elb_log.rb +189 -108
- metadata +19 -59
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 966e83a0c3cce1f14ff40f3a54a4e6c5cc3c88fd
|
4
|
+
data.tar.gz: 0c6319930698bc00048dfd3ebe50f7c0904086b1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 595c8db95152913481d55f0edd42730dade97d9f5d68c1943ffda3d9d6677360aafff78548c680d0c794e0137013d76a9d1a26ff4d8bfa07162013c0369624e2
|
7
|
+
data.tar.gz: 157c1301e07758ae19abbf742a32c3350a0d39d3c390fb5c7b195330e41665d26d3fbecc63f72b83bcbfd0ee3c5bcc781262fe20b060446915e3ea47cfdfa825
|
data/README.md
CHANGED
@@ -11,6 +11,32 @@
|
|
11
11
|
- settings see: [Elastic Load Balancing](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-access-logs.html)
|
12
12
|
- developer guide: [](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html)
|
13
13
|
|
14
|
+
## Different from version 0.1.x
|
15
|
+
- Using version 2 of the AWS SDK for Ruby.
|
16
|
+
- add parameter
|
17
|
+
- region (required. see:http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region)
|
18
|
+
- tag (optional)
|
19
|
+
- remove parameter
|
20
|
+
- s3_endpoint
|
21
|
+
- output data change
|
22
|
+
- add items
|
23
|
+
- region (AWS Region name)
|
24
|
+
- key (source S3 object name)
|
25
|
+
- prefix
|
26
|
+
- elb_timestamp_unixtime (converted to unix timestamp)
|
27
|
+
- user_agent
|
28
|
+
- option1, option2, option3
|
29
|
+
|
30
|
+
## When SSL certification error
|
31
|
+
log:
|
32
|
+
```
|
33
|
+
SSL_connect returned=1 errno=0 state=SSLv3 read server certificate B: certificate verify failed
|
34
|
+
```
|
35
|
+
Do env setting follows:
|
36
|
+
```
|
37
|
+
SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
|
38
|
+
```
|
39
|
+
|
14
40
|
## Configuration
|
15
41
|
|
16
42
|
```config
|
@@ -18,12 +44,13 @@
|
|
18
44
|
type elb_log
|
19
45
|
|
20
46
|
# following attibutes are required
|
21
|
-
|
47
|
+
region <region name>
|
22
48
|
s3_bucketname <bucketname>
|
23
49
|
s3_prefix <elb log's prefix>
|
24
50
|
timestamp_file <proc last file timestamp record filename>
|
25
|
-
refresh_interval <interval number by second>
|
26
51
|
buf_file <buffer file path>
|
52
|
+
refresh_interval <interval number by second>
|
53
|
+
tag <tag name(default: elb.access)>
|
27
54
|
|
28
55
|
# following attibutes are required if you don't use IAM Role
|
29
56
|
access_key_id <access_key>
|
@@ -35,14 +62,15 @@
|
|
35
62
|
```config
|
36
63
|
<source>
|
37
64
|
type elb_log
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
s3_bucketname bucketname
|
42
|
-
s3_prefix prefix
|
65
|
+
region us-east-1
|
66
|
+
s3_bucketname my-elblog-bucket
|
67
|
+
s3_prefix prodcution/web
|
43
68
|
timestamp_file elb_last_at.dat
|
44
|
-
refresh_interval 300
|
45
69
|
buf_file /tmp/fluentd-elblog.tmpfile
|
70
|
+
refresh_interval 300
|
71
|
+
tag elb.access
|
72
|
+
access_key_id XXXXXXXXXXXXXXXXXXXX
|
73
|
+
secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
46
74
|
</source>
|
47
75
|
|
48
76
|
<match **>
|
@@ -53,28 +81,36 @@
|
|
53
81
|
### json output example
|
54
82
|
```
|
55
83
|
{
|
56
|
-
"account_id":
|
57
|
-
"
|
58
|
-
"
|
59
|
-
"
|
60
|
-
"
|
61
|
-
"
|
62
|
-
"
|
63
|
-
"
|
64
|
-
"
|
65
|
-
"
|
66
|
-
"
|
67
|
-
"
|
68
|
-
"
|
69
|
-
"
|
70
|
-
"
|
71
|
-
"
|
72
|
-
"request_processing_time":
|
73
|
-
"
|
74
|
-
"
|
75
|
-
"
|
76
|
-
"
|
77
|
-
"
|
84
|
+
"account_id":"123456789012",
|
85
|
+
"region":"ap-northeast-1",
|
86
|
+
"logfile_date":"2015/06/15",
|
87
|
+
"logfile_elb_name":"my-elb-name",
|
88
|
+
"elb_ip_address":"52.0.0.0",
|
89
|
+
"logfile_hash":"12squv5w",
|
90
|
+
"elb_timestamp":"20150615T0400Z",
|
91
|
+
"key":"TEST/AWSLogs/123456789012/elasticloadbalancing/ap-northeast-1/2015/06/15/123456789012_elasticloadbalancing_ap-northeast-1_my-elb-name_20150615T0400Z_52.68.215.138_69squv5w.log",
|
92
|
+
"prefix":"TEST",
|
93
|
+
"elb_timestamp_unixtime":1434340800,
|
94
|
+
"time":"2015-06-15T03:47:12.728427+0000",
|
95
|
+
"elb":"my-elb-name",
|
96
|
+
"client":"54.1.1.1",
|
97
|
+
"client_port":"43759",
|
98
|
+
"backend":"10.0.0.1",
|
99
|
+
"backend_port":"80",
|
100
|
+
"request_processing_time":4.0e-05,
|
101
|
+
"backend_processing_time":0.105048,
|
102
|
+
"response_processing_time":2.4e-05,
|
103
|
+
"elb_status_code":"200",
|
104
|
+
"backend_status_code":"200",
|
105
|
+
"received_bytes":0,
|
106
|
+
"sent_bytes":4622,
|
107
|
+
"request_method":"GET",
|
108
|
+
"request_uri":"https://my-elb-test.example.com/",
|
109
|
+
"request_protocol":"HTTP/1.1",
|
110
|
+
"user_agent":"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
|
111
|
+
"option1":"DHE-RSA-AES128-SHA",
|
112
|
+
"option2":"TLSv1.2",
|
113
|
+
"option3":null
|
78
114
|
}
|
79
115
|
```
|
80
116
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-elb-log"
|
7
|
-
spec.version = "0.
|
7
|
+
spec.version = "0.2.0"
|
8
8
|
spec.authors = ["shinsaka"]
|
9
9
|
spec.email = ["shinx1265@gmail.com"]
|
10
10
|
spec.summary = "Amazon ELB log input plugin"
|
@@ -17,10 +17,8 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
18
|
spec.require_paths = ["lib"]
|
19
19
|
|
20
|
-
spec.add_dependency "fluentd"
|
21
|
-
spec.add_dependency "aws-sdk", "~>
|
20
|
+
spec.add_dependency "fluentd", "~> 0"
|
21
|
+
spec.add_dependency "aws-sdk", "~> 2"
|
22
22
|
|
23
|
-
spec.add_development_dependency "bundler", "~> 1.
|
24
|
-
spec.add_development_dependency "rake"
|
25
|
-
spec.add_development_dependency "webmock"
|
23
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
26
24
|
end
|
@@ -2,13 +2,14 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
2
2
|
Fluent::Plugin.register_input('elb_log', self)
|
3
3
|
|
4
4
|
LOGFILE_REGEXP = /^((?<prefix>.+?)\/|)AWSLogs\/(?<account_id>[0-9]{12})\/elasticloadbalancing\/(?<region>.+?)\/(?<logfile_date>[0-9]{4}\/[0-9]{2}\/[0-9]{2})\/[0-9]{12}_elasticloadbalancing_.+?_(?<logfile_elb_name>[^_]+)_(?<elb_timestamp>[0-9]{8}T[0-9]{4}Z)_(?<elb_ip_address>.+?)_(?<logfile_hash>.+)\.log$/
|
5
|
-
ACCESSLOG_REGEXP = /^(?<time>\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z) (?<elb>.+?) (?<client
|
5
|
+
ACCESSLOG_REGEXP = /^(?<time>\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z) (?<elb>.+?) (?<client>.+?)\:(?<client_port>.+?) (?<backend>.+?)\:(?<backend_port>.+?) (?<request_processing_time>.+?) (?<backend_processing_time>.+?) (?<response_processing_time>.+?) (?<elb_status_code>.+?) (?<backend_status_code>.+?) (?<received_bytes>.+?) (?<sent_bytes>.+?) \"(?<request_method>.+?) (?<request_uri>.+?) (?<request_protocol>.+?)\" \"(?<user_agent>.*?)\" (?<option1>.+?) (?<option2>.+)(| (?<option3>.*))/
|
6
6
|
|
7
7
|
config_param :access_key_id, :string, :default => nil
|
8
8
|
config_param :secret_access_key, :string, :default => nil
|
9
|
+
config_param :region, :string, :default => nil
|
9
10
|
config_param :s3_bucketname, :string, :default => nil
|
10
11
|
config_param :s3_prefix, :string, :default => nil
|
11
|
-
config_param :
|
12
|
+
config_param :tag, :string, :default => 'elb.access'
|
12
13
|
config_param :timestamp_file, :string, :default => nil
|
13
14
|
config_param :refresh_interval, :integer, :default => 300
|
14
15
|
config_param :buf_file, :string, :default => './fluentd_elb_log_buf_file'
|
@@ -18,24 +19,23 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
18
19
|
super
|
19
20
|
require 'aws-sdk'
|
20
21
|
|
21
|
-
|
22
|
-
|
22
|
+
raise Fluent::ConfigError.new("region is required") unless @region
|
23
|
+
if !has_iam_role?
|
24
|
+
raise Fluent::ConfigError.new("access_key_id is required") if @access_key_id.nil?
|
25
|
+
raise Fluent::ConfigError.new("secret_access_key is required") if @secret_access_key.nil?
|
23
26
|
end
|
24
|
-
if @secret_access_key.nil? and has_not_iam_role?
|
25
|
-
raise Fluent::ConfigError.new("secret_access_key is required")
|
26
|
-
end
|
27
|
-
|
28
27
|
raise Fluent::ConfigError.new("s3_bucketname is required") unless @s3_bucketname
|
29
28
|
raise Fluent::ConfigError.new("timestamp_file is required") unless @timestamp_file
|
30
|
-
raise Fluent::ConfigError.new("s3 bucket fetch error #{@s3_bucketname}") if init_s3bucket.nil?
|
31
|
-
FileUtils.touch(@buf_file)
|
32
29
|
end
|
33
30
|
|
34
31
|
def start
|
35
32
|
super
|
36
33
|
|
37
|
-
|
38
|
-
@timestamp_file.
|
34
|
+
# files touch
|
35
|
+
File.open(@timestamp_file, File::RDWR|File::CREAT).close
|
36
|
+
File.open(@buf_file, File::RDWR|File::CREAT).close
|
37
|
+
|
38
|
+
raise StandardError.new("s3 bucket not found #{@s3_bucketname}") unless s3bucket_is_ok()
|
39
39
|
|
40
40
|
@loop = Coolio::Loop.new
|
41
41
|
timer_trigger = TimerWatcher.new(@refresh_interval, true, &method(:input))
|
@@ -47,142 +47,223 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
47
47
|
super
|
48
48
|
@loop.stop
|
49
49
|
@thread.join
|
50
|
-
@timestamp_file.close
|
51
50
|
end
|
52
51
|
|
53
52
|
private
|
54
53
|
|
55
|
-
def
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
options[:s3_endpoint] = @s3_endpoint if @s3_endpoint
|
62
|
-
if @proxy_uri
|
63
|
-
options[:proxy_uri] = @proxy_uri
|
54
|
+
def has_iam_role?
|
55
|
+
begin
|
56
|
+
ec2 = Aws::EC2::Client.new(region: @region)
|
57
|
+
!ec2.config.credentials.nil?
|
58
|
+
rescue => e
|
59
|
+
$log.warn "EC2 Client error occurred: #{e.message}"
|
64
60
|
end
|
61
|
+
end
|
65
62
|
|
63
|
+
def get_timestamp_file
|
66
64
|
begin
|
67
|
-
|
68
|
-
|
65
|
+
# get timestamp last proc
|
66
|
+
timestamp = 0
|
67
|
+
$log.debug "timestamp file #{@timestamp_file} read"
|
68
|
+
File.open(@timestamp_file, File::RDONLY) do |file|
|
69
|
+
timestamp = file.read.to_i
|
70
|
+
end
|
71
|
+
$log.debug "timestamp start at:" + Time.at(timestamp).to_s
|
72
|
+
return timestamp
|
69
73
|
rescue => e
|
70
|
-
$log.warn "
|
71
|
-
nil
|
74
|
+
$log.warn "timestamp file get and parse error occurred: #{e.message}"
|
72
75
|
end
|
73
76
|
end
|
74
77
|
|
75
|
-
def
|
76
|
-
|
78
|
+
def put_timestamp_file(timestamp)
|
79
|
+
begin
|
80
|
+
$log.debug "timestamp file #{@timestamp_file} write"
|
81
|
+
File.open(@timestamp_file, File::WRONLY|File::CREAT|File::TRUNC) do |file|
|
82
|
+
file.puts timestamp.to_s
|
83
|
+
end
|
84
|
+
rescue => e
|
85
|
+
$log.warn "timestamp file get and parse error occurred: #{e.message}"
|
86
|
+
end
|
77
87
|
end
|
78
88
|
|
79
|
-
def
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
next unless matches
|
93
|
-
next if !@s3_prefix.nil? && matches[:prefix] != @s3_prefix
|
94
|
-
log_objects.push obj
|
89
|
+
def s3_client
|
90
|
+
begin
|
91
|
+
options = {
|
92
|
+
:region => @region,
|
93
|
+
}
|
94
|
+
if @access_key_id && @secret_access_key
|
95
|
+
options[:access_key_id] = @access_key_id
|
96
|
+
options[:secret_access_key] = @secret_access_key
|
97
|
+
end
|
98
|
+
$log.debug "S3 client connect"
|
99
|
+
Aws::S3::Client.new(options)
|
100
|
+
rescue => e
|
101
|
+
$log.warn "S3 Client error occurred: #{e.message}"
|
95
102
|
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def s3bucket_is_ok
|
106
|
+
begin
|
107
|
+
$log.debug "search bucket #{@s3_bucketname}"
|
96
108
|
|
97
|
-
|
98
|
-
|
99
|
-
|
109
|
+
s3_client.list_buckets.buckets.any? do |bucket|
|
110
|
+
bucket.name == @s3_bucketname
|
111
|
+
end
|
112
|
+
rescue => e
|
113
|
+
$log.warn "S3 Client error occurred: #{e.message}"
|
100
114
|
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def input
|
118
|
+
$log.debug "start"
|
119
|
+
|
120
|
+
timestamp = get_timestamp_file()
|
121
|
+
|
122
|
+
object_keys = get_object_keys(timestamp)
|
123
|
+
object_keys = sort_object_key(object_keys)
|
124
|
+
|
125
|
+
$log.info "processing #{object_keys.count} object(s)."
|
101
126
|
|
102
|
-
|
103
|
-
matches = LOGFILE_REGEXP.match(obj.key)
|
104
|
-
timestamp = matches[:elb_timestamp].to_i
|
127
|
+
object_keys.each do |object_key|
|
105
128
|
record_common = {
|
106
|
-
"account_id" =>
|
107
|
-
"region" =>
|
108
|
-
"logfile_date" =>
|
109
|
-
"logfile_elb_name" =>
|
110
|
-
"elb_ip_address" =>
|
111
|
-
"logfile_hash" =>
|
112
|
-
"elb_timestamp" =>
|
129
|
+
"account_id" => object_key[:account_id],
|
130
|
+
"region" => object_key[:region],
|
131
|
+
"logfile_date" => object_key[:logfile_date],
|
132
|
+
"logfile_elb_name" => object_key[:logfile_elb_name],
|
133
|
+
"elb_ip_address" => object_key[:elb_ip_address],
|
134
|
+
"logfile_hash" => object_key[:logfile_hash],
|
135
|
+
"elb_timestamp" => object_key[:elb_timestamp],
|
136
|
+
"key" => object_key[:key],
|
137
|
+
"prefix" => object_key[:prefix],
|
138
|
+
"elb_timestamp_unixtime" => object_key[:elb_timestamp_unixtime],
|
113
139
|
}
|
114
140
|
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
141
|
+
get_file_from_s3(object_key[:key])
|
142
|
+
emit_lines_from_buffer_file(record_common)
|
143
|
+
|
144
|
+
put_timestamp_file(object_key[:elb_timestamp_unixtime])
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
def get_object_keys(timestamp)
|
149
|
+
# get values from object file name
|
150
|
+
begin
|
151
|
+
object_keys = []
|
152
|
+
|
153
|
+
objects = s3_client.list_objects(
|
154
|
+
bucket: @s3_bucketname,
|
155
|
+
max_keys: 100,
|
156
|
+
prefix: @s3_prefix,
|
157
|
+
)
|
158
|
+
|
159
|
+
objects.each do |object|
|
160
|
+
object.contents.each do |content|
|
161
|
+
matches = LOGFILE_REGEXP.match(content.key)
|
162
|
+
next unless matches
|
121
163
|
|
122
|
-
|
164
|
+
# snip old items
|
165
|
+
elb_timestamp_unixtime = Time.parse(matches[:elb_timestamp]).to_i
|
166
|
+
next if elb_timestamp_unixtime <= timestamp
|
123
167
|
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
168
|
+
$log.debug content.key
|
169
|
+
object_keys << {
|
170
|
+
key: content.key,
|
171
|
+
prefix: matches[:prefix],
|
172
|
+
account_id: matches[:account_id],
|
173
|
+
region: matches[:region],
|
174
|
+
logfile_date: matches[:logfile_date],
|
175
|
+
logfile_elb_name: matches[:logfile_elb_name],
|
176
|
+
elb_timestamp: matches[:elb_timestamp],
|
177
|
+
elb_ip_address: matches[:elb_ip_address],
|
178
|
+
logfile_hash: matches[:logfile_hash],
|
179
|
+
elb_timestamp_unixtime: elb_timestamp_unixtime,
|
180
|
+
}
|
181
|
+
end
|
182
|
+
end
|
183
|
+
return object_keys
|
184
|
+
rescue => e
|
185
|
+
$log.warn "error occurred: #{e.message}"
|
129
186
|
end
|
130
187
|
end
|
131
188
|
|
132
|
-
def
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
line_match = ACCESSLOG_REGEXP.match(line)
|
137
|
-
next unless line_match
|
138
|
-
|
139
|
-
record = {
|
140
|
-
"time" => line_match[:time].gsub(/Z/, "+0000"),
|
141
|
-
"elb" => line_match[:elb],
|
142
|
-
"client" => line_match[:client],
|
143
|
-
"client_port" => line_match[:client_port],
|
144
|
-
"backend" => line_match[:backend],
|
145
|
-
"backend_port" => line_match[:backend_port],
|
146
|
-
"request_processing_time" => line_match[:request_processing_time].to_f,
|
147
|
-
"backend_processing_time" => line_match[:backend_processing_time].to_f,
|
148
|
-
"response_processing_time" => line_match[:response_processing_time].to_f,
|
149
|
-
"elb_status_code" => line_match[:elb_status_code],
|
150
|
-
"backend_status_code" => line_match[:backend_status_code],
|
151
|
-
"received_bytes" => line_match[:received_bytes].to_i,
|
152
|
-
"sent_bytes" => line_match[:sent_bytes].to_i,
|
153
|
-
"request_method" => line_match[:request_method],
|
154
|
-
"request_uri" => line_match[:request_uri],
|
155
|
-
"request_protocol" => line_match[:request_protocol],
|
156
|
-
}
|
157
|
-
|
158
|
-
Fluent::Engine.emit("elb.access", Fluent::Engine.now, record_common.merge(record))
|
189
|
+
def sort_object_key(src_object_keys)
|
190
|
+
begin
|
191
|
+
src_object_keys.sort do |a, b|
|
192
|
+
a[:elb_timestamp_unixtime] <=> b[:elb_timestamp_unixtime]
|
159
193
|
end
|
194
|
+
rescue => e
|
195
|
+
$log.warn "error occurred: #{e.message}"
|
160
196
|
end
|
161
197
|
end
|
162
198
|
|
163
|
-
def
|
164
|
-
|
199
|
+
def get_file_from_s3(object_name)
|
200
|
+
begin
|
201
|
+
$log.debug "getting object from s3 name is #{object_name}"
|
165
202
|
|
166
|
-
|
167
|
-
|
203
|
+
# read an object from S3 to a file and write buffer file
|
204
|
+
File.open(@buf_file, File::WRONLY|File::CREAT|File::TRUNC) do |file|
|
205
|
+
s3_client.get_object(
|
206
|
+
bucket: @s3_bucketname,
|
207
|
+
key: object_name
|
208
|
+
) do |chunk|
|
209
|
+
file.write(chunk)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
rescue => e
|
213
|
+
$log.warn "error occurred: #{e.message}"
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def emit_lines_from_buffer_file(record_common)
|
168
218
|
begin
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
219
|
+
# emit per line
|
220
|
+
File.open(@buf_file, File::RDONLY) do |file|
|
221
|
+
file.each_line do |line|
|
222
|
+
line_match = ACCESSLOG_REGEXP.match(line)
|
223
|
+
unless line_match
|
224
|
+
$log.info "nomatch log found: #{line} in #{record_common['key']}"
|
225
|
+
next
|
226
|
+
end
|
227
|
+
|
228
|
+
record = {
|
229
|
+
"time" => line_match[:time].gsub(/Z/, "+0000"),
|
230
|
+
"elb" => line_match[:elb],
|
231
|
+
"client" => line_match[:client],
|
232
|
+
"client_port" => line_match[:client_port],
|
233
|
+
"backend" => line_match[:backend],
|
234
|
+
"backend_port" => line_match[:backend_port],
|
235
|
+
"request_processing_time" => line_match[:request_processing_time].to_f,
|
236
|
+
"backend_processing_time" => line_match[:backend_processing_time].to_f,
|
237
|
+
"response_processing_time" => line_match[:response_processing_time].to_f,
|
238
|
+
"elb_status_code" => line_match[:elb_status_code],
|
239
|
+
"backend_status_code" => line_match[:backend_status_code],
|
240
|
+
"received_bytes" => line_match[:received_bytes].to_i,
|
241
|
+
"sent_bytes" => line_match[:sent_bytes].to_i,
|
242
|
+
"request_method" => line_match[:request_method],
|
243
|
+
"request_uri" => line_match[:request_uri],
|
244
|
+
"request_protocol" => line_match[:request_protocol],
|
245
|
+
"user_agent" => line_match[:user_agent],
|
246
|
+
"option1" => line_match[:option1],
|
247
|
+
"option2" => line_match[:option2],
|
248
|
+
"option3" => line_match[:option3],
|
249
|
+
}
|
250
|
+
|
251
|
+
Fluent::Engine.emit(@tag, Fluent::Engine.now, record_common.merge(record))
|
252
|
+
end
|
253
|
+
end
|
173
254
|
rescue => e
|
174
|
-
$log.warn "
|
255
|
+
$log.warn "error occurred: #{e.message}"
|
175
256
|
end
|
176
|
-
@has_iam_role
|
177
257
|
end
|
178
258
|
|
179
|
-
def
|
180
|
-
|
259
|
+
def run
|
260
|
+
@loop.run
|
181
261
|
end
|
182
262
|
|
183
263
|
class TimerWatcher < Coolio::TimerWatcher
|
184
264
|
def initialize(interval, repeat, &callback)
|
185
265
|
@callback = callback
|
266
|
+
on_timer # first call
|
186
267
|
super(interval, repeat)
|
187
268
|
end
|
188
269
|
|
metadata
CHANGED
@@ -1,96 +1,57 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elb-log
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.2.0
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- shinsaka
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2015-
|
11
|
+
date: 2015-06-16 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: fluentd
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - "~>"
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: '0'
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - "~>"
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: '0'
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: aws-sdk
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- - ~>
|
31
|
+
- - "~>"
|
36
32
|
- !ruby/object:Gem::Version
|
37
|
-
version: '
|
33
|
+
version: '2'
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- - ~>
|
38
|
+
- - "~>"
|
44
39
|
- !ruby/object:Gem::Version
|
45
|
-
version: '
|
40
|
+
version: '2'
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: bundler
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- - ~>
|
45
|
+
- - "~>"
|
52
46
|
- !ruby/object:Gem::Version
|
53
|
-
version: '1.
|
47
|
+
version: '1.7'
|
54
48
|
type: :development
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- - ~>
|
52
|
+
- - "~>"
|
60
53
|
- !ruby/object:Gem::Version
|
61
|
-
version: '1.
|
62
|
-
- !ruby/object:Gem::Dependency
|
63
|
-
name: rake
|
64
|
-
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
|
-
requirements:
|
67
|
-
- - ! '>='
|
68
|
-
- !ruby/object:Gem::Version
|
69
|
-
version: '0'
|
70
|
-
type: :development
|
71
|
-
prerelease: false
|
72
|
-
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
|
-
requirements:
|
75
|
-
- - ! '>='
|
76
|
-
- !ruby/object:Gem::Version
|
77
|
-
version: '0'
|
78
|
-
- !ruby/object:Gem::Dependency
|
79
|
-
name: webmock
|
80
|
-
requirement: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
|
-
requirements:
|
83
|
-
- - ! '>='
|
84
|
-
- !ruby/object:Gem::Version
|
85
|
-
version: '0'
|
86
|
-
type: :development
|
87
|
-
prerelease: false
|
88
|
-
version_requirements: !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
|
-
requirements:
|
91
|
-
- - ! '>='
|
92
|
-
- !ruby/object:Gem::Version
|
93
|
-
version: '0'
|
54
|
+
version: '1.7'
|
94
55
|
description: Amazon ELB log input plugin for fluentd
|
95
56
|
email:
|
96
57
|
- shinx1265@gmail.com
|
@@ -98,8 +59,8 @@ executables: []
|
|
98
59
|
extensions: []
|
99
60
|
extra_rdoc_files: []
|
100
61
|
files:
|
101
|
-
- .gitignore
|
102
|
-
- .travis.yml
|
62
|
+
- ".gitignore"
|
63
|
+
- ".travis.yml"
|
103
64
|
- Gemfile
|
104
65
|
- LICENSE.txt
|
105
66
|
- README.md
|
@@ -112,27 +73,26 @@ files:
|
|
112
73
|
homepage: https://github.com/shinsaka/fluent-plugin-elb-log
|
113
74
|
licenses:
|
114
75
|
- MIT
|
76
|
+
metadata: {}
|
115
77
|
post_install_message:
|
116
78
|
rdoc_options: []
|
117
79
|
require_paths:
|
118
80
|
- lib
|
119
81
|
required_ruby_version: !ruby/object:Gem::Requirement
|
120
|
-
none: false
|
121
82
|
requirements:
|
122
|
-
- -
|
83
|
+
- - ">="
|
123
84
|
- !ruby/object:Gem::Version
|
124
85
|
version: '0'
|
125
86
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
126
|
-
none: false
|
127
87
|
requirements:
|
128
|
-
- -
|
88
|
+
- - ">="
|
129
89
|
- !ruby/object:Gem::Version
|
130
90
|
version: '0'
|
131
91
|
requirements: []
|
132
92
|
rubyforge_project:
|
133
|
-
rubygems_version:
|
93
|
+
rubygems_version: 2.2.1
|
134
94
|
signing_key:
|
135
|
-
specification_version:
|
95
|
+
specification_version: 4
|
136
96
|
summary: Amazon ELB log input plugin
|
137
97
|
test_files:
|
138
98
|
- test/helper.rb
|