fluent-plugin-elb-log 0.7.0 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/fluent-plugin-elb-log.gemspec +1 -1
- data/lib/fluent/plugin/in_elb_log.rb +65 -67
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 45e03965c1d8f3c0c81cac0793a2fd9dd08d5c06
|
4
|
+
data.tar.gz: c23f025998cb293ae4bc9472d6f4e97456532192
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 41381c1563f713ef5561a984c6790f9870f6bfa97cffd154a37f28508cc5114e554200ccaa9ff495a1e321c5c38421de165c3bd1904d1e855a1af56c3d00b489
|
7
|
+
data.tar.gz: d9d6f310ee099df38efdd65cf76e04b167bcd03877fc994460491bd65d60cc599542b33762e8da62fcf570e0650195b077c685ad2c1e58e04aee375858dff05e
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-elb-log"
|
7
|
-
spec.version = "0.
|
7
|
+
spec.version = "0.8.0"
|
8
8
|
spec.authors = ["shinsaka"]
|
9
9
|
spec.email = ["shinx1265@gmail.com"]
|
10
10
|
spec.summary = "Amazon ELB log input plugin"
|
@@ -4,6 +4,7 @@ require 'fileutils'
|
|
4
4
|
require 'aws-sdk-s3'
|
5
5
|
require 'aws-sdk-ec2'
|
6
6
|
require 'fluent/input'
|
7
|
+
require 'digest/sha1'
|
7
8
|
|
8
9
|
class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
|
9
10
|
Fluent::Plugin.register_input('elb_log', self)
|
@@ -25,6 +26,8 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
|
|
25
26
|
config_param :http_proxy, :string, default: nil
|
26
27
|
config_param :start_time, :string, default: nil
|
27
28
|
config_param :delete, :bool, default: false
|
29
|
+
config_param :num_nodes, :integer, default: 1
|
30
|
+
config_param :node_no, :integer, default: 0
|
28
31
|
|
29
32
|
def configure(conf)
|
30
33
|
super
|
@@ -66,7 +69,12 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
|
|
66
69
|
timestamp = start_time.to_i
|
67
70
|
log.debug "timestamp file #{@timestamp_file} read"
|
68
71
|
File.open(@timestamp_file, File::RDONLY) do |file|
|
69
|
-
|
72
|
+
if file.size > 0
|
73
|
+
timestamp_from_file = file.read.to_i
|
74
|
+
if timestamp_from_file > timestamp
|
75
|
+
timestamp = timestamp_from_file
|
76
|
+
end
|
77
|
+
end
|
70
78
|
end
|
71
79
|
log.debug "timestamp start at:" + Time.at(timestamp).to_s
|
72
80
|
return timestamp
|
@@ -118,70 +126,39 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
|
|
118
126
|
end
|
119
127
|
|
120
128
|
def input
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
get_file_from_s3(object_key[:key])
|
146
|
-
emit_lines_from_buffer_file(record_common)
|
147
|
-
|
148
|
-
put_timestamp_file(object_key[:s3_last_modified_unixtime])
|
129
|
+
begin
|
130
|
+
log.debug "start"
|
131
|
+
timestamp = get_timestamp_file()
|
132
|
+
|
133
|
+
object_keys = get_object_keys(timestamp)
|
134
|
+
object_keys = sort_object_key(object_keys)
|
135
|
+
|
136
|
+
log.info "processing #{object_keys.count} object(s)."
|
137
|
+
|
138
|
+
object_keys.each do |object_key|
|
139
|
+
record_common = {
|
140
|
+
"account_id" => object_key[:account_id],
|
141
|
+
"region" => object_key[:region],
|
142
|
+
"logfile_date" => object_key[:logfile_date],
|
143
|
+
"logfile_elb_name" => object_key[:logfile_elb_name],
|
144
|
+
"elb_ip_address" => object_key[:elb_ip_address],
|
145
|
+
"logfile_hash" => object_key[:logfile_hash],
|
146
|
+
"elb_timestamp" => object_key[:elb_timestamp],
|
147
|
+
"key" => object_key[:key],
|
148
|
+
"prefix" => object_key[:prefix],
|
149
|
+
"elb_timestamp_unixtime" => object_key[:elb_timestamp_unixtime],
|
150
|
+
"s3_last_modified_unixtime" => object_key[:s3_last_modified_unixtime],
|
151
|
+
}
|
149
152
|
|
150
|
-
|
151
|
-
|
152
|
-
end
|
153
|
-
end
|
154
|
-
end
|
153
|
+
get_file_from_s3(object_key[:key])
|
154
|
+
emit_lines_from_buffer_file(record_common)
|
155
155
|
|
156
|
-
|
157
|
-
begin
|
158
|
-
object_keys = []
|
159
|
-
get_object_contents.each do |content|
|
160
|
-
# snip old items
|
161
|
-
s3_last_modified_unixtime = content.last_modified.to_i
|
162
|
-
next if s3_last_modified_unixtime <= timestamp
|
156
|
+
put_timestamp_file(object_key[:s3_last_modified_unixtime])
|
163
157
|
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
log.debug "matched"
|
169
|
-
log.debug object_key
|
170
|
-
object_keys << {
|
171
|
-
key: object_key,
|
172
|
-
prefix: matches[:prefix],
|
173
|
-
account_id: matches[:account_id],
|
174
|
-
region: matches[:region],
|
175
|
-
logfile_date: matches[:logfile_date],
|
176
|
-
logfile_elb_name: matches[:logfile_elb_name],
|
177
|
-
elb_timestamp: matches[:elb_timestamp],
|
178
|
-
elb_ip_address: matches[:elb_ip_address],
|
179
|
-
logfile_hash: matches[:logfile_hash],
|
180
|
-
elb_timestamp_unixtime: Time.parse(matches[:elb_timestamp]).to_i,
|
181
|
-
s3_last_modified_unixtime: s3_last_modified_unixtime,
|
182
|
-
}
|
158
|
+
if @delete
|
159
|
+
delete_file_from_s3(object_key[:key])
|
160
|
+
end
|
183
161
|
end
|
184
|
-
return object_keys
|
185
162
|
rescue => e
|
186
163
|
log.warn "error occurred: #{e.message}"
|
187
164
|
end
|
@@ -205,8 +182,8 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
|
|
205
182
|
)
|
206
183
|
end
|
207
184
|
|
208
|
-
def
|
209
|
-
|
185
|
+
def get_object_keys(timestamp)
|
186
|
+
object_keys = []
|
210
187
|
|
211
188
|
resp = s3_client.list_objects_v2(
|
212
189
|
bucket: @s3_bucketname,
|
@@ -215,11 +192,32 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
|
|
215
192
|
|
216
193
|
loop do
|
217
194
|
resp.contents.each do |content|
|
218
|
-
|
195
|
+
s3_last_modified_unixtime = content.last_modified.to_i
|
196
|
+
|
197
|
+
object_key = content.key
|
198
|
+
node_no = Digest::SHA1.hexdigest(object_key).to_i(16) % @num_nodes
|
199
|
+
next unless node_no == @node_no
|
200
|
+
|
201
|
+
matches = LOGFILE_REGEXP.match(object_key)
|
202
|
+
if s3_last_modified_unixtime > timestamp and matches
|
203
|
+
object_keys << {
|
204
|
+
key: object_key,
|
205
|
+
prefix: matches[:prefix],
|
206
|
+
account_id: matches[:account_id],
|
207
|
+
region: matches[:region],
|
208
|
+
logfile_date: matches[:logfile_date],
|
209
|
+
logfile_elb_name: matches[:logfile_elb_name],
|
210
|
+
elb_timestamp: matches[:elb_timestamp],
|
211
|
+
elb_ip_address: matches[:elb_ip_address],
|
212
|
+
logfile_hash: matches[:logfile_hash],
|
213
|
+
elb_timestamp_unixtime: Time.parse(matches[:elb_timestamp]).to_i,
|
214
|
+
s3_last_modified_unixtime: s3_last_modified_unixtime,
|
215
|
+
}
|
216
|
+
end
|
219
217
|
end
|
220
218
|
|
221
219
|
if !resp.is_truncated
|
222
|
-
|
220
|
+
return object_keys
|
223
221
|
end
|
224
222
|
|
225
223
|
resp = s3_client.list_objects_v2(
|
@@ -229,7 +227,7 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
|
|
229
227
|
)
|
230
228
|
end
|
231
229
|
|
232
|
-
return
|
230
|
+
return object_keys
|
233
231
|
end
|
234
232
|
|
235
233
|
def inflate(srcfile, dstfile)
|
@@ -322,4 +320,4 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
|
|
322
320
|
"option3" => item[:option3]
|
323
321
|
}
|
324
322
|
end
|
325
|
-
end
|
323
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elb-log
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- shinsaka
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-05-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|