fluent-plugin-elb-log 0.0.8 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/fluent-plugin-elb-log.gemspec +1 -1
- data/lib/fluent/plugin/in_elb_log.rb +31 -9
- data/test/plugin/in_elb_log.rb +1 -12
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: eb31f022a24f4237678c1666bbe9d407eaa858b3
|
4
|
+
data.tar.gz: 6c353760ad26a95b9bfc9293548750a8fe842058
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 32102c4c6e34209171d5fd95f474d07bd227308bf85a7c4b67f1eeeb6e3822fa6824136271a771d5aa6933fb1d9080b1d46ba7c648cc536e3308acce67462b63
|
7
|
+
data.tar.gz: fe24ac63617e936272d650a55b1746101c31eebfd77de93f20f4cfccfa569697baf5f3065b83e73d80721f9fa9648052d11f55e413f78eadf5fa31346c2773ea
|
data/README.md
CHANGED
@@ -23,6 +23,7 @@
|
|
23
23
|
s3_prefix <elb log's prefix>
|
24
24
|
timestamp_file <proc last file timestamp record filename>
|
25
25
|
refresh_interval <interval number by second>
|
26
|
+
buf_file <buffer file path>
|
26
27
|
|
27
28
|
# following attibutes are required if you don't use IAM Role
|
28
29
|
access_key_id <access_key>
|
@@ -41,6 +42,7 @@
|
|
41
42
|
s3_prefix prefix
|
42
43
|
timestamp_file elb_last_at.dat
|
43
44
|
refresh_interval 300
|
45
|
+
buf_file /tmp/fluentd-elblog.tmpfile
|
44
46
|
</source>
|
45
47
|
|
46
48
|
<match **>
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-elb-log"
|
7
|
-
spec.version = "0.0
|
7
|
+
spec.version = "0.1.0"
|
8
8
|
spec.authors = ["shinsaka"]
|
9
9
|
spec.email = ["shinx1265@gmail.com"]
|
10
10
|
spec.summary = "Amazon ELB log input plugin"
|
@@ -11,6 +11,7 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
11
11
|
config_param :s3_endpoint, :string, :default => 's3.amazon.com'
|
12
12
|
config_param :timestamp_file, :string, :default => nil
|
13
13
|
config_param :refresh_interval, :integer, :default => 300
|
14
|
+
config_param :buf_file, :string, :default => './fluentd_elb_log_buf_file'
|
14
15
|
|
15
16
|
def configure(conf)
|
16
17
|
super
|
@@ -25,13 +26,13 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
25
26
|
|
26
27
|
raise Fluent::ConfigError.new("s3_bucketname is required") unless @s3_bucketname
|
27
28
|
raise Fluent::ConfigError.new("timestamp_file is required") unless @timestamp_file
|
29
|
+
raise Fluent::ConfigError.new("s3 bucket fetch error #{@s3_bucketname}") if init_s3bucket.nil?
|
30
|
+
FileUtils.touch(@buf_file)
|
28
31
|
end
|
29
32
|
|
30
33
|
def start
|
31
34
|
super
|
32
35
|
|
33
|
-
init_s3bucket
|
34
|
-
|
35
36
|
@timestamp_file = File.open(@timestamp_file, File::RDWR|File::CREAT)
|
36
37
|
@timestamp_file.sync = true
|
37
38
|
|
@@ -58,7 +59,13 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
58
59
|
end
|
59
60
|
options[:s3_endpoint] = @s3_endpoint if @s3_endpoint
|
60
61
|
|
61
|
-
|
62
|
+
begin
|
63
|
+
@bucket = AWS::S3.new(options).buckets[@s3_bucketname]
|
64
|
+
@bucket.objects.count
|
65
|
+
rescue => e
|
66
|
+
$log.warn "fluent-plugin-elb-log: s3 bucket fetch error: #{e.message}"
|
67
|
+
nil
|
68
|
+
end
|
62
69
|
end
|
63
70
|
|
64
71
|
def run
|
@@ -101,7 +108,27 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
101
108
|
elb_timestamp: matches[:elb_timestamp],
|
102
109
|
}
|
103
110
|
|
104
|
-
|
111
|
+
# read an object from S3 to a file and write buffer file
|
112
|
+
File.open(@buf_file, File::WRONLY) do |file|
|
113
|
+
obj.read do |chunk|
|
114
|
+
file.write(chunk)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
emit_lines_from_buffer_file record_common
|
119
|
+
|
120
|
+
# timestamp save
|
121
|
+
@timestamp_file.rewind
|
122
|
+
@timestamp_file.write(obj.last_modified.to_i)
|
123
|
+
@timestamp_file.truncate(@timestamp_file.tell)
|
124
|
+
$log.info "fluent-plugin-elb-log: timestamp save: " + obj.last_modified.to_s
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def emit_lines_from_buffer_file(record_common)
|
129
|
+
# emit per line
|
130
|
+
File.open(@buf_file, File::RDONLY) do |file|
|
131
|
+
file.each_line do |line|
|
105
132
|
line_match = ACCESSLOG_REGEXP.match(line)
|
106
133
|
next unless line_match
|
107
134
|
|
@@ -126,11 +153,6 @@ class Fluent::Elb_LogInput < Fluent::Input
|
|
126
153
|
|
127
154
|
Fluent::Engine.emit("elb.access", Fluent::Engine.now, record_common.merge(record))
|
128
155
|
end
|
129
|
-
# timestamp save
|
130
|
-
@timestamp_file.rewind
|
131
|
-
@timestamp_file.write(obj.last_modified.to_i)
|
132
|
-
@timestamp_file.truncate(@timestamp_file.tell)
|
133
|
-
$log.info "fluent-plugin-elb-log: timestamp save: " + obj.last_modified.to_s
|
134
156
|
end
|
135
157
|
end
|
136
158
|
|
data/test/plugin/in_elb_log.rb
CHANGED
@@ -98,16 +98,5 @@ class Elb_LogInputTest < Test::Unit::TestCase
|
|
98
98
|
}
|
99
99
|
assert_equal('secret_access_key is required', exception.message)
|
100
100
|
end
|
101
|
-
end
|
102
101
|
|
103
|
-
|
104
|
-
#
|
105
|
-
# def setup
|
106
|
-
# Fluent::Test.setup
|
107
|
-
# end
|
108
|
-
#
|
109
|
-
# def test_format
|
110
|
-
#
|
111
|
-
# end
|
112
|
-
#
|
113
|
-
#end
|
102
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elb-log
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- shinsaka
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-07-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|