fluent-plugin-forward-aws 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +13 -0
- data/README.md +171 -0
- data/Rakefile +10 -0
- data/fluent-plugin-forward-aws.gemspec +22 -0
- data/lib/fluent/plugin/in_forward_aws.rb +198 -0
- data/lib/fluent/plugin/out_forward_aws.rb +147 -0
- data/test/awsconfig.yml.sample +9 -0
- data/test/helper.rb +29 -0
- data/test/plugin/test_in_foward_aws.rb +75 -0
- data/test/plugin/test_out_foward_aws.rb +91 -0
- metadata +93 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright (c) 2013 - Tomohisa Ota
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,171 @@
|
|
1
|
+
# Fluent::Plugin::Forward-AWS
|
2
|
+
|
3
|
+
Forward-AWS plugin forwards log through Amazon Web Service.
|
4
|
+
It uses S3 as log storage, and SNS+SQS for notification.
|
5
|
+
|
6
|
+
Please see [wiki](https://github.com/tomohisaota/fluent-plugin-forward-aws/wiki/Forward-AWS-plugin-concept) to understand the concept.
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
Use ruby gem as :
|
11
|
+
|
12
|
+
$ gem install fluent-plugin-forward-aws
|
13
|
+
|
14
|
+
Or, if you're using td-client, you can call td-client's gem
|
15
|
+
|
16
|
+
$ /usr/lib64/fluent/ruby/bin/gem install fluent-plugin-forward-aws
|
17
|
+
|
18
|
+
## AWS Configuration
|
19
|
+
### IAM
|
20
|
+
It is recommended to create IAM user for logging.
|
21
|
+
Create IAM user with credential, and memorize following parameters
|
22
|
+
+ aws_access_key_id
|
23
|
+
+ aws_secret_access_key
|
24
|
+
|
25
|
+
### S3
|
26
|
+
Create bucket, and memorize following parameters
|
27
|
+
+ aws_s3_endpoint
|
28
|
+
+ aws_s3_bucketname
|
29
|
+
|
30
|
+
### SNS
|
31
|
+
Create SNS Topic, and memorize following parameters
|
32
|
+
+ aws_sns_endpoint
|
33
|
+
+ aws_sns_topic_arn
|
34
|
+
|
35
|
+
### SQS
|
36
|
+
Create SQS Queue, subscribe to SNS, and memorize following parameters
|
37
|
+
+ aws_sqs_endpoint
|
38
|
+
+ aws_sqs_queue_url
|
39
|
+
|
40
|
+
#### How to configure SQS as SNS subscriber
|
41
|
+
In short, change SQS's access policy to accept "SendMessage" from your SNS ARN, And add SQS ARN to SNS subscribers.
|
42
|
+
You can do the above step in one shot from SQS Management Console.
|
43
|
+
For more detail, check [amazon official document](http://docs.aws.amazon.com/sns/latest/gsg/SendMessageToSQS.html)
|
44
|
+
|
45
|
+
## Out Plugin Configuration
|
46
|
+
|
47
|
+
###Required AWS permission
|
48
|
+
+ s3:PutObject
|
49
|
+
+ sns:Publish
|
50
|
+
|
51
|
+
### Basic configuration
|
52
|
+
Use "default" channel for all the log data
|
53
|
+
```
|
54
|
+
<match **>
|
55
|
+
type forward_aws
|
56
|
+
aws_access_key_id XXXXXXXXXXXXXXXXXXXX
|
57
|
+
aws_secret_access_key XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
58
|
+
|
59
|
+
aws_s3_endpoint s3-ap-northeast-1.amazonaws.com
|
60
|
+
aws_s3_bucketname XXXXXXXXXXXXXXXXXXXX
|
61
|
+
|
62
|
+
aws_sns_endpoint sns.ap-northeast-1.amazonaws.com
|
63
|
+
aws_sns_topic_arn arn:aws:sns:ap-northeast-1:XXXXXXXXXXXXXXXXXXXX
|
64
|
+
|
65
|
+
buffer_type file
|
66
|
+
buffer_path /var/log/td-agent/buffer/forward_aws
|
67
|
+
time_slice_wait 1m
|
68
|
+
utc
|
69
|
+
time_slice_format %Y/%m/%d/%H/%M
|
70
|
+
</match>
|
71
|
+
```
|
72
|
+
|
73
|
+
### Advanced configuration using forest plugin
|
74
|
+
Use tag as channel
|
75
|
+
```
|
76
|
+
<match **>
|
77
|
+
type forest
|
78
|
+
subtype forward_aws
|
79
|
+
<template>
|
80
|
+
channel ${tag}
|
81
|
+
aws_access_key_id XXXXXXXXXXXXXXXXXXXX
|
82
|
+
aws_secret_access_key XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
83
|
+
|
84
|
+
aws_s3_endpoint s3-ap-northeast-1.amazonaws.com
|
85
|
+
aws_s3_bucketname XXXXXXXXXXXXXXXXXXXX
|
86
|
+
|
87
|
+
aws_sns_endpoint sns.ap-northeast-1.amazonaws.com
|
88
|
+
aws_sns_topic_arn arn:aws:sns:ap-northeast-1:XXXXXXXXXXXXXXXXXXXX
|
89
|
+
|
90
|
+
buffer_type file
|
91
|
+
buffer_path /var/log/td-agent/buffer/forward_aws-${tag}
|
92
|
+
time_slice_wait 1m
|
93
|
+
utc
|
94
|
+
time_slice_format %Y/%m/%d/%H/%M
|
95
|
+
</template>
|
96
|
+
</match>
|
97
|
+
```
|
98
|
+
|
99
|
+
## In Plugin Configuration
|
100
|
+
|
101
|
+
###Required AWS permission
|
102
|
+
+ s3:GetObject
|
103
|
+
+ sqs:ReceiveMessage
|
104
|
+
+ sqs:DeleteMessage
|
105
|
+
|
106
|
+
### Basic configuration
|
107
|
+
Default config is to listen to "default" channel.
|
108
|
+
```
|
109
|
+
<source>
|
110
|
+
type forward_aws
|
111
|
+
aws_access_key_id XXXXXXXXXXXXXXXXXXXX
|
112
|
+
aws_secret_access_key XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
113
|
+
|
114
|
+
aws_s3_endpoint s3-ap-northeast-1.amazonaws.com
|
115
|
+
aws_s3_bucketname XXXXXXXXXXXXXXXXXXXX
|
116
|
+
|
117
|
+
aws_sqs_endpoint sqs.ap-northeast-1.amazonaws.com
|
118
|
+
aws_sqs_queue_url https://sqs.ap-northeast-1.amazonaws.com/XXXXXXXXXXXXXXXXXXXX
|
119
|
+
</source>
|
120
|
+
```
|
121
|
+
|
122
|
+
### Advanced configuration
|
123
|
+
Use regex to filter channel
|
124
|
+
```
|
125
|
+
<source>
|
126
|
+
type forward_aws
|
127
|
+
channel .*
|
128
|
+
channelEnableRegEx true
|
129
|
+
aws_access_key_id XXXXXXXXXXXXXXXXXXXX
|
130
|
+
aws_secret_access_key XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
131
|
+
|
132
|
+
aws_s3_endpoint s3-ap-northeast-1.amazonaws.com
|
133
|
+
aws_s3_bucketname XXXXXXXXXXXXXXXXXXXX
|
134
|
+
|
135
|
+
aws_sqs_endpoint sqs.ap-northeast-1.amazonaws.com
|
136
|
+
aws_sqs_queue_url https://sqs.ap-northeast-1.amazonaws.com/XXXXXXXXXXXXXXXXXXXX
|
137
|
+
</source>
|
138
|
+
```
|
139
|
+
|
140
|
+
## Tips
|
141
|
+
### How to delete buffer objects on S3
|
142
|
+
Forward-AWS plugin do not delete buffer objects on S3.
|
143
|
+
Use [S3's lifecycle management](http://docs.aws.amazon.com/AmazonS3/latest/dev/manage-lifecycle-using-console.html) to automatically archive or delete old buffer objects.
|
144
|
+
|
145
|
+
### How to use buffer objects as raw input
|
146
|
+
Each buffer object is msgpack stream object with gzip compression.
|
147
|
+
|
148
|
+
### Want to forward log per minute, but have archive per day
|
149
|
+
You can configure forward per minute, and setup another receiver for archiving.
|
150
|
+
|
151
|
+
### Skipping test at startup
|
152
|
+
You can control startup test by following optional parameters. Default value is false
|
153
|
+
+ aws_s3_skiptest
|
154
|
+
+ aws_sns_skiptest
|
155
|
+
+ aws_sqs_skiptest
|
156
|
+
|
157
|
+
|
158
|
+
## Contributing
|
159
|
+
I am newbie for both of Ruby and Fluentd.
|
160
|
+
Feel free to send me pull request.
|
161
|
+
|
162
|
+
1. Fork it
|
163
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
164
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
165
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
166
|
+
5. Create new Pull Request
|
167
|
+
|
168
|
+
## Copyright
|
169
|
+
|
170
|
+
Copyright:: Copyright (c) 2013 Tomohisa Ota
|
171
|
+
License:: Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
|
5
|
+
Gem::Specification.new do |gem|
|
6
|
+
gem.name = "fluent-plugin-forward-aws"
|
7
|
+
gem.version = "0.1.1"
|
8
|
+
gem.authors = ["Tomohisa Ota"]
|
9
|
+
gem.email = ["tomohisa.ota+github@gmail.com"]
|
10
|
+
gem.description = "Fluentd Forward Plugin using Amazon Web Service"
|
11
|
+
gem.summary = gem.description
|
12
|
+
gem.homepage = ""
|
13
|
+
|
14
|
+
gem.files = `git ls-files`.split($/)
|
15
|
+
gem.files.reject! { |fn| fn.include? "doc/" }
|
16
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
17
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
|
+
gem.require_paths = ["lib"]
|
19
|
+
|
20
|
+
gem.add_runtime_dependency "fluentd"
|
21
|
+
gem.add_runtime_dependency "aws-sdk", "~> 1.8.2"
|
22
|
+
end
|
@@ -0,0 +1,198 @@
|
|
1
|
+
class Fluent::ForwardAWSInput < Fluent::Input
|
2
|
+
Fluent::Plugin.register_input('forward_aws', self)
|
3
|
+
|
4
|
+
# config_param :hoge, :string, :default => 'hoge'
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
super
|
8
|
+
require 'aws-sdk'
|
9
|
+
require 'zlib'
|
10
|
+
require 'tempfile'
|
11
|
+
require 'json'
|
12
|
+
@locker = Mutex::new
|
13
|
+
end
|
14
|
+
|
15
|
+
config_param :channel, :string, :default => "default"
|
16
|
+
config_param :channelEnableRegEx, :bool, :default => false
|
17
|
+
|
18
|
+
config_param :aws_access_key_id, :string, :default => nil
|
19
|
+
config_param :aws_secret_access_key, :string, :default => nil
|
20
|
+
|
21
|
+
config_param :aws_s3_endpoint, :string, :default => nil
|
22
|
+
config_param :aws_s3_bucketname, :string, :default => nil
|
23
|
+
config_param :aws_s3_skiptest, :bool, :default => false
|
24
|
+
|
25
|
+
config_param :aws_sqs_endpoint, :string, :default => nil
|
26
|
+
config_param :aws_sqs_queue_url, :string, :default => nil
|
27
|
+
config_param :aws_sqs_skiptest, :bool, :default => false
|
28
|
+
|
29
|
+
# Not documented parameters. Subject to change in future release
|
30
|
+
config_param :aws_sqs_process_interval, :integer, :default => 1
|
31
|
+
config_param :aws_sqs_monitor_interval, :integer, :default => 10
|
32
|
+
config_param :aws_s3_testobjectname, :string, :default => "Config Check Test Object"
|
33
|
+
config_param :start_thread, :bool, :default => true
|
34
|
+
|
35
|
+
def configure(conf)
|
36
|
+
super
|
37
|
+
if /^\s*$/ =~ @channel
|
38
|
+
raise Fluent::ConfigError.new("channel is invalid. Exp=[\w]+")
|
39
|
+
end
|
40
|
+
unless @aws_access_key_id
|
41
|
+
raise Fluent::ConfigError.new("aws_access_key_id is required")
|
42
|
+
end
|
43
|
+
unless @aws_secret_access_key
|
44
|
+
raise Fluent::ConfigError.new("aws_secret_access_key is required")
|
45
|
+
end
|
46
|
+
unless @aws_s3_endpoint
|
47
|
+
raise Fluent::ConfigError.new("aws_s3_endpoint is required")
|
48
|
+
end
|
49
|
+
unless @aws_s3_bucketname
|
50
|
+
raise Fluent::ConfigError.new("aws_s3_bucketname is required")
|
51
|
+
end
|
52
|
+
unless @aws_sqs_endpoint
|
53
|
+
raise Fluent::ConfigError.new("aws_sqs_endpoint is required")
|
54
|
+
end
|
55
|
+
unless @aws_sqs_queue_url
|
56
|
+
raise Fluent::ConfigError.new("aws_sqs_queue_url is required")
|
57
|
+
end
|
58
|
+
unless(@aws_s3_skiptest)
|
59
|
+
init_aws_s3_bucket()
|
60
|
+
begin
|
61
|
+
@bucket.objects[@aws_s3_testobjectname].write("TEST", :content_type => 'text/plain')
|
62
|
+
rescue
|
63
|
+
raise Fluent::ConfigError.new("Cannot put object to S3. Need s3:PutObject permission for resource arn:aws:s3:::" + @aws_s3_bucketname+"/*")
|
64
|
+
end
|
65
|
+
end
|
66
|
+
unless(@aws_sqs_skiptest)
|
67
|
+
init_aws_sqs_queue()
|
68
|
+
begin
|
69
|
+
@queue.receive_message() do |msg|
|
70
|
+
end
|
71
|
+
rescue => e
|
72
|
+
raise Fluent::ConfigError.new("Cannot fetch queue from SQS. Need sqs:ReceiveMessage sqs:DeleteMessage permission for resource " + @aws_sqs_queue_url)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def start
|
78
|
+
super
|
79
|
+
init_aws_s3_bucket()
|
80
|
+
init_aws_sqs_queue()
|
81
|
+
if(@start_thread)
|
82
|
+
@thread = Thread.new(&method(:run))
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def run
|
87
|
+
@running = true
|
88
|
+
while true
|
89
|
+
msg = @queue.receive_message()
|
90
|
+
if msg
|
91
|
+
if(process(JSON.parse(msg.as_sns_message.body)))
|
92
|
+
msg.delete()
|
93
|
+
end
|
94
|
+
sleep @aws_sqs_process_interval
|
95
|
+
@locker.synchronize do
|
96
|
+
return unless @running
|
97
|
+
end
|
98
|
+
next
|
99
|
+
end
|
100
|
+
sleep @aws_sqs_monitor_interval
|
101
|
+
@locker.synchronize do
|
102
|
+
return unless @running
|
103
|
+
end
|
104
|
+
end
|
105
|
+
rescue => e
|
106
|
+
puts e
|
107
|
+
end
|
108
|
+
|
109
|
+
def process(msg)
|
110
|
+
if msg["type"] == "ping"
|
111
|
+
# Ignore ping message
|
112
|
+
return true
|
113
|
+
end
|
114
|
+
if msg["type"] == "out"
|
115
|
+
# Silently ignore non matching logs
|
116
|
+
if msg["bucketname"] != @aws_s3_bucketname
|
117
|
+
return true
|
118
|
+
end
|
119
|
+
if(@channelEnableRegEx)
|
120
|
+
return true unless Regexp.new(@channel).match(msg["channel"])
|
121
|
+
else
|
122
|
+
return true unless @channel == msg["channel"]
|
123
|
+
end
|
124
|
+
|
125
|
+
tmpFile = Tempfile.new("forward-aws-")
|
126
|
+
begin
|
127
|
+
#Download log file to temporary file
|
128
|
+
@bucket.objects[msg["path"]].read do |chunk|
|
129
|
+
tmpFile.write(chunk)
|
130
|
+
end
|
131
|
+
tmpFile.close()
|
132
|
+
#gunzip and decode log file
|
133
|
+
streamUnpacker = MessagePack::Unpacker.new()
|
134
|
+
Zlib::GzipReader.open(tmpFile) {|reader|
|
135
|
+
streamUnpacker.feed(reader.read())
|
136
|
+
streamUnpacker.each {|event|
|
137
|
+
(tag, time, record) = event
|
138
|
+
Fluent::Engine.emit(tag,time,record)
|
139
|
+
}
|
140
|
+
}
|
141
|
+
return true
|
142
|
+
ensure
|
143
|
+
tmp.close(true) rescue nil
|
144
|
+
end
|
145
|
+
end
|
146
|
+
# Unknown notification. Do not delete
|
147
|
+
return false
|
148
|
+
end
|
149
|
+
|
150
|
+
def shutdown
|
151
|
+
super
|
152
|
+
# Stop Thread and join
|
153
|
+
@locker.synchronize do
|
154
|
+
@running = false
|
155
|
+
end
|
156
|
+
if(@thread)
|
157
|
+
@thread.run
|
158
|
+
@thread.join
|
159
|
+
@thread = nil
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
private
|
164
|
+
|
165
|
+
def check_aws_credential
|
166
|
+
unless @aws_access_key_id
|
167
|
+
raise Fluent::ConfigError.new("aws_access_key_id is required")
|
168
|
+
end
|
169
|
+
unless @aws_secret_access_key
|
170
|
+
raise Fluent::ConfigError.new("aws_secret_access_key is required")
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
private
|
175
|
+
|
176
|
+
def init_aws_s3_bucket
|
177
|
+
unless @bucket
|
178
|
+
options = {}
|
179
|
+
options[:access_key_id] = @aws_access_key_id
|
180
|
+
options[:secret_access_key] = @aws_secret_access_key
|
181
|
+
options[:s3_endpoint] = @aws_s3_endpoint
|
182
|
+
options[:use_ssl] = true
|
183
|
+
s3 = AWS::S3.new(options)
|
184
|
+
@bucket = s3.buckets[@aws_s3_bucketname]
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
def init_aws_sqs_queue
|
189
|
+
unless @queue
|
190
|
+
options = {}
|
191
|
+
options[:access_key_id] = @aws_access_key_id
|
192
|
+
options[:secret_access_key] = @aws_secret_access_key
|
193
|
+
options[:sqs_endpoint] = @aws_sqs_endpoint
|
194
|
+
sqs = AWS::SQS.new(options)
|
195
|
+
@queue = sqs.queues[@aws_sqs_queue_url]
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
@@ -0,0 +1,147 @@
|
|
1
|
+
class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
|
2
|
+
Fluent::Plugin.register_output('forward_aws', self)
|
3
|
+
|
4
|
+
# config_param :hoge, :string, :default => 'hoge'
|
5
|
+
|
6
|
+
config_param :channel, :string, :default => "default"
|
7
|
+
|
8
|
+
config_param :aws_access_key_id, :string, :default => nil
|
9
|
+
config_param :aws_secret_access_key, :string, :default => nil
|
10
|
+
|
11
|
+
config_param :aws_s3_endpoint, :string, :default => nil
|
12
|
+
config_param :aws_s3_bucketname, :string, :default => nil
|
13
|
+
config_param :aws_s3_skiptest, :bool, :default => false
|
14
|
+
|
15
|
+
config_param :aws_sns_endpoint, :string, :default => nil
|
16
|
+
config_param :aws_sns_topic_arn, :string, :default => nil
|
17
|
+
config_param :aws_sns_skiptest, :bool, :default => false
|
18
|
+
|
19
|
+
# Not documented parameters. Subject to change in future release
|
20
|
+
config_param :aws_s3_testobjectname, :string, :default => "Config Check Test Object"
|
21
|
+
config_param :aws_sns_emailsubject, :string, :default => "SNS Message"
|
22
|
+
|
23
|
+
def initialize
|
24
|
+
super
|
25
|
+
require 'aws-sdk'
|
26
|
+
require 'zlib'
|
27
|
+
require 'tempfile'
|
28
|
+
require 'json'
|
29
|
+
require 'securerandom'
|
30
|
+
end
|
31
|
+
|
32
|
+
def configure(conf)
|
33
|
+
super
|
34
|
+
if /^\s*$/ =~ @channel
|
35
|
+
raise Fluent::ConfigError.new("channel is invalid. Exp=[\w]+")
|
36
|
+
end
|
37
|
+
unless @aws_access_key_id
|
38
|
+
raise Fluent::ConfigError.new("aws_access_key_id is required")
|
39
|
+
end
|
40
|
+
unless @aws_secret_access_key
|
41
|
+
raise Fluent::ConfigError.new("aws_secret_access_key is required")
|
42
|
+
end
|
43
|
+
unless @aws_s3_endpoint
|
44
|
+
raise Fluent::ConfigError.new("aws_s3_endpoint is required")
|
45
|
+
end
|
46
|
+
unless @aws_s3_bucketname
|
47
|
+
raise Fluent::ConfigError.new("aws_s3_bucketname is required")
|
48
|
+
end
|
49
|
+
unless @aws_sns_endpoint
|
50
|
+
raise Fluent::ConfigError.new("aws_sns_endpoint is required")
|
51
|
+
end
|
52
|
+
unless @aws_sns_topic_arn
|
53
|
+
raise Fluent::ConfigError.new("aws_sns_topic_arn is required")
|
54
|
+
end
|
55
|
+
unless(@aws_s3_skiptest)
|
56
|
+
init_aws_s3_bucket()
|
57
|
+
begin
|
58
|
+
@bucket.objects[@aws_s3_testobjectname].write("TEST", :content_type => 'text/plain')
|
59
|
+
rescue
|
60
|
+
raise Fluent::ConfigError.new("Cannot put object to S3. Need s3:PutObject permission for resource arn:aws:s3:::" + @aws_s3_bucketname+"/*")
|
61
|
+
end
|
62
|
+
end
|
63
|
+
unless(@aws_sns_skiptest)
|
64
|
+
init_aws_sns_topic()
|
65
|
+
begin
|
66
|
+
notification = {
|
67
|
+
"type" => "ping"
|
68
|
+
}
|
69
|
+
topic = @sns.topics[@aws_sns_topic_arn]
|
70
|
+
topic.publish(JSON.pretty_generate(notification), :subject => @aws_sns_emailsubject)
|
71
|
+
rescue
|
72
|
+
raise Fluent::ConfigError.new("Cannot post notification to SNS. Need sns:Publish permission for resource " + @aws_sns_topic_arn)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def start
|
78
|
+
super
|
79
|
+
init_aws_s3_bucket()
|
80
|
+
init_aws_sns_topic()
|
81
|
+
end
|
82
|
+
|
83
|
+
def shutdown
|
84
|
+
super
|
85
|
+
# destroy
|
86
|
+
end
|
87
|
+
|
88
|
+
def format(tag, time, record)
|
89
|
+
[tag, time, record].to_msgpack
|
90
|
+
end
|
91
|
+
|
92
|
+
def write(chunk)
|
93
|
+
#Add UUID to avoid name conflict
|
94
|
+
#Current version supports only msgpack + gz, but add suffix for future extentions
|
95
|
+
format = "msgpack"
|
96
|
+
compression = "gzip"
|
97
|
+
compression_suffix = "gz"
|
98
|
+
s3path = "#{@channel}/#{chunk.key}/#{SecureRandom.uuid}.#{format}.#{compression_suffix}"
|
99
|
+
|
100
|
+
# Create temp gzip file
|
101
|
+
tmpFile = Tempfile.new("forward-aws-")
|
102
|
+
writer = Zlib::GzipWriter.new(tmpFile)
|
103
|
+
begin
|
104
|
+
chunk.write_to(writer)
|
105
|
+
writer.close
|
106
|
+
@bucket.objects[s3path].write(Pathname.new(tmpFile.path), :content_type => 'application/x-gzip')
|
107
|
+
notification = {
|
108
|
+
"type" => "out",
|
109
|
+
"channel" => @channel,
|
110
|
+
"bucketname" => @aws_s3_bucketname,
|
111
|
+
"path" => s3path,
|
112
|
+
"format" => format,
|
113
|
+
"compression" => compression
|
114
|
+
}
|
115
|
+
topic = @sns.topics[@aws_sns_topic_arn]
|
116
|
+
topic.publish(JSON.pretty_generate(notification), :subject => @aws_sns_emailsubject)
|
117
|
+
ensure
|
118
|
+
writer.close rescue nil
|
119
|
+
tmp.close(true) rescue nil
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
private
|
124
|
+
|
125
|
+
def init_aws_s3_bucket
|
126
|
+
unless @bucket
|
127
|
+
options = {}
|
128
|
+
options[:access_key_id] = @aws_access_key_id
|
129
|
+
options[:secret_access_key] = @aws_secret_access_key
|
130
|
+
options[:s3_endpoint] = @aws_s3_endpoint
|
131
|
+
options[:use_ssl] = true
|
132
|
+
s3 = AWS::S3.new(options)
|
133
|
+
@bucket = s3.buckets[@aws_s3_bucketname]
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def init_aws_sns_topic
|
138
|
+
unless @sns
|
139
|
+
options = {}
|
140
|
+
options[:access_key_id] = @aws_access_key_id
|
141
|
+
options[:secret_access_key] = @aws_secret_access_key
|
142
|
+
options[:sns_endpoint] = @aws_sns_endpoint
|
143
|
+
@sns = AWS::SNS.new(options)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
end
|
@@ -0,0 +1,9 @@
|
|
1
|
+
aws_s3_endpoint: s3-ap-northeast-1.amazonaws.com
|
2
|
+
aws_sns_endpoint: sns.ap-northeast-1.amazonaws.com
|
3
|
+
aws_sqs_endpoint: sqs.ap-northeast-1.amazonaws.com
|
4
|
+
|
5
|
+
aws_access_key_id:
|
6
|
+
aws_secret_access_key:
|
7
|
+
aws_s3_bucketname:
|
8
|
+
aws_sns_topic_arn:
|
9
|
+
aws_sqs_queue_url:
|
data/test/helper.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
unless ENV.has_key?('VERBOSE')
|
16
|
+
nulllogger = Object.new
|
17
|
+
nulllogger.instance_eval {|obj|
|
18
|
+
def method_missing(method, *args)
|
19
|
+
# pass
|
20
|
+
end
|
21
|
+
}
|
22
|
+
$log = nulllogger
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'fluent/plugin/in_forward_aws'
|
26
|
+
require 'fluent/plugin/out_forward_aws'
|
27
|
+
|
28
|
+
class Test::Unit::TestCase
|
29
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class ForwardAWSInputTest < Test::Unit::TestCase
|
4
|
+
DUMMYCONFIG = %[
|
5
|
+
aws_access_key_id TEST_AWS_ACCESS_KEY_ID
|
6
|
+
aws_secret_access_key TEST_AWS_SECRET_ACCESS_KEY
|
7
|
+
|
8
|
+
aws_s3_endpoint TEST_AWS_S3_ENDPOINT
|
9
|
+
aws_s3_bucketname TEST_AWS_S3_BUCKETNAME
|
10
|
+
aws_s3_skiptest true
|
11
|
+
|
12
|
+
aws_sqs_endpoint TEST_AWS_SQS_ENDPOINT
|
13
|
+
aws_sqs_queue_url TEST_AWS_SQS_QUEUE_URL
|
14
|
+
aws_sqs_skiptest true
|
15
|
+
|
16
|
+
start_thread false
|
17
|
+
]
|
18
|
+
|
19
|
+
def setup
|
20
|
+
Fluent::Test.setup
|
21
|
+
begin
|
22
|
+
require 'yaml'
|
23
|
+
@AWSTESTCONFIG = YAML.load_file(File.expand_path('../../awsconfig.yml', __FILE__))
|
24
|
+
|
25
|
+
@CONFIG = %[
|
26
|
+
aws_access_key_id #{@AWSTESTCONFIG["aws_access_key_id"]}
|
27
|
+
aws_secret_access_key #{@AWSTESTCONFIG["aws_secret_access_key"]}
|
28
|
+
|
29
|
+
aws_s3_endpoint #{@AWSTESTCONFIG["aws_s3_endpoint"]}
|
30
|
+
aws_s3_bucketname #{@AWSTESTCONFIG["aws_s3_bucketname"]}
|
31
|
+
aws_s3_skiptest true
|
32
|
+
|
33
|
+
aws_sqs_endpoint #{@AWSTESTCONFIG["aws_sqs_endpoint"]}
|
34
|
+
aws_sqs_queue_url #{@AWSTESTCONFIG["aws_sqs_queue_url"]}
|
35
|
+
aws_sqs_skiptest true
|
36
|
+
|
37
|
+
start_thread false
|
38
|
+
]
|
39
|
+
rescue => e
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def create_driver(conf)
|
44
|
+
Fluent::Test::InputTestDriver.new(Fluent::ForwardAWSInput).configure(conf)
|
45
|
+
end
|
46
|
+
|
47
|
+
def test_configure
|
48
|
+
d = create_driver(DUMMYCONFIG)
|
49
|
+
### check configurations
|
50
|
+
assert_equal( 'TEST_AWS_ACCESS_KEY_ID', d.instance.aws_access_key_id)
|
51
|
+
assert_equal( 'TEST_AWS_SECRET_ACCESS_KEY', d.instance.aws_secret_access_key)
|
52
|
+
|
53
|
+
assert_equal( 'TEST_AWS_S3_ENDPOINT', d.instance.aws_s3_endpoint)
|
54
|
+
assert_equal( 'TEST_AWS_S3_BUCKETNAME', d.instance.aws_s3_bucketname)
|
55
|
+
|
56
|
+
assert_equal( 'TEST_AWS_SQS_ENDPOINT', d.instance.aws_sqs_endpoint)
|
57
|
+
assert_equal( 'TEST_AWS_SQS_QUEUE_URL', d.instance.aws_sqs_queue_url)
|
58
|
+
end
|
59
|
+
|
60
|
+
def test_check_aws_s3
|
61
|
+
unless(@CONFIG)
|
62
|
+
# Skip Test
|
63
|
+
return
|
64
|
+
end
|
65
|
+
create_driver(@CONFIG + "aws_s3_skiptest false").run()
|
66
|
+
end
|
67
|
+
|
68
|
+
def test_check_aws_sqs
|
69
|
+
unless(@CONFIG)
|
70
|
+
# Skip Test
|
71
|
+
return
|
72
|
+
end
|
73
|
+
create_driver(@CONFIG + "aws_sqs_skiptest false").run()
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class ForwardAWSOutputTest < Test::Unit::TestCase
|
4
|
+
DUMMYCONFIG = %[
|
5
|
+
aws_access_key_id TEST_AWS_ACCESS_KEY_ID
|
6
|
+
aws_secret_access_key TEST_AWS_SECRET_ACCESS_KEY
|
7
|
+
|
8
|
+
aws_s3_endpoint TEST_AWS_S3_ENDPOINT
|
9
|
+
aws_s3_bucketname TEST_AWS_S3_BUCKETNAME
|
10
|
+
aws_s3_skiptest true
|
11
|
+
|
12
|
+
aws_sns_endpoint TEST_AWS_SNS_ENDPOINT
|
13
|
+
aws_sns_topic_arn TEST_AWS_SNS_TOPIC_ARN
|
14
|
+
aws_sns_skiptest true
|
15
|
+
|
16
|
+
buffer_type memory
|
17
|
+
]
|
18
|
+
|
19
|
+
def setup
|
20
|
+
Fluent::Test.setup
|
21
|
+
|
22
|
+
begin
|
23
|
+
require 'yaml'
|
24
|
+
@AWSTESTCONFIG = YAML.load_file(File.expand_path('../../awsconfig.yml', __FILE__))
|
25
|
+
|
26
|
+
@CONFIG = %[
|
27
|
+
aws_access_key_id #{@AWSTESTCONFIG["aws_access_key_id"]}
|
28
|
+
aws_secret_access_key #{@AWSTESTCONFIG["aws_secret_access_key"]}
|
29
|
+
|
30
|
+
aws_s3_endpoint #{@AWSTESTCONFIG["aws_s3_endpoint"]}
|
31
|
+
aws_s3_bucketname #{@AWSTESTCONFIG["aws_s3_bucketname"]}
|
32
|
+
aws_s3_skiptest true
|
33
|
+
|
34
|
+
aws_sns_endpoint #{@AWSTESTCONFIG["aws_sns_endpoint"]}
|
35
|
+
aws_sns_topic_arn #{@AWSTESTCONFIG["aws_sns_topic_arn"]}
|
36
|
+
aws_sns_skiptest true
|
37
|
+
|
38
|
+
buffer_type memory
|
39
|
+
]
|
40
|
+
rescue => e
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def create_driver(conf)
|
45
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::ForwardAWSOutput) do
|
46
|
+
def write(chunk)
|
47
|
+
chunk.read
|
48
|
+
end
|
49
|
+
end.configure(conf)
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_configure
|
53
|
+
d = create_driver(DUMMYCONFIG)
|
54
|
+
### check configurations
|
55
|
+
assert_equal( 'TEST_AWS_ACCESS_KEY_ID', d.instance.aws_access_key_id)
|
56
|
+
assert_equal( 'TEST_AWS_SECRET_ACCESS_KEY', d.instance.aws_secret_access_key)
|
57
|
+
|
58
|
+
assert_equal( 'TEST_AWS_S3_ENDPOINT', d.instance.aws_s3_endpoint)
|
59
|
+
assert_equal( 'TEST_AWS_S3_BUCKETNAME', d.instance.aws_s3_bucketname)
|
60
|
+
|
61
|
+
assert_equal( 'TEST_AWS_SNS_ENDPOINT', d.instance.aws_sns_endpoint)
|
62
|
+
assert_equal( 'TEST_AWS_SNS_TOPIC_ARN', d.instance.aws_sns_topic_arn)
|
63
|
+
end
|
64
|
+
|
65
|
+
def test_format
|
66
|
+
d = create_driver(DUMMYCONFIG)
|
67
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
68
|
+
d.emit({"a"=>1}, time)
|
69
|
+
d.emit({"a"=>2}, time)
|
70
|
+
|
71
|
+
d.expect_format ["test",time,{"a"=>1}].to_msgpack
|
72
|
+
d.expect_format ["test",time,{"a"=>2}].to_msgpack
|
73
|
+
d.run
|
74
|
+
end
|
75
|
+
|
76
|
+
def test_check_aws_s3
|
77
|
+
unless(@CONFIG)
|
78
|
+
# Skip Test
|
79
|
+
return
|
80
|
+
end
|
81
|
+
create_driver(@CONFIG + "aws_s3_skiptest false").run()
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_check_aws_sns
|
85
|
+
unless(@CONFIG)
|
86
|
+
# Skip Test
|
87
|
+
return
|
88
|
+
end
|
89
|
+
create_driver(@CONFIG + "aws_sns_skiptest false").run()
|
90
|
+
end
|
91
|
+
end
|
metadata
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-forward-aws
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Tomohisa Ota
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-02-27 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fluentd
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: aws-sdk
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: 1.8.2
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 1.8.2
|
46
|
+
description: Fluentd Forward Plugin using Amazon Web Service
|
47
|
+
email:
|
48
|
+
- tomohisa.ota+github@gmail.com
|
49
|
+
executables: []
|
50
|
+
extensions: []
|
51
|
+
extra_rdoc_files: []
|
52
|
+
files:
|
53
|
+
- .gitignore
|
54
|
+
- Gemfile
|
55
|
+
- LICENSE.txt
|
56
|
+
- README.md
|
57
|
+
- Rakefile
|
58
|
+
- fluent-plugin-forward-aws.gemspec
|
59
|
+
- lib/fluent/plugin/in_forward_aws.rb
|
60
|
+
- lib/fluent/plugin/out_forward_aws.rb
|
61
|
+
- test/awsconfig.yml.sample
|
62
|
+
- test/helper.rb
|
63
|
+
- test/plugin/test_in_foward_aws.rb
|
64
|
+
- test/plugin/test_out_foward_aws.rb
|
65
|
+
homepage: ''
|
66
|
+
licenses: []
|
67
|
+
post_install_message:
|
68
|
+
rdoc_options: []
|
69
|
+
require_paths:
|
70
|
+
- lib
|
71
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - ! '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
78
|
+
none: false
|
79
|
+
requirements:
|
80
|
+
- - ! '>='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
requirements: []
|
84
|
+
rubyforge_project:
|
85
|
+
rubygems_version: 1.8.25
|
86
|
+
signing_key:
|
87
|
+
specification_version: 3
|
88
|
+
summary: Fluentd Forward Plugin using Amazon Web Service
|
89
|
+
test_files:
|
90
|
+
- test/awsconfig.yml.sample
|
91
|
+
- test/helper.rb
|
92
|
+
- test/plugin/test_in_foward_aws.rb
|
93
|
+
- test/plugin/test_out_foward_aws.rb
|