fluent-plugin-forward-aws 0.1.5 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +14 -5
- data/fluent-plugin-forward-aws.gemspec +1 -1
- data/lib/fluent/plugin/in_forward_aws.rb +23 -17
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6a407c6eeec19a09a1eb1536c8dbb7f21fe5ac7e
|
4
|
+
data.tar.gz: 90449701adb6eb32977b87e2c0c6beba11480ec9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c9874949fc9d9969a16910afc606eb3beb7c6373412eec2138c3fdbd31907054ee36608bc1716b7e554e377c052927ec9a650d00144c4de978ee02d2a934913b
|
7
|
+
data.tar.gz: d91d3db1dc5d8e4c2001f3ea63ebf7985ca2e8789649d90f62e9365642f740b6716b67879920be17de0def6171c2af739e4a416753d8f7f446d8463718f79b56
|
data/README.md
CHANGED
@@ -121,11 +121,16 @@ Use tag as forward-AWS channel
|
|
121
121
|
## In Plugin Configuration
|
122
122
|
### Parameters
|
123
123
|
|
124
|
-
name
|
125
|
-
|
126
|
-
aws_sqs_endpoint
|
127
|
-
aws_sqs_queue_url
|
128
|
-
aws_sqs_skiptest
|
124
|
+
name | type | description
|
125
|
+
---------------------------|---------------------------------|---------------------------
|
126
|
+
aws_sqs_endpoint | string (required) | [SQS Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#sqs_region) for your topic
|
127
|
+
aws_sqs_queue_url | string (required) | SQS Queue URL (not ARN)
|
128
|
+
aws_sqs_skiptest | bool (default = false) | Skip SQS Related test at startup
|
129
|
+
aws_sqs_wait_time_seconds | integer(default = 5) | long polling
|
130
|
+
aws_sqs_limit | integer(default = 10) | The maximum number of messages to receive
|
131
|
+
aws_sqs_visibilitiy_timeout| integer(default = 300) | Duration that the received messages are hidden
|
132
|
+
|
133
|
+
|
129
134
|
channelEnableRegEx | bool (default = false) | Enabled Regular Expression when checking channel
|
130
135
|
dry_run | bool (default = false) | Do not delete notification after processing
|
131
136
|
|
@@ -173,6 +178,10 @@ Use regex to filter channel
|
|
173
178
|
Forward-AWS plugin do not delete buffer objects on S3.
|
174
179
|
Use [S3's lifecycle management](http://docs.aws.amazon.com/AmazonS3/latest/dev/manage-lifecycle-using-console.html) to automatically archive or delete old buffer objects.
|
175
180
|
|
181
|
+
### SQS long polling
|
182
|
+
forward-AWS uses SQS long polling by default.
|
183
|
+
Long polling could slow down fluentd shutdown sequence.
|
184
|
+
|
176
185
|
### How to use buffer objects as raw input
|
177
186
|
Each buffer object is msgpack stream object with gzip compression.
|
178
187
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "fluent-plugin-forward-aws"
|
7
|
-
gem.version = "0.1.
|
7
|
+
gem.version = "0.1.6"
|
8
8
|
gem.authors = ["Tomohisa Ota"]
|
9
9
|
gem.email = ["tomohisa.ota+github@gmail.com"]
|
10
10
|
gem.description = "Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)"
|
@@ -26,6 +26,9 @@ class Fluent::ForwardAWSInput < Fluent::Input
|
|
26
26
|
config_param :aws_sqs_endpoint, :string, :default => nil
|
27
27
|
config_param :aws_sqs_queue_url, :string, :default => nil
|
28
28
|
config_param :aws_sqs_skiptest, :bool, :default => false
|
29
|
+
config_param :aws_sqs_wait_time_seconds, :integer, :default => 5
|
30
|
+
config_param :aws_sqs_limit, :integer, :default => 10
|
31
|
+
config_param :aws_sqs_visibilitiy_timeout,:integer, :default => 300
|
29
32
|
|
30
33
|
config_param :add_tag_prefix, :string, :default => nil
|
31
34
|
config_param :remove_tag_prefix, :string, :default => nil
|
@@ -33,9 +36,9 @@ class Fluent::ForwardAWSInput < Fluent::Input
|
|
33
36
|
config_param :dry_run, :bool, :default => false
|
34
37
|
|
35
38
|
# Not documented parameters. Subject to change in future release
|
36
|
-
config_param :aws_sqs_process_interval,
|
37
|
-
config_param :aws_sqs_monitor_interval,
|
38
|
-
|
39
|
+
config_param :aws_sqs_process_interval, :integer, :default => 0
|
40
|
+
config_param :aws_sqs_monitor_interval, :integer, :default => 25
|
41
|
+
|
39
42
|
config_param :aws_s3_testobjectname, :string, :default => "Config Check Test Object"
|
40
43
|
config_param :start_thread, :bool, :default => true
|
41
44
|
|
@@ -87,19 +90,25 @@ class Fluent::ForwardAWSInput < Fluent::Input
|
|
87
90
|
init_aws_s3_bucket()
|
88
91
|
init_aws_sqs_queue()
|
89
92
|
if(@start_thread)
|
90
|
-
@thread = Thread.new(&method(:run))
|
93
|
+
@thread = Thread.new(&method(:run)) unless @thread
|
91
94
|
end
|
92
95
|
end
|
93
96
|
|
94
97
|
def run
|
95
98
|
@running = true
|
96
99
|
while true
|
97
|
-
$log.debug "Polling SQS"
|
98
|
-
notificationRaws = @queue.receive_message({
|
100
|
+
$log.debug "Long Polling SQS for #{@aws_sqs_wait_time_seconds} secs with message limit #{@aws_sqs_limit}"
|
101
|
+
notificationRaws = @queue.receive_message({
|
102
|
+
:limit => @aws_sqs_limit,
|
103
|
+
:wait_time_seconds => @aws_sqs_wait_time_seconds,
|
104
|
+
:visibilitiy_timeout => @aws_sqs_visibilitiy_timeout
|
105
|
+
})
|
106
|
+
$log.debug "Polling finished"
|
99
107
|
if(notificationRaws && !notificationRaws.instance_of?(Array))
|
100
108
|
notificationRaws = [notificationRaws]
|
101
109
|
end
|
102
110
|
if notificationRaws && notificationRaws.size != 0
|
111
|
+
$log.debug "Received #{notificationRaws.size} messages"
|
103
112
|
notificationRaws.each{ |notificationRaw|
|
104
113
|
notification = JSON.parse(notificationRaw.as_sns_message.body)
|
105
114
|
$log.debug "Received Notification#{notification}"
|
@@ -114,12 +123,19 @@ class Fluent::ForwardAWSInput < Fluent::Input
|
|
114
123
|
$log.error "Could not process notification, pending... #{notification}"
|
115
124
|
end
|
116
125
|
}
|
126
|
+
@locker.synchronize do
|
127
|
+
return unless @running
|
128
|
+
end
|
117
129
|
sleep @aws_sqs_process_interval if(@aws_sqs_process_interval > 0)
|
118
130
|
@locker.synchronize do
|
119
131
|
return unless @running
|
120
132
|
end
|
121
133
|
next
|
122
134
|
end
|
135
|
+
$log.debug "No messages in queue, sleep for #{@aws_sqs_monitor_interval} secs"
|
136
|
+
@locker.synchronize do
|
137
|
+
return unless @running
|
138
|
+
end
|
123
139
|
sleep @aws_sqs_monitor_interval
|
124
140
|
@locker.synchronize do
|
125
141
|
return unless @running
|
@@ -190,6 +206,7 @@ class Fluent::ForwardAWSInput < Fluent::Input
|
|
190
206
|
super
|
191
207
|
# Stop Thread and join
|
192
208
|
@locker.synchronize do
|
209
|
+
$log.debug "Stopping thread"
|
193
210
|
@running = false
|
194
211
|
end
|
195
212
|
if(@thread)
|
@@ -201,17 +218,6 @@ class Fluent::ForwardAWSInput < Fluent::Input
|
|
201
218
|
|
202
219
|
private
|
203
220
|
|
204
|
-
def check_aws_credential
|
205
|
-
unless @aws_access_key_id
|
206
|
-
raise Fluent::ConfigError.new("aws_access_key_id is required")
|
207
|
-
end
|
208
|
-
unless @aws_secret_access_key
|
209
|
-
raise Fluent::ConfigError.new("aws_secret_access_key is required")
|
210
|
-
end
|
211
|
-
end
|
212
|
-
|
213
|
-
private
|
214
|
-
|
215
221
|
def init_aws_s3_bucket
|
216
222
|
unless @bucket
|
217
223
|
options = {}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-forward-aws
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tomohisa Ota
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-03-
|
11
|
+
date: 2013-03-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|