fluent-plugin-forward-aws 0.1.4 → 0.1.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/fluent-plugin-forward-aws.gemspec +1 -1
- data/lib/fluent/plugin/in_forward_aws.rb +29 -14
- metadata +5 -11
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 82dcc40302aaebc4f395387518c5bc0ad5b10964
|
4
|
+
data.tar.gz: 0c05f8555f790f65d2472d6bd05065f97d99c060
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6cb2f549924e3a261a33b8d691ae51899d60dcdaa57925f551ead3ee585ded3726465f719451fd6d07d6be38947a16c93ac5dfcc06f0e33907996263b7999953
|
7
|
+
data.tar.gz: afaeaad61e2e74be0f7795382474d1f5ae0097253abd3d3a7f3767eeff778e65d684fae0238bf053d065e86e3dcdf7ea672a127b0e12aa41c152f189c479fc2b
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "fluent-plugin-forward-aws"
|
7
|
-
gem.version = "0.1.
|
7
|
+
gem.version = "0.1.5"
|
8
8
|
gem.authors = ["Tomohisa Ota"]
|
9
9
|
gem.email = ["tomohisa.ota+github@gmail.com"]
|
10
10
|
gem.description = "Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)"
|
@@ -33,11 +33,13 @@ class Fluent::ForwardAWSInput < Fluent::Input
|
|
33
33
|
config_param :dry_run, :bool, :default => false
|
34
34
|
|
35
35
|
# Not documented parameters. Subject to change in future release
|
36
|
-
config_param :aws_sqs_process_interval, :integer, :default =>
|
36
|
+
config_param :aws_sqs_process_interval, :integer, :default => 0
|
37
37
|
config_param :aws_sqs_monitor_interval, :integer, :default => 10
|
38
|
+
config_param :aws_sqs_limit, :integer, :default => 10
|
38
39
|
config_param :aws_s3_testobjectname, :string, :default => "Config Check Test Object"
|
39
40
|
config_param :start_thread, :bool, :default => true
|
40
41
|
|
42
|
+
|
41
43
|
def configure(conf)
|
42
44
|
super
|
43
45
|
if /^\s*$/ =~ @channel
|
@@ -93,21 +95,26 @@ class Fluent::ForwardAWSInput < Fluent::Input
|
|
93
95
|
@running = true
|
94
96
|
while true
|
95
97
|
$log.debug "Polling SQS"
|
96
|
-
|
97
|
-
if
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
98
|
+
notificationRaws = @queue.receive_message({:limit => @aws_sqs_limit})
|
99
|
+
if(notificationRaws && !notificationRaws.instance_of?(Array))
|
100
|
+
notificationRaws = [notificationRaws]
|
101
|
+
end
|
102
|
+
if notificationRaws && notificationRaws.size != 0
|
103
|
+
notificationRaws.each{ |notificationRaw|
|
104
|
+
notification = JSON.parse(notificationRaw.as_sns_message.body)
|
105
|
+
$log.debug "Received Notification#{notification}"
|
106
|
+
if(process(notification))
|
107
|
+
if dry_run
|
108
|
+
$log.info "Notification processed in dry-run mode #{notification}"
|
109
|
+
else
|
110
|
+
notificationRaw.delete()
|
111
|
+
$log.debug "Deleted processed notification #{notification}"
|
112
|
+
end
|
103
113
|
else
|
104
|
-
|
105
|
-
$log.debug "Deleted processed notification #{notification}"
|
114
|
+
$log.error "Could not process notification, pending... #{notification}"
|
106
115
|
end
|
107
|
-
|
108
|
-
|
109
|
-
end
|
110
|
-
sleep @aws_sqs_process_interval
|
116
|
+
}
|
117
|
+
sleep @aws_sqs_process_interval if(@aws_sqs_process_interval > 0)
|
111
118
|
@locker.synchronize do
|
112
119
|
return unless @running
|
113
120
|
end
|
@@ -163,6 +170,14 @@ class Fluent::ForwardAWSInput < Fluent::Input
|
|
163
170
|
}
|
164
171
|
}
|
165
172
|
return true
|
173
|
+
rescue => e
|
174
|
+
if(e.message == "Access Denied")
|
175
|
+
$log.warn "Access Denied for key #{notification["path"]}"
|
176
|
+
# Object may have been deleted. Do not retry
|
177
|
+
return true
|
178
|
+
else
|
179
|
+
$log.error e
|
180
|
+
end
|
166
181
|
ensure
|
167
182
|
tmp.close(true) rescue nil
|
168
183
|
end
|
metadata
CHANGED
@@ -1,20 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-forward-aws
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
5
|
-
prerelease:
|
4
|
+
version: 0.1.5
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Tomohisa Ota
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2013-03-
|
11
|
+
date: 2013-03-04 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: fluentd
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
17
|
- - ! '>='
|
20
18
|
- !ruby/object:Gem::Version
|
@@ -22,7 +20,6 @@ dependencies:
|
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
24
|
- - ! '>='
|
28
25
|
- !ruby/object:Gem::Version
|
@@ -30,7 +27,6 @@ dependencies:
|
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: aws-sdk
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
31
|
- - ~>
|
36
32
|
- !ruby/object:Gem::Version
|
@@ -38,7 +34,6 @@ dependencies:
|
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
38
|
- - ~>
|
44
39
|
- !ruby/object:Gem::Version
|
@@ -66,27 +61,26 @@ files:
|
|
66
61
|
- test/plugin/test_out_foward_aws.rb
|
67
62
|
homepage: http://github.com/tomohisaota/fluent-plugin-forward-aws
|
68
63
|
licenses: []
|
64
|
+
metadata: {}
|
69
65
|
post_install_message:
|
70
66
|
rdoc_options: []
|
71
67
|
require_paths:
|
72
68
|
- lib
|
73
69
|
required_ruby_version: !ruby/object:Gem::Requirement
|
74
|
-
none: false
|
75
70
|
requirements:
|
76
71
|
- - ! '>='
|
77
72
|
- !ruby/object:Gem::Version
|
78
73
|
version: '0'
|
79
74
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
80
|
-
none: false
|
81
75
|
requirements:
|
82
76
|
- - ! '>='
|
83
77
|
- !ruby/object:Gem::Version
|
84
78
|
version: '0'
|
85
79
|
requirements: []
|
86
80
|
rubyforge_project:
|
87
|
-
rubygems_version:
|
81
|
+
rubygems_version: 2.0.0
|
88
82
|
signing_key:
|
89
|
-
specification_version:
|
83
|
+
specification_version: 4
|
90
84
|
summary: Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)
|
91
85
|
test_files:
|
92
86
|
- test/awsconfig.yml.sample
|