fluent-plugin-forward-aws 0.1.6 → 0.1.7
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +9 -2
- data/fluent-plugin-forward-aws.gemspec +1 -1
- data/lib/fluent/plugin/in_forward_aws.rb +17 -8
- data/lib/fluent/plugin/out_forward_aws.rb +14 -7
- data/test/plugin/test_out_foward_aws.rb +1 -1
- metadata +11 -8
- checksums.yaml +0 -7
- data/lib/fluent/plugin/forward_aws_util.rb +0 -22
- data/test/plugin/test_forward_aws_util.rb +0 -35
data/README.md
CHANGED
@@ -52,10 +52,17 @@ aws_secret_access_key | string (required) | AWS Secket Access Key
|
|
52
52
|
aws_s3_endpoint | string (required) | [s3 Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) for your bucket
|
53
53
|
aws_s3_bucketname | string (required) | S3 Bucketname
|
54
54
|
aws_s3_skiptest | bool (default = false) | Skip S3 Related test at startup
|
55
|
-
add_tag_prefix | string (default = nil) | Add specified prefix to tag before processing log
|
56
|
-
remove_tag_prefix | string (default = nil) | Remove specified prefix from tag before processing log
|
57
55
|
channel | string (default = "default") | Tag that Forward-AWS plugin uses for grouping logs.
|
58
56
|
|
57
|
+
### Parameters by supported by HandleTagNameMixin
|
58
|
+
name | type
|
59
|
+
----------------------|---------------------
|
60
|
+
remove_tag_prefix | string (default = nil)
|
61
|
+
add_tag_prefix | string (default = nil)
|
62
|
+
remove_tag_suffix | string (default = nil)
|
63
|
+
add_tag_suffix | string (default = nil)
|
64
|
+
|
65
|
+
|
59
66
|
## Out Plugin Configuration
|
60
67
|
### Parameters
|
61
68
|
name | type | description
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "fluent-plugin-forward-aws"
|
7
|
-
gem.version = "0.1.
|
7
|
+
gem.version = "0.1.7"
|
8
8
|
gem.authors = ["Tomohisa Ota"]
|
9
9
|
gem.email = ["tomohisa.ota+github@gmail.com"]
|
10
10
|
gem.description = "Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)"
|
@@ -1,9 +1,20 @@
|
|
1
1
|
class Fluent::ForwardAWSInput < Fluent::Input
|
2
2
|
Fluent::Plugin.register_input('forward_aws', self)
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
4
|
+
include Fluent::HandleTagNameMixin
|
5
|
+
|
6
|
+
# Workaround for HandleTagNameMixin bug
|
7
|
+
# https://github.com/fluent/fluentd/pull/109
|
8
|
+
def format_stream(tag, es)
|
9
|
+
out = ''
|
10
|
+
es.each {|time,record|
|
11
|
+
tag_temp = String.new(tag)
|
12
|
+
filter_record(tag_temp, time, record)
|
13
|
+
out << format(tag_temp, time, record)
|
14
|
+
}
|
15
|
+
out
|
16
|
+
end
|
17
|
+
|
7
18
|
def initialize
|
8
19
|
super
|
9
20
|
require 'aws-sdk'
|
@@ -29,10 +40,7 @@ class Fluent::ForwardAWSInput < Fluent::Input
|
|
29
40
|
config_param :aws_sqs_wait_time_seconds, :integer, :default => 5
|
30
41
|
config_param :aws_sqs_limit, :integer, :default => 10
|
31
42
|
config_param :aws_sqs_visibilitiy_timeout,:integer, :default => 300
|
32
|
-
|
33
|
-
config_param :add_tag_prefix, :string, :default => nil
|
34
|
-
config_param :remove_tag_prefix, :string, :default => nil
|
35
|
-
|
43
|
+
|
36
44
|
config_param :dry_run, :bool, :default => false
|
37
45
|
|
38
46
|
# Not documented parameters. Subject to change in future release
|
@@ -181,7 +189,8 @@ class Fluent::ForwardAWSInput < Fluent::Input
|
|
181
189
|
streamUnpacker.feed(reader.read())
|
182
190
|
streamUnpacker.each {|event|
|
183
191
|
(tag, time, record) = event
|
184
|
-
|
192
|
+
# Apply HandleTagNameMixin manually
|
193
|
+
filter_record(tag, time, record)
|
185
194
|
Fluent::Engine.emit(tag,time,record)
|
186
195
|
}
|
187
196
|
}
|
@@ -1,9 +1,20 @@
|
|
1
1
|
class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
|
2
2
|
Fluent::Plugin.register_output('forward_aws', self)
|
3
3
|
|
4
|
-
|
5
|
-
include ForwardAWSUtil
|
4
|
+
include Fluent::HandleTagNameMixin
|
6
5
|
|
6
|
+
# Workaround for HandleTagNameMixin bug
|
7
|
+
# https://github.com/fluent/fluentd/pull/109
|
8
|
+
def format_stream(tag, es)
|
9
|
+
out = ''
|
10
|
+
es.each {|time,record|
|
11
|
+
tag_temp = String.new(tag)
|
12
|
+
filter_record(tag_temp, time, record)
|
13
|
+
out << format(tag_temp, time, record)
|
14
|
+
}
|
15
|
+
out
|
16
|
+
end
|
17
|
+
|
7
18
|
config_param :channel, :string, :default => "default"
|
8
19
|
|
9
20
|
config_param :aws_access_key_id, :string, :default => nil
|
@@ -17,9 +28,6 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
|
|
17
28
|
config_param :aws_sns_topic_arn, :string, :default => nil
|
18
29
|
config_param :aws_sns_skiptest, :bool, :default => false
|
19
30
|
|
20
|
-
config_param :add_tag_prefix, :string, :default => nil
|
21
|
-
config_param :remove_tag_prefix, :string, :default => nil
|
22
|
-
|
23
31
|
# Not documented parameters. Subject to change in future release
|
24
32
|
config_param :aws_s3_testobjectname, :string, :default => "Config Check Test Object"
|
25
33
|
config_param :aws_sns_emailsubject, :string, :default => "SNS Message"
|
@@ -90,7 +98,6 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
|
|
90
98
|
end
|
91
99
|
|
92
100
|
def format(tag, time, record)
|
93
|
-
tag = ForwardAWSUtil.filtertag(tag,@add_tag_prefix,@remove_tag_prefix)
|
94
101
|
[tag, time, record].to_msgpack
|
95
102
|
end
|
96
103
|
|
@@ -126,7 +133,7 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
|
|
126
133
|
tmp.close(true) rescue nil
|
127
134
|
end
|
128
135
|
end
|
129
|
-
|
136
|
+
|
130
137
|
private
|
131
138
|
|
132
139
|
def init_aws_s3_bucket
|
@@ -74,7 +74,7 @@ class ForwardAWSOutputTest < Test::Unit::TestCase
|
|
74
74
|
end
|
75
75
|
|
76
76
|
def test_format_addprefixtest
|
77
|
-
d = create_driver(DUMMYCONFIG + "add_tag_prefix addprefixtest")
|
77
|
+
d = create_driver(DUMMYCONFIG + "add_tag_prefix addprefixtest.")
|
78
78
|
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
79
79
|
d.emit({"a"=>1}, time)
|
80
80
|
d.emit({"a"=>2}, time)
|
metadata
CHANGED
@@ -1,18 +1,20 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-forward-aws
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.7
|
5
|
+
prerelease:
|
5
6
|
platform: ruby
|
6
7
|
authors:
|
7
8
|
- Tomohisa Ota
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date: 2013-03-
|
12
|
+
date: 2013-03-06 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
14
15
|
name: fluentd
|
15
16
|
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
16
18
|
requirements:
|
17
19
|
- - ! '>='
|
18
20
|
- !ruby/object:Gem::Version
|
@@ -20,6 +22,7 @@ dependencies:
|
|
20
22
|
type: :runtime
|
21
23
|
prerelease: false
|
22
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
23
26
|
requirements:
|
24
27
|
- - ! '>='
|
25
28
|
- !ruby/object:Gem::Version
|
@@ -27,6 +30,7 @@ dependencies:
|
|
27
30
|
- !ruby/object:Gem::Dependency
|
28
31
|
name: aws-sdk
|
29
32
|
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
30
34
|
requirements:
|
31
35
|
- - ~>
|
32
36
|
- !ruby/object:Gem::Version
|
@@ -34,6 +38,7 @@ dependencies:
|
|
34
38
|
type: :runtime
|
35
39
|
prerelease: false
|
36
40
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
37
42
|
requirements:
|
38
43
|
- - ~>
|
39
44
|
- !ruby/object:Gem::Version
|
@@ -51,40 +56,38 @@ files:
|
|
51
56
|
- README.md
|
52
57
|
- Rakefile
|
53
58
|
- fluent-plugin-forward-aws.gemspec
|
54
|
-
- lib/fluent/plugin/forward_aws_util.rb
|
55
59
|
- lib/fluent/plugin/in_forward_aws.rb
|
56
60
|
- lib/fluent/plugin/out_forward_aws.rb
|
57
61
|
- test/awsconfig.yml.sample
|
58
62
|
- test/helper.rb
|
59
|
-
- test/plugin/test_forward_aws_util.rb
|
60
63
|
- test/plugin/test_in_foward_aws.rb
|
61
64
|
- test/plugin/test_out_foward_aws.rb
|
62
65
|
homepage: http://github.com/tomohisaota/fluent-plugin-forward-aws
|
63
66
|
licenses: []
|
64
|
-
metadata: {}
|
65
67
|
post_install_message:
|
66
68
|
rdoc_options: []
|
67
69
|
require_paths:
|
68
70
|
- lib
|
69
71
|
required_ruby_version: !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
70
73
|
requirements:
|
71
74
|
- - ! '>='
|
72
75
|
- !ruby/object:Gem::Version
|
73
76
|
version: '0'
|
74
77
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
78
|
+
none: false
|
75
79
|
requirements:
|
76
80
|
- - ! '>='
|
77
81
|
- !ruby/object:Gem::Version
|
78
82
|
version: '0'
|
79
83
|
requirements: []
|
80
84
|
rubyforge_project:
|
81
|
-
rubygems_version:
|
85
|
+
rubygems_version: 1.8.25
|
82
86
|
signing_key:
|
83
|
-
specification_version:
|
87
|
+
specification_version: 3
|
84
88
|
summary: Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)
|
85
89
|
test_files:
|
86
90
|
- test/awsconfig.yml.sample
|
87
91
|
- test/helper.rb
|
88
|
-
- test/plugin/test_forward_aws_util.rb
|
89
92
|
- test/plugin/test_in_foward_aws.rb
|
90
93
|
- test/plugin/test_out_foward_aws.rb
|
checksums.yaml
DELETED
@@ -1,7 +0,0 @@
|
|
1
|
-
---
|
2
|
-
SHA1:
|
3
|
-
metadata.gz: 6a407c6eeec19a09a1eb1536c8dbb7f21fe5ac7e
|
4
|
-
data.tar.gz: 90449701adb6eb32977b87e2c0c6beba11480ec9
|
5
|
-
SHA512:
|
6
|
-
metadata.gz: c9874949fc9d9969a16910afc606eb3beb7c6373412eec2138c3fdbd31907054ee36608bc1716b7e554e377c052927ec9a650d00144c4de978ee02d2a934913b
|
7
|
-
data.tar.gz: d91d3db1dc5d8e4c2001f3ea63ebf7985ca2e8789649d90f62e9365642f740b6716b67879920be17de0def6171c2af739e4a416753d8f7f446d8463718f79b56
|
@@ -1,22 +0,0 @@
|
|
1
|
-
module ForwardAWSUtil
|
2
|
-
|
3
|
-
def self.filtertag(tag,add_prefix=nil,remove_prefix=nil)
|
4
|
-
# Remove Prefix First
|
5
|
-
if remove_prefix && tag
|
6
|
-
tag = tag.sub(/^#{Regexp.escape(remove_prefix)}\b\.?/,"")
|
7
|
-
end
|
8
|
-
|
9
|
-
# Add Prefix
|
10
|
-
if add_prefix
|
11
|
-
if tag && tag.length > 0
|
12
|
-
tag = "#{add_prefix}.#{tag}"
|
13
|
-
else
|
14
|
-
tag = add_prefix
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
# Return Result
|
19
|
-
return tag
|
20
|
-
end
|
21
|
-
|
22
|
-
end
|
@@ -1,35 +0,0 @@
|
|
1
|
-
require 'helper'
|
2
|
-
|
3
|
-
class ForwardAWSUtilTest < Test::Unit::TestCase
|
4
|
-
require_relative "../../lib/fluent/plugin/forward_aws_util"
|
5
|
-
include ForwardAWSUtil
|
6
|
-
|
7
|
-
def test_filtertag
|
8
|
-
assert_equal('aaa',ForwardAWSUtil.filtertag("aaa"))
|
9
|
-
assert_equal('aaa.bbb',ForwardAWSUtil.filtertag("aaa.bbb"))
|
10
|
-
assert_equal('aaa.bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc"))
|
11
|
-
|
12
|
-
# Add prefix
|
13
|
-
assert_equal('ddd.aaa',ForwardAWSUtil.filtertag("aaa","ddd"))
|
14
|
-
assert_equal('ddd.aaa.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ddd"))
|
15
|
-
assert_equal('ddd.aaa.bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc","ddd"))
|
16
|
-
|
17
|
-
# Remove prefix
|
18
|
-
assert_equal('',ForwardAWSUtil.filtertag("aaa",nil,"aaa"))
|
19
|
-
assert_equal('bbb',ForwardAWSUtil.filtertag("aaa.bbb",nil,"aaa"))
|
20
|
-
assert_equal('bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc",nil,"aaa"))
|
21
|
-
|
22
|
-
# Add and remove
|
23
|
-
assert_equal('ccc.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ccc","aaa"))
|
24
|
-
|
25
|
-
# Corner cases
|
26
|
-
# Do not remove tag in middle
|
27
|
-
assert_equal('aaa.bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc",nil,"bbb"))
|
28
|
-
# Do not remove incomplete tag
|
29
|
-
assert_equal('aaaa.bbb',ForwardAWSUtil.filtertag("aaaa.bbb",nil,"aaa"))
|
30
|
-
# Remove tag does not affect newly added tag
|
31
|
-
assert_equal('ccc.ddd.aaa.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ccc.ddd","ccc"))
|
32
|
-
assert_equal('ccc.ddd.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ccc.ddd","aaa"))
|
33
|
-
|
34
|
-
end
|
35
|
-
end
|