fluent-plugin-forward-aws 0.1.6 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md CHANGED
@@ -52,10 +52,17 @@ aws_secret_access_key | string (required) | AWS Secket Access Key
52
52
  aws_s3_endpoint | string (required) | [s3 Endpoint](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) for your bucket
53
53
  aws_s3_bucketname | string (required) | S3 Bucketname
54
54
  aws_s3_skiptest | bool (default = false) | Skip S3 Related test at startup
55
- add_tag_prefix | string (default = nil) | Add specified prefix to tag before processing log
56
- remove_tag_prefix | string (default = nil) | Remove specified prefix from tag before processing log
57
55
  channel | string (default = "default") | Tag that Forward-AWS plugin uses for grouping logs.
58
56
 
57
+ ### Parameters by supported by HandleTagNameMixin
58
+ name | type
59
+ ----------------------|---------------------
60
+ remove_tag_prefix | string (default = nil)
61
+ add_tag_prefix | string (default = nil)
62
+ remove_tag_suffix | string (default = nil)
63
+ add_tag_suffix | string (default = nil)
64
+
65
+
59
66
  ## Out Plugin Configuration
60
67
  ### Parameters
61
68
  name | type | description
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |gem|
6
6
  gem.name = "fluent-plugin-forward-aws"
7
- gem.version = "0.1.6"
7
+ gem.version = "0.1.7"
8
8
  gem.authors = ["Tomohisa Ota"]
9
9
  gem.email = ["tomohisa.ota+github@gmail.com"]
10
10
  gem.description = "Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)"
@@ -1,9 +1,20 @@
1
1
  class Fluent::ForwardAWSInput < Fluent::Input
2
2
  Fluent::Plugin.register_input('forward_aws', self)
3
3
 
4
- require_relative "forward_aws_util"
5
- include ForwardAWSUtil
6
-
4
+ include Fluent::HandleTagNameMixin
5
+
6
+ # Workaround for HandleTagNameMixin bug
7
+ # https://github.com/fluent/fluentd/pull/109
8
+ def format_stream(tag, es)
9
+ out = ''
10
+ es.each {|time,record|
11
+ tag_temp = String.new(tag)
12
+ filter_record(tag_temp, time, record)
13
+ out << format(tag_temp, time, record)
14
+ }
15
+ out
16
+ end
17
+
7
18
  def initialize
8
19
  super
9
20
  require 'aws-sdk'
@@ -29,10 +40,7 @@ class Fluent::ForwardAWSInput < Fluent::Input
29
40
  config_param :aws_sqs_wait_time_seconds, :integer, :default => 5
30
41
  config_param :aws_sqs_limit, :integer, :default => 10
31
42
  config_param :aws_sqs_visibilitiy_timeout,:integer, :default => 300
32
-
33
- config_param :add_tag_prefix, :string, :default => nil
34
- config_param :remove_tag_prefix, :string, :default => nil
35
-
43
+
36
44
  config_param :dry_run, :bool, :default => false
37
45
 
38
46
  # Not documented parameters. Subject to change in future release
@@ -181,7 +189,8 @@ class Fluent::ForwardAWSInput < Fluent::Input
181
189
  streamUnpacker.feed(reader.read())
182
190
  streamUnpacker.each {|event|
183
191
  (tag, time, record) = event
184
- tag = ForwardAWSUtil.filtertag(tag,@add_tag_prefix,@remove_tag_prefix)
192
+ # Apply HandleTagNameMixin manually
193
+ filter_record(tag, time, record)
185
194
  Fluent::Engine.emit(tag,time,record)
186
195
  }
187
196
  }
@@ -1,9 +1,20 @@
1
1
  class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
2
2
  Fluent::Plugin.register_output('forward_aws', self)
3
3
 
4
- require_relative "forward_aws_util"
5
- include ForwardAWSUtil
4
+ include Fluent::HandleTagNameMixin
6
5
 
6
+ # Workaround for HandleTagNameMixin bug
7
+ # https://github.com/fluent/fluentd/pull/109
8
+ def format_stream(tag, es)
9
+ out = ''
10
+ es.each {|time,record|
11
+ tag_temp = String.new(tag)
12
+ filter_record(tag_temp, time, record)
13
+ out << format(tag_temp, time, record)
14
+ }
15
+ out
16
+ end
17
+
7
18
  config_param :channel, :string, :default => "default"
8
19
 
9
20
  config_param :aws_access_key_id, :string, :default => nil
@@ -17,9 +28,6 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
17
28
  config_param :aws_sns_topic_arn, :string, :default => nil
18
29
  config_param :aws_sns_skiptest, :bool, :default => false
19
30
 
20
- config_param :add_tag_prefix, :string, :default => nil
21
- config_param :remove_tag_prefix, :string, :default => nil
22
-
23
31
  # Not documented parameters. Subject to change in future release
24
32
  config_param :aws_s3_testobjectname, :string, :default => "Config Check Test Object"
25
33
  config_param :aws_sns_emailsubject, :string, :default => "SNS Message"
@@ -90,7 +98,6 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
90
98
  end
91
99
 
92
100
  def format(tag, time, record)
93
- tag = ForwardAWSUtil.filtertag(tag,@add_tag_prefix,@remove_tag_prefix)
94
101
  [tag, time, record].to_msgpack
95
102
  end
96
103
 
@@ -126,7 +133,7 @@ class Fluent::ForwardAWSOutput < Fluent::TimeSlicedOutput
126
133
  tmp.close(true) rescue nil
127
134
  end
128
135
  end
129
-
136
+
130
137
  private
131
138
 
132
139
  def init_aws_s3_bucket
@@ -74,7 +74,7 @@ class ForwardAWSOutputTest < Test::Unit::TestCase
74
74
  end
75
75
 
76
76
  def test_format_addprefixtest
77
- d = create_driver(DUMMYCONFIG + "add_tag_prefix addprefixtest")
77
+ d = create_driver(DUMMYCONFIG + "add_tag_prefix addprefixtest.")
78
78
  time = Time.parse("2011-01-02 13:14:15 UTC").to_i
79
79
  d.emit({"a"=>1}, time)
80
80
  d.emit({"a"=>2}, time)
metadata CHANGED
@@ -1,18 +1,20 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-forward-aws
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.6
4
+ version: 0.1.7
5
+ prerelease:
5
6
  platform: ruby
6
7
  authors:
7
8
  - Tomohisa Ota
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2013-03-05 00:00:00.000000000 Z
12
+ date: 2013-03-06 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
14
15
  name: fluentd
15
16
  requirement: !ruby/object:Gem::Requirement
17
+ none: false
16
18
  requirements:
17
19
  - - ! '>='
18
20
  - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
20
22
  type: :runtime
21
23
  prerelease: false
22
24
  version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
23
26
  requirements:
24
27
  - - ! '>='
25
28
  - !ruby/object:Gem::Version
@@ -27,6 +30,7 @@ dependencies:
27
30
  - !ruby/object:Gem::Dependency
28
31
  name: aws-sdk
29
32
  requirement: !ruby/object:Gem::Requirement
33
+ none: false
30
34
  requirements:
31
35
  - - ~>
32
36
  - !ruby/object:Gem::Version
@@ -34,6 +38,7 @@ dependencies:
34
38
  type: :runtime
35
39
  prerelease: false
36
40
  version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
37
42
  requirements:
38
43
  - - ~>
39
44
  - !ruby/object:Gem::Version
@@ -51,40 +56,38 @@ files:
51
56
  - README.md
52
57
  - Rakefile
53
58
  - fluent-plugin-forward-aws.gemspec
54
- - lib/fluent/plugin/forward_aws_util.rb
55
59
  - lib/fluent/plugin/in_forward_aws.rb
56
60
  - lib/fluent/plugin/out_forward_aws.rb
57
61
  - test/awsconfig.yml.sample
58
62
  - test/helper.rb
59
- - test/plugin/test_forward_aws_util.rb
60
63
  - test/plugin/test_in_foward_aws.rb
61
64
  - test/plugin/test_out_foward_aws.rb
62
65
  homepage: http://github.com/tomohisaota/fluent-plugin-forward-aws
63
66
  licenses: []
64
- metadata: {}
65
67
  post_install_message:
66
68
  rdoc_options: []
67
69
  require_paths:
68
70
  - lib
69
71
  required_ruby_version: !ruby/object:Gem::Requirement
72
+ none: false
70
73
  requirements:
71
74
  - - ! '>='
72
75
  - !ruby/object:Gem::Version
73
76
  version: '0'
74
77
  required_rubygems_version: !ruby/object:Gem::Requirement
78
+ none: false
75
79
  requirements:
76
80
  - - ! '>='
77
81
  - !ruby/object:Gem::Version
78
82
  version: '0'
79
83
  requirements: []
80
84
  rubyforge_project:
81
- rubygems_version: 2.0.0
85
+ rubygems_version: 1.8.25
82
86
  signing_key:
83
- specification_version: 4
87
+ specification_version: 3
84
88
  summary: Fluentd In/Out plugin to forward log through AWS(S3/SNS/SQS)
85
89
  test_files:
86
90
  - test/awsconfig.yml.sample
87
91
  - test/helper.rb
88
- - test/plugin/test_forward_aws_util.rb
89
92
  - test/plugin/test_in_foward_aws.rb
90
93
  - test/plugin/test_out_foward_aws.rb
checksums.yaml DELETED
@@ -1,7 +0,0 @@
1
- ---
2
- SHA1:
3
- metadata.gz: 6a407c6eeec19a09a1eb1536c8dbb7f21fe5ac7e
4
- data.tar.gz: 90449701adb6eb32977b87e2c0c6beba11480ec9
5
- SHA512:
6
- metadata.gz: c9874949fc9d9969a16910afc606eb3beb7c6373412eec2138c3fdbd31907054ee36608bc1716b7e554e377c052927ec9a650d00144c4de978ee02d2a934913b
7
- data.tar.gz: d91d3db1dc5d8e4c2001f3ea63ebf7985ca2e8789649d90f62e9365642f740b6716b67879920be17de0def6171c2af739e4a416753d8f7f446d8463718f79b56
@@ -1,22 +0,0 @@
1
- module ForwardAWSUtil
2
-
3
- def self.filtertag(tag,add_prefix=nil,remove_prefix=nil)
4
- # Remove Prefix First
5
- if remove_prefix && tag
6
- tag = tag.sub(/^#{Regexp.escape(remove_prefix)}\b\.?/,"")
7
- end
8
-
9
- # Add Prefix
10
- if add_prefix
11
- if tag && tag.length > 0
12
- tag = "#{add_prefix}.#{tag}"
13
- else
14
- tag = add_prefix
15
- end
16
- end
17
-
18
- # Return Result
19
- return tag
20
- end
21
-
22
- end
@@ -1,35 +0,0 @@
1
- require 'helper'
2
-
3
- class ForwardAWSUtilTest < Test::Unit::TestCase
4
- require_relative "../../lib/fluent/plugin/forward_aws_util"
5
- include ForwardAWSUtil
6
-
7
- def test_filtertag
8
- assert_equal('aaa',ForwardAWSUtil.filtertag("aaa"))
9
- assert_equal('aaa.bbb',ForwardAWSUtil.filtertag("aaa.bbb"))
10
- assert_equal('aaa.bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc"))
11
-
12
- # Add prefix
13
- assert_equal('ddd.aaa',ForwardAWSUtil.filtertag("aaa","ddd"))
14
- assert_equal('ddd.aaa.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ddd"))
15
- assert_equal('ddd.aaa.bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc","ddd"))
16
-
17
- # Remove prefix
18
- assert_equal('',ForwardAWSUtil.filtertag("aaa",nil,"aaa"))
19
- assert_equal('bbb',ForwardAWSUtil.filtertag("aaa.bbb",nil,"aaa"))
20
- assert_equal('bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc",nil,"aaa"))
21
-
22
- # Add and remove
23
- assert_equal('ccc.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ccc","aaa"))
24
-
25
- # Corner cases
26
- # Do not remove tag in middle
27
- assert_equal('aaa.bbb.ccc',ForwardAWSUtil.filtertag("aaa.bbb.ccc",nil,"bbb"))
28
- # Do not remove incomplete tag
29
- assert_equal('aaaa.bbb',ForwardAWSUtil.filtertag("aaaa.bbb",nil,"aaa"))
30
- # Remove tag does not affect newly added tag
31
- assert_equal('ccc.ddd.aaa.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ccc.ddd","ccc"))
32
- assert_equal('ccc.ddd.bbb',ForwardAWSUtil.filtertag("aaa.bbb","ccc.ddd","aaa"))
33
-
34
- end
35
- end