fluent-plugin-grassland 0.0.3 → 0.0.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (4) hide show
  1. checksums.yaml +4 -4
  2. data/fluent-plugin-grassland.gemspec +3 -2
  3. data/lib/fluent/plugin/out_grassland.rb +50 -15
  4. metadata +18 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9b012fd760e3516bfda38ae05c4023cc847511af
4
- data.tar.gz: 10cf5a7ab421364b7881a289f4a568c7df8c3b07
3
+ metadata.gz: 1e17d6eedf5c9571d2634e94ddb5634dd4c2a2d0
4
+ data.tar.gz: c1226059029286dc67b98560ed39798f9977c1a1
5
5
  SHA512:
6
- metadata.gz: f06c72fa10e193da42ad7431a92d202a96305b9ca95ae0f80d804440a1cd7b2c53b449ea3c971005712fcc120880fe8544ac3096ac6fdf4c4fd81098ed7c0cdb
7
- data.tar.gz: ec306504bfc139a5df78a61efe7061e942e102b579e2c0affe560ec9d93f9a29657cd08518f0f7c0e4978503f62d37d00f07f70acb3a8f2ebf7dd797ebf144e6
6
+ metadata.gz: 0126eab89651d364191dd563d88e8217657b7564dea3cdebccea0b34ec9307bdf1fa9559a49f6d13daa72f17a0c2d3ebc6729dd2954c71c0aa6f301a5331dc77
7
+ data.tar.gz: 9b4fcaadcb516068817c6fbfa5187b42c35af1d3d0bfe54e75d372a84550a7e2d0765a7440c41ba6acfa48bba4cc932908d0d329419de0cc6990e7a2922bfa96
data/fluent-plugin-grassland.gemspec CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = 'fluent-plugin-grassland'
7
- spec.version = '0.0.3'
7
+ spec.version = '0.0.4'
8
8
  spec.authors = ['Ripplation Inc.']
9
9
  # spec.email = ['xxxxxx@ripplation.co.jp']
10
10
  spec.description = 'Output filter plugin for Grassland'
@@ -17,7 +17,8 @@ Gem::Specification.new do |spec|
17
17
  spec.require_paths = ['lib']
18
18
 
19
19
  spec.add_dependency 'fluentd'
20
- spec.add_dependency 'aws-sdk', '~> 1.38'
20
+ spec.add_dependency 'eventmachine', '~> 1.0.3'
21
+ spec.add_dependency 'aws-sdk', '~> 1.40.3'
21
22
  spec.add_dependency 'json'
22
23
  # spec.add_development_dependency 'bundler', '~> 1.3'
23
24
  # spec.add_development_dependency 'rake'
data/lib/fluent/plugin/out_grassland.rb CHANGED
@@ -2,7 +2,9 @@ module Fluent
2
2
  class GrasslandOutput < Fluent::BufferedOutput
3
3
  Fluent::Plugin.register_output('grassland', self)
4
4
 
5
- attr_accessor :stream_name, :access_key_id, :secret_access_key, :region
5
+ attr_accessor :random
6
+ attr_accessor :kinesis
7
+ attr_accessor :stream_name, :access_key_id, :secret_access_key, :region, :sessionToken, :partitionKeys
6
8
 
7
9
  def initialize
8
10
  super
@@ -12,11 +14,24 @@ module Fluent
12
14
  require 'logger'
13
15
  require 'net/http'
14
16
  require 'uri'
17
+ @random = Random.new
15
18
  end
16
19
 
17
- config_param :apiuri, :string, :default => 'https://grassland-api.elasticbeanstalk.com/credential'
18
- config_param :key, :string, :default => 'nil'
19
- config_param :debug, :bool, :default => false
20
+ config_param :apiuri, :string, :default => 'https://grassland.biz/credentials'
21
+ config_param :id, :string, :default => 'nil'
22
+ config_param :key, :string, :default => 'nil'
23
+ config_param :debug, :bool, :default => false
24
+ config_param :resetCredentialTimer, :integer, :default => 86400
25
+ # config_param :resetCredentialTimer, :integer, :default => 20
26
+
27
+ def set_interval(delay)
28
+ Thread.new do
29
+ loop do
30
+ sleep delay
31
+ yield # call passed block
32
+ end
33
+ end
34
+ end
20
35
 
21
36
  def configure(conf)
22
37
  super
@@ -30,25 +45,39 @@ module Fluent
30
45
 
31
46
  def start
32
47
  super
33
- setCredential
34
- configure_aws
35
- AWS.kinesis.client.put_record({
36
- :stream_name => @stream_name,
37
- :data => "test",
38
- :partition_key => "#{rand(999)}"
39
- })
48
+ set_interval(@resetCredentialTimer){
49
+ resetAwsCredential
50
+ }
51
+ resetAwsCredential
40
52
  end
41
53
 
42
54
  def shutdown
43
55
  super
44
56
  end
45
57
 
58
+ def resetAwsCredential()
59
+ begin
60
+ setCredential
61
+ configure_aws
62
+ AWS.kinesis.client.put_record({
63
+ :stream_name => @stream_name,
64
+ :data => "test",
65
+ :partition_key => "#{random.rand(999)}"
66
+ })
67
+ puts "fluentd: reset credential"
68
+ rescue => e
69
+ puts [e.class, e].join(" : initialize error.")
70
+ end
71
+ end
72
+
46
73
  def setCredential()
47
74
  credential = get_json("#{@apiuri}?key=#{@key}")
48
75
  @stream_name = credential['streamName']
49
76
  @access_key_id = credential['accessKeyId']
50
77
  @secret_access_key = credential['secretAccessKey']
51
78
  @region = credential['region']
79
+ @sessionToken = credential['SessionToken']
80
+ @partitionKeys = credential['SessionToken']
52
81
  end
53
82
 
54
83
  def get_json(location, limit = 3)
@@ -80,7 +109,7 @@ module Fluent
80
109
 
81
110
  def format(tag, time, record)
82
111
  # print(record)
83
- ['cid', 'dt', 'uid', 'd'].each do |key|
112
+ ['dt', 'uid', 'd'].each do |key|
84
113
  unless record.has_key?(key)
85
114
  puts "input data error: '#{key}' is required"
86
115
  return ""
@@ -89,6 +118,9 @@ module Fluent
89
118
  unless record.has_key?('pt')
90
119
  record['pt'] = time
91
120
  end
121
+ unless record.has_key?('cid')
122
+ record['cid'] = @id
123
+ end
92
124
 
93
125
  record['pk'] = record['cid'] + record['dt']
94
126
  return "#{record.to_json},"
@@ -111,7 +143,8 @@ module Fluent
111
143
  AWS.kinesis.client.put_record({
112
144
  :stream_name => @stream_name,
113
145
  :data => "["+bufList[":#{data['pk']}"].chop+"]",
114
- :partition_key => data['pk']
146
+ :partition_key => partitionKeys[random.rand(partitionKeys.length)]
147
+ # :partition_key => data['pk']
115
148
  })
116
149
  bufList.delete(":#{data['pk']}")
117
150
  end
@@ -121,7 +154,8 @@ module Fluent
121
154
  AWS.kinesis.client.put_record({
122
155
  :stream_name => @stream_name,
123
156
  :data => "["+bufList[":#{data['pk']}"].chop+"]",
124
- :partition_key => data['pk']
157
+ :partition_key => partitionKeys[random.rand(partitionKeys.length)]
158
+ # :partition_key => data['pk']
125
159
  })
126
160
  bufList.delete(":#{data['pk']}")
127
161
  end
@@ -137,7 +171,8 @@ module Fluent
137
171
  options = {
138
172
  :access_key_id => @access_key_id,
139
173
  :secret_access_key => @secret_access_key,
140
- :region => @region
174
+ :region => @region,
175
+ :session_token => @sessionToken
141
176
  }
142
177
 
143
178
  if @debug
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-grassland
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ripplation Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-04-23 00:00:00.000000000 Z
11
+ date: 2014-06-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -24,20 +24,34 @@ dependencies:
24
24
  - - '>='
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: eventmachine
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: 1.0.3
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ~>
39
+ - !ruby/object:Gem::Version
40
+ version: 1.0.3
27
41
  - !ruby/object:Gem::Dependency
28
42
  name: aws-sdk
29
43
  requirement: !ruby/object:Gem::Requirement
30
44
  requirements:
31
45
  - - ~>
32
46
  - !ruby/object:Gem::Version
33
- version: '1.38'
47
+ version: 1.40.3
34
48
  type: :runtime
35
49
  prerelease: false
36
50
  version_requirements: !ruby/object:Gem::Requirement
37
51
  requirements:
38
52
  - - ~>
39
53
  - !ruby/object:Gem::Version
40
- version: '1.38'
54
+ version: 1.40.3
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: json
43
57
  requirement: !ruby/object:Gem::Requirement