fluent-plugin-grassland 0.0.3 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. checksums.yaml +4 -4
  2. data/fluent-plugin-grassland.gemspec +3 -2
  3. data/lib/fluent/plugin/out_grassland.rb +50 -15
  4. metadata +18 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9b012fd760e3516bfda38ae05c4023cc847511af
4
- data.tar.gz: 10cf5a7ab421364b7881a289f4a568c7df8c3b07
3
+ metadata.gz: 1e17d6eedf5c9571d2634e94ddb5634dd4c2a2d0
4
+ data.tar.gz: c1226059029286dc67b98560ed39798f9977c1a1
5
5
  SHA512:
6
- metadata.gz: f06c72fa10e193da42ad7431a92d202a96305b9ca95ae0f80d804440a1cd7b2c53b449ea3c971005712fcc120880fe8544ac3096ac6fdf4c4fd81098ed7c0cdb
7
- data.tar.gz: ec306504bfc139a5df78a61efe7061e942e102b579e2c0affe560ec9d93f9a29657cd08518f0f7c0e4978503f62d37d00f07f70acb3a8f2ebf7dd797ebf144e6
6
+ metadata.gz: 0126eab89651d364191dd563d88e8217657b7564dea3cdebccea0b34ec9307bdf1fa9559a49f6d13daa72f17a0c2d3ebc6729dd2954c71c0aa6f301a5331dc77
7
+ data.tar.gz: 9b4fcaadcb516068817c6fbfa5187b42c35af1d3d0bfe54e75d372a84550a7e2d0765a7440c41ba6acfa48bba4cc932908d0d329419de0cc6990e7a2922bfa96
data/fluent-plugin-grassland.gemspec CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = 'fluent-plugin-grassland'
7
- spec.version = '0.0.3'
7
+ spec.version = '0.0.4'
8
8
  spec.authors = ['Ripplation Inc.']
9
9
  # spec.email = ['xxxxxx@ripplation.co.jp']
10
10
  spec.description = 'Output filter plugin for Grassland'
@@ -17,7 +17,8 @@ Gem::Specification.new do |spec|
17
17
  spec.require_paths = ['lib']
18
18
 
19
19
  spec.add_dependency 'fluentd'
20
- spec.add_dependency 'aws-sdk', '~> 1.38'
20
+ spec.add_dependency 'eventmachine', '~> 1.0.3'
21
+ spec.add_dependency 'aws-sdk', '~> 1.40.3'
21
22
  spec.add_dependency 'json'
22
23
  # spec.add_development_dependency 'bundler', '~> 1.3'
23
24
  # spec.add_development_dependency 'rake'
data/lib/fluent/plugin/out_grassland.rb CHANGED
@@ -2,7 +2,9 @@ module Fluent
2
2
  class GrasslandOutput < Fluent::BufferedOutput
3
3
  Fluent::Plugin.register_output('grassland', self)
4
4
 
5
- attr_accessor :stream_name, :access_key_id, :secret_access_key, :region
5
+ attr_accessor :random
6
+ attr_accessor :kinesis
7
+ attr_accessor :stream_name, :access_key_id, :secret_access_key, :region, :sessionToken, :partitionKeys
6
8
 
7
9
  def initialize
8
10
  super
@@ -12,11 +14,24 @@ module Fluent
12
14
  require 'logger'
13
15
  require 'net/http'
14
16
  require 'uri'
17
+ @random = Random.new
15
18
  end
16
19
 
17
- config_param :apiuri, :string, :default => 'https://grassland-api.elasticbeanstalk.com/credential'
18
- config_param :key, :string, :default => 'nil'
19
- config_param :debug, :bool, :default => false
20
+ config_param :apiuri, :string, :default => 'https://grassland.biz/credentials'
21
+ config_param :id, :string, :default => 'nil'
22
+ config_param :key, :string, :default => 'nil'
23
+ config_param :debug, :bool, :default => false
24
+ config_param :resetCredentialTimer, :integer, :default => 86400
25
+ # config_param :resetCredentialTimer, :integer, :default => 20
26
+
27
+ def set_interval(delay)
28
+ Thread.new do
29
+ loop do
30
+ sleep delay
31
+ yield # call passed block
32
+ end
33
+ end
34
+ end
20
35
 
21
36
  def configure(conf)
22
37
  super
@@ -30,25 +45,39 @@ module Fluent
30
45
 
31
46
  def start
32
47
  super
33
- setCredential
34
- configure_aws
35
- AWS.kinesis.client.put_record({
36
- :stream_name => @stream_name,
37
- :data => "test",
38
- :partition_key => "#{rand(999)}"
39
- })
48
+ set_interval(@resetCredentialTimer){
49
+ resetAwsCredential
50
+ }
51
+ resetAwsCredential
40
52
  end
41
53
 
42
54
  def shutdown
43
55
  super
44
56
  end
45
57
 
58
+ def resetAwsCredential()
59
+ begin
60
+ setCredential
61
+ configure_aws
62
+ AWS.kinesis.client.put_record({
63
+ :stream_name => @stream_name,
64
+ :data => "test",
65
+ :partition_key => "#{random.rand(999)}"
66
+ })
67
+ puts "fluentd: reset credential"
68
+ rescue => e
69
+ puts [e.class, e].join(" : initialize error.")
70
+ end
71
+ end
72
+
46
73
  def setCredential()
47
74
  credential = get_json("#{@apiuri}?key=#{@key}")
48
75
  @stream_name = credential['streamName']
49
76
  @access_key_id = credential['accessKeyId']
50
77
  @secret_access_key = credential['secretAccessKey']
51
78
  @region = credential['region']
79
+ @sessionToken = credential['SessionToken']
80
+ @partitionKeys = credential['SessionToken']
52
81
  end
53
82
 
54
83
  def get_json(location, limit = 3)
@@ -80,7 +109,7 @@ module Fluent
80
109
 
81
110
  def format(tag, time, record)
82
111
  # print(record)
83
- ['cid', 'dt', 'uid', 'd'].each do |key|
112
+ ['dt', 'uid', 'd'].each do |key|
84
113
  unless record.has_key?(key)
85
114
  puts "input data error: '#{key}' is required"
86
115
  return ""
@@ -89,6 +118,9 @@ module Fluent
89
118
  unless record.has_key?('pt')
90
119
  record['pt'] = time
91
120
  end
121
+ unless record.has_key?('cid')
122
+ record['cid'] = @id
123
+ end
92
124
 
93
125
  record['pk'] = record['cid'] + record['dt']
94
126
  return "#{record.to_json},"
@@ -111,7 +143,8 @@ module Fluent
111
143
  AWS.kinesis.client.put_record({
112
144
  :stream_name => @stream_name,
113
145
  :data => "["+bufList[":#{data['pk']}"].chop+"]",
114
- :partition_key => data['pk']
146
+ :partition_key => partitionKeys[random.rand(partitionKeys.length)]
147
+ # :partition_key => data['pk']
115
148
  })
116
149
  bufList.delete(":#{data['pk']}")
117
150
  end
@@ -121,7 +154,8 @@ module Fluent
121
154
  AWS.kinesis.client.put_record({
122
155
  :stream_name => @stream_name,
123
156
  :data => "["+bufList[":#{data['pk']}"].chop+"]",
124
- :partition_key => data['pk']
157
+ :partition_key => partitionKeys[random.rand(partitionKeys.length)]
158
+ # :partition_key => data['pk']
125
159
  })
126
160
  bufList.delete(":#{data['pk']}")
127
161
  end
@@ -137,7 +171,8 @@ module Fluent
137
171
  options = {
138
172
  :access_key_id => @access_key_id,
139
173
  :secret_access_key => @secret_access_key,
140
- :region => @region
174
+ :region => @region,
175
+ :session_token => @sessionToken
141
176
  }
142
177
 
143
178
  if @debug
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-grassland
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ripplation Inc.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-04-23 00:00:00.000000000 Z
11
+ date: 2014-06-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -24,20 +24,34 @@ dependencies:
24
24
  - - '>='
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: eventmachine
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: 1.0.3
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ~>
39
+ - !ruby/object:Gem::Version
40
+ version: 1.0.3
27
41
  - !ruby/object:Gem::Dependency
28
42
  name: aws-sdk
29
43
  requirement: !ruby/object:Gem::Requirement
30
44
  requirements:
31
45
  - - ~>
32
46
  - !ruby/object:Gem::Version
33
- version: '1.38'
47
+ version: 1.40.3
34
48
  type: :runtime
35
49
  prerelease: false
36
50
  version_requirements: !ruby/object:Gem::Requirement
37
51
  requirements:
38
52
  - - ~>
39
53
  - !ruby/object:Gem::Version
40
- version: '1.38'
54
+ version: 1.40.3
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: json
43
57
  requirement: !ruby/object:Gem::Requirement