fluent-plugin-ec2-metadata 0.0.10 → 0.0.11

Sign up to get free protection for your applications and to get access to all the features.
Files changed (18) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.travis.yml +13 -6
  4. data/Gemfile +1 -1
  5. data/Gemfile.fluentd.0.10 +9 -0
  6. data/Gemfile.fluentd.0.12 +9 -0
  7. data/README.md +8 -0
  8. data/Rakefile +10 -2
  9. data/fluent-plugin-ec2-metadata.gemspec +5 -4
  10. data/lib/fluent/plugin/ec2_metadata.rb +26 -17
  11. data/lib/fluent/plugin/filter_ec2_metadata.rb +1 -0
  12. data/lib/fluent/plugin/out_ec2_metadata.rb +1 -0
  13. data/test/cassettes/ec2-classic.yml +39 -0
  14. data/test/cassettes/ec2-vpc.yml +39 -0
  15. data/test/helper.rb +0 -3
  16. data/test/plugin/test_filter_ec2_metadata.rb +1 -0
  17. data/test/plugin/test_out_ec2_metadata.rb +2 -0
  18. metadata +21 -19
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: dbc6f8c80499b4f5d1035976e73dbed3ab175983
4
- data.tar.gz: dfc17e48483cc59aa02039c5355debc7a6ac4cc3
3
+ metadata.gz: 0a17fec18c90bd8bcc707ece5b24581641daa7ca
4
+ data.tar.gz: 1e2fe6ea713bb57924f91cc76e957b6c6e8be4a7
5
5
  SHA512:
6
- metadata.gz: 95e7f3548ba2b4d8e0dea2ddd86020dac3f7cfcccdbd0454ed874520a79140f0fb85bb9d821e575036e8ca67f9a9cca7f1665e36a5dd8506b5503fcfb11029c8
7
- data.tar.gz: 27470dc7107882b981585d3ffd4ea5277e2463af28b713f8cee6390886222e4908ba83b7fbad0bb46f84acfbe754b644f69f706dac7a1cfafe5a6c824982c4fc
6
+ metadata.gz: 5c04b42547a32d00cfaca0e83a719b4adfaf9ad83ce2ad4ccffbcc3c926c70fe22e5f4fa40f9ea3cc92f21eb879697eed1a9d030168cd3002a98d152cf49a5ac
7
+ data.tar.gz: 4c2b7efe0e5d18f531ab515e89308c4718f248831b37572eb1a1efb29f397f7927c160f4e8cf0071832b12061f8f3cf2209646c7567e61abb185783384d6e8f0
data/.gitignore CHANGED
@@ -3,6 +3,7 @@
3
3
  .bundle
4
4
  .config
5
5
  .yardoc
6
+ .idea
6
7
  Gemfile.lock
7
8
  InstalledFiles
8
9
  _yardoc
data/.travis.yml CHANGED
@@ -1,7 +1,14 @@
1
- sudo: false
2
- language: ruby
3
1
  rvm:
4
- - 2.2
5
- - 2.1
6
- - 2.0.0
7
- - 1.9.3
2
+ - 2.0.*
3
+ - 2.1.*
4
+ - 2.2.*
5
+ - 2.3.0
6
+ gemfile:
7
+ - Gemfile
8
+ - Gemfile.fluentd.0.12
9
+ - Gemfile.fluentd.0.10
10
+ matrix:
11
+ exclude:
12
+ - rvm: 2.0.*
13
+ gemfile: Gemfile
14
+ before_install: gem update bundler
data/Gemfile CHANGED
@@ -3,5 +3,5 @@ source 'https://rubygems.org'
3
3
  gemspec
4
4
 
5
5
  group :test do
6
- gem "codeclimate-test-reporter", require: nil
6
+ gem 'codeclimate-test-reporter', require: nil
7
7
  end
data/Gemfile.fluentd.0.10 ADDED
@@ -0,0 +1,9 @@
1
+ source 'http://rubygems.org'
2
+
3
+ gem 'fluentd', '~> 0.10.43'
4
+
5
+ gemspec
6
+
7
+ group :test do
8
+ gem 'codeclimate-test-reporter', require: nil
9
+ end
data/Gemfile.fluentd.0.12 ADDED
@@ -0,0 +1,9 @@
1
+ source 'http://rubygems.org'
2
+
3
+ gem 'fluentd', '~> 0.12.0'
4
+
5
+ gemspec
6
+
7
+ group :test do
8
+ gem 'codeclimate-test-reporter', require: nil
9
+ end
data/README.md CHANGED
@@ -24,12 +24,15 @@ Example:
24
24
  aws_key_id YOUR_AWS_KEY_ID
25
25
  aws_sec_key YOUR_AWS_SECRET/KEY
26
26
 
27
+ metadata_refresh_seconds 300 # Optional, default 300 seconds
28
+
27
29
  output_tag ${instance_id}.${tag}
28
30
  <record>
29
31
  hostname ${tagset_name}
30
32
  instance_id ${instance_id}
31
33
  instance_type ${instance_type}
32
34
  az ${availability_zone}
35
+ private_ip ${private_ip}
33
36
  vpc_id ${vpc_id}
34
37
  ami_id ${image_id}
35
38
  account_id ${account_id}
@@ -50,6 +53,7 @@ i-28b5ee77.foo.bar {
50
53
  "instance_id" : "i-28b5ee77",
51
54
  "instance_type" : "m1.large",
52
55
  "az" : "us-west-1b",
56
+ "private_ip : "10.21.34.200",
53
57
  "vpc_id" : "vpc-25dab194",
54
58
  "account_id" : "123456789",
55
59
  "image_id" : "ami-123456",
@@ -65,10 +69,13 @@ Or you can use filter version:
65
69
  aws_key_id YOUR_AWS_KEY_ID
66
70
  aws_sec_key YOUR_AWS_SECRET/KEY
67
71
 
72
+ metadata_refresh_seconds 300 # Optional, default 300 seconds
73
+
68
74
  <record>
69
75
  hostname ${tagset_name}
70
76
  instance_id ${instance_id}
71
77
  instance_type ${instance_type}
78
+ private_ip ${private_ip}
72
79
  az ${availability_zone}
73
80
  vpc_id ${vpc_id}
74
81
  ami_id ${image_id}
@@ -86,6 +93,7 @@ The following placeholders are always available:
86
93
  * ${instance_type} instance type
87
94
  * ${availability_zone} availability zone
88
95
  * ${region} region
96
+ * ${private_ip} private ip
89
97
  * ${mac} MAC address
90
98
  * ${vpc_id} vpc id
91
99
  * ${subnet_id} subnet id
data/Rakefile CHANGED
@@ -1,9 +1,17 @@
1
1
  require "bundler/gem_tasks"
2
+ require "fluent/version"
3
+ require "rake/testtask"
2
4
 
3
- require 'rake/testtask'
4
5
  Rake::TestTask.new(:test) do |test|
5
6
  test.libs << 'lib' << 'test'
6
- test.pattern = 'test/**/test_*.rb'
7
+
8
+ test.test_files = case
9
+ when Fluent::VERSION.start_with?("0.10.")
10
+ ["test/plugin/test_out_ec2_metadata.rb"]
11
+ else
12
+ Dir['test/**/test_*.rb']
13
+ end
14
+
7
15
  test.verbose = true
8
16
  end
9
17
 
data/fluent-plugin-ec2-metadata.gemspec CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-ec2-metadata"
7
- spec.version = "0.0.10"
7
+ spec.version = "0.0.11"
8
8
  spec.authors = ["SAKAMOTO Takumi"]
9
9
  spec.email = ["takumi.saka@gmail.com"]
10
10
  spec.description = %q{Fluentd output plugin to add ec2 metadata fields to a event record}
@@ -17,11 +17,12 @@ Gem::Specification.new do |spec|
17
17
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
18
  spec.require_paths = ["lib"]
19
19
 
20
+ spec.add_runtime_dependency "fluentd"
21
+ spec.add_runtime_dependency "oj"
22
+ spec.add_runtime_dependency "aws-sdk"
23
+
20
24
  spec.add_development_dependency "rake"
21
25
  spec.add_development_dependency "vcr"
22
26
  spec.add_development_dependency "webmock"
23
27
  spec.add_development_dependency "test-unit", ">= 3.1.0"
24
- spec.add_runtime_dependency "fluentd"
25
- spec.add_runtime_dependency "oj"
26
- spec.add_runtime_dependency "aws-sdk"
27
28
  end
data/lib/fluent/plugin/ec2_metadata.rb CHANGED
@@ -22,37 +22,46 @@ module Fluent
22
22
 
23
23
  @placeholder_expander = PlaceholderExpander.new
24
24
 
25
- set_metadata
25
+ # get metadata first and then setup a refresh thread
26
+ set_metadata
26
27
  set_tag
28
+ @refresh_thread = Thread.new {
29
+ while true
30
+ sleep @metadata_refresh_seconds
31
+ set_metadata
32
+ set_tag
33
+ end
34
+ }
27
35
  end
28
36
 
29
37
  private
30
38
 
31
39
  def set_metadata()
32
- @ec2_metadata = {}
33
-
40
+ ec2_metadata = {}
34
41
 
35
42
  instance_identity = Oj.load(get_dynamic_data("instance-identity/document"))
36
- @ec2_metadata['account_id'] = instance_identity["accountId"]
37
- @ec2_metadata['image_id'] = instance_identity["imageId"]
38
-
39
- @ec2_metadata['instance_id'] = get_metadata('instance-id')
40
- @ec2_metadata['instance_type'] = get_metadata('instance-type')
41
- @ec2_metadata['availability_zone'] = get_metadata('placement/availability-zone')
42
- @ec2_metadata['region'] = @ec2_metadata['availability_zone'].chop
43
- @ec2_metadata['mac'] = get_metadata('mac')
43
+ ec2_metadata['account_id'] = instance_identity["accountId"]
44
+ ec2_metadata['image_id'] = instance_identity["imageId"]
45
+
46
+ ec2_metadata['instance_id'] = get_metadata('instance-id')
47
+ ec2_metadata['instance_type'] = get_metadata('instance-type')
48
+ ec2_metadata['availability_zone'] = get_metadata('placement/availability-zone')
49
+ ec2_metadata['region'] = ec2_metadata['availability_zone'].chop
50
+ ec2_metadata['private_ip'] = get_metadata('local-ipv4')
51
+ ec2_metadata['mac'] = get_metadata('mac')
44
52
  begin
45
- @ec2_metadata['vpc_id'] = get_metadata("network/interfaces/macs/#{@ec2_metadata['mac']}/vpc-id")
53
+ ec2_metadata['vpc_id'] = get_metadata("network/interfaces/macs/#{ec2_metadata['mac']}/vpc-id")
46
54
  rescue
47
- @ec2_metadata['vpc_id'] = nil
48
- $log.info "ec2-metadata: 'vpc_id' is undefined #{@ec2_metadata['instance_id']} is not in VPC}"
55
+ ec2_metadata['vpc_id'] = nil
56
+ $log.info "ec2-metadata: 'vpc_id' is undefined #{ec2_metadata['instance_id']} is not in VPC}"
49
57
  end
50
58
  begin
51
- @ec2_metadata['subnet_id'] = get_metadata("network/interfaces/macs/#{@ec2_metadata['mac']}/subnet-id")
59
+ ec2_metadata['subnet_id'] = get_metadata("network/interfaces/macs/#{ec2_metadata['mac']}/subnet-id")
52
60
  rescue
53
- @ec2_metadata['subnet_id'] = nil
54
- $log.info "ec2-metadata: 'subnet_id' is undefined because #{@ec2_metadata['instance_id']} is not in VPC}"
61
+ ec2_metadata['subnet_id'] = nil
62
+ $log.info "ec2-metadata: 'subnet_id' is undefined because #{ec2_metadata['instance_id']} is not in VPC}"
55
63
  end
64
+ @ec2_metadata=ec2_metadata
56
65
  end
57
66
 
58
67
  def get_dynamic_data(f)
data/lib/fluent/plugin/filter_ec2_metadata.rb CHANGED
@@ -13,6 +13,7 @@ module Fluent
13
13
 
14
14
  config_param :aws_key_id, :string, :default => ENV['AWS_ACCESS_KEY_ID'], :secret => true
15
15
  config_param :aws_sec_key, :string, :default => ENV['AWS_SECRET_ACCESS_KEY'], :secret => true
16
+ config_param :metadata_refresh_seconds, :integer, :default => 300
16
17
 
17
18
  attr_reader :ec2_metadata
18
19
 
data/lib/fluent/plugin/out_ec2_metadata.rb CHANGED
@@ -14,6 +14,7 @@ module Fluent
14
14
  config_param :output_tag, :string
15
15
  config_param :aws_key_id, :string, :default => ENV['AWS_ACCESS_KEY_ID'], :secret => true
16
16
  config_param :aws_sec_key, :string, :default => ENV['AWS_SECRET_ACCESS_KEY'], :secret => true
17
+ config_param :metadata_refresh_seconds, :integer, :default => 300
17
18
 
18
19
  attr_reader :ec2_metadata
19
20
 
data/test/cassettes/ec2-classic.yml CHANGED
@@ -117,6 +117,45 @@ http_interactions:
117
117
  string: ap-northeast-1b
118
118
  http_version:
119
119
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
120
+ - request:
121
+ method: get
122
+ uri: http://169.254.169.254/latest/meta-data/local-ipv4
123
+ body:
124
+ encoding: US-ASCII
125
+ string: ''
126
+ headers:
127
+ Accept-Encoding:
128
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
129
+ Accept:
130
+ - '*/*'
131
+ User-Agent:
132
+ - Ruby
133
+ response:
134
+ status:
135
+ code: 200
136
+ message: OK
137
+ headers:
138
+ Content-Type:
139
+ - text/plain
140
+ Accept-Ranges:
141
+ - bytes
142
+ Etag:
143
+ - '"1870317889"'
144
+ Last-Modified:
145
+ - Thu, 02 Jul 2015 12:49:17 GMT
146
+ Content-Length:
147
+ - '15'
148
+ Connection:
149
+ - close
150
+ Date:
151
+ - Sun, 13 Dec 2015 06:37:00 GMT
152
+ Server:
153
+ - EC2ws
154
+ body:
155
+ encoding: UTF-8
156
+ string: 10.21.34.200
157
+ http_version:
158
+ recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
120
159
  - request:
121
160
  method: get
122
161
  uri: http://169.254.169.254/latest/meta-data/mac
data/test/cassettes/ec2-vpc.yml CHANGED
@@ -117,6 +117,45 @@ http_interactions:
117
117
  string: ap-northeast-1b
118
118
  http_version:
119
119
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
120
+ - request:
121
+ method: get
122
+ uri: http://169.254.169.254/latest/meta-data/local-ipv4
123
+ body:
124
+ encoding: US-ASCII
125
+ string: ''
126
+ headers:
127
+ Accept-Encoding:
128
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
129
+ Accept:
130
+ - '*/*'
131
+ User-Agent:
132
+ - Ruby
133
+ response:
134
+ status:
135
+ code: 200
136
+ message: OK
137
+ headers:
138
+ Content-Type:
139
+ - text/plain
140
+ Accept-Ranges:
141
+ - bytes
142
+ Etag:
143
+ - '"1870317889"'
144
+ Last-Modified:
145
+ - Thu, 02 Jul 2015 12:49:17 GMT
146
+ Content-Length:
147
+ - '15'
148
+ Connection:
149
+ - close
150
+ Date:
151
+ - Sun, 13 Dec 2015 06:37:00 GMT
152
+ Server:
153
+ - EC2ws
154
+ body:
155
+ encoding: UTF-8
156
+ string: 10.21.34.200
157
+ http_version:
158
+ recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
120
159
  - request:
121
160
  method: get
122
161
  uri: http://169.254.169.254/latest/meta-data/mac
data/test/helper.rb CHANGED
@@ -33,8 +33,5 @@ VCR.configure do |config|
33
33
  config.ignore_hosts 'codeclimate.com'
34
34
  end
35
35
 
36
- require 'fluent/plugin/out_ec2_metadata'
37
- require 'fluent/plugin/filter_ec2_metadata'
38
-
39
36
  class Test::Unit::TestCase
40
37
  end
data/test/plugin/test_filter_ec2_metadata.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require 'helper'
2
+ require 'fluent/plugin/filter_ec2_metadata'
2
3
 
3
4
  require 'webmock/test_unit'
4
5
  WebMock.disable_net_connect!
data/test/plugin/test_out_ec2_metadata.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require 'helper'
2
+ require 'fluent/plugin/out_ec2_metadata'
2
3
 
3
4
  require 'webmock/test_unit'
4
5
  WebMock.disable_net_connect!
@@ -43,6 +44,7 @@ class EC2MetadataOutputTest < Test::Unit::TestCase
43
44
 
44
45
  assert_equal("ami-123456", d.instance.ec2_metadata['image_id'])
45
46
  assert_equal("123456789", d.instance.ec2_metadata['account_id'])
47
+ assert_equal("10.21.34.200", d.instance.ec2_metadata['private_ip'])
46
48
 
47
49
  assert_equal("i-0c0c0000", d.instance.ec2_metadata['instance_id'])
48
50
  assert_equal("m3.large", d.instance.ec2_metadata['instance_type'])
metadata CHANGED
@@ -1,23 +1,23 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-ec2-metadata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.10
4
+ version: 0.0.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - SAKAMOTO Takumi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-05-17 00:00:00.000000000 Z
11
+ date: 2016-07-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: rake
14
+ name: fluentd
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
19
  version: '0'
20
- type: :development
20
+ type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
@@ -25,13 +25,13 @@ dependencies:
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
- name: vcr
28
+ name: oj
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
33
  version: '0'
34
- type: :development
34
+ type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
@@ -39,13 +39,13 @@ dependencies:
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
- name: webmock
42
+ name: aws-sdk
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0'
48
- type: :development
48
+ type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
@@ -53,27 +53,27 @@ dependencies:
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
- name: test-unit
56
+ name: rake
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - ">="
60
60
  - !ruby/object:Gem::Version
61
- version: 3.1.0
61
+ version: '0'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - ">="
67
67
  - !ruby/object:Gem::Version
68
- version: 3.1.0
68
+ version: '0'
69
69
  - !ruby/object:Gem::Dependency
70
- name: fluentd
70
+ name: vcr
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - ">="
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
- type: :runtime
76
+ type: :development
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
@@ -81,13 +81,13 @@ dependencies:
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
- name: oj
84
+ name: webmock
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
87
  - - ">="
88
88
  - !ruby/object:Gem::Version
89
89
  version: '0'
90
- type: :runtime
90
+ type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
@@ -95,19 +95,19 @@ dependencies:
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
97
  - !ruby/object:Gem::Dependency
98
- name: aws-sdk
98
+ name: test-unit
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
101
  - - ">="
102
102
  - !ruby/object:Gem::Version
103
- version: '0'
104
- type: :runtime
103
+ version: 3.1.0
104
+ type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - ">="
109
109
  - !ruby/object:Gem::Version
110
- version: '0'
110
+ version: 3.1.0
111
111
  description: Fluentd output plugin to add ec2 metadata fields to a event record
112
112
  email:
113
113
  - takumi.saka@gmail.com
@@ -118,6 +118,8 @@ files:
118
118
  - ".gitignore"
119
119
  - ".travis.yml"
120
120
  - Gemfile
121
+ - Gemfile.fluentd.0.10
122
+ - Gemfile.fluentd.0.12
121
123
  - LICENSE.txt
122
124
  - README.md
123
125
  - Rakefile