fluent-plugin-ec2-metadata 0.0.10 → 0.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.travis.yml +13 -6
  4. data/Gemfile +1 -1
  5. data/Gemfile.fluentd.0.10 +9 -0
  6. data/Gemfile.fluentd.0.12 +9 -0
  7. data/README.md +8 -0
  8. data/Rakefile +10 -2
  9. data/fluent-plugin-ec2-metadata.gemspec +5 -4
  10. data/lib/fluent/plugin/ec2_metadata.rb +26 -17
  11. data/lib/fluent/plugin/filter_ec2_metadata.rb +1 -0
  12. data/lib/fluent/plugin/out_ec2_metadata.rb +1 -0
  13. data/test/cassettes/ec2-classic.yml +39 -0
  14. data/test/cassettes/ec2-vpc.yml +39 -0
  15. data/test/helper.rb +0 -3
  16. data/test/plugin/test_filter_ec2_metadata.rb +1 -0
  17. data/test/plugin/test_out_ec2_metadata.rb +2 -0
  18. metadata +21 -19
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: dbc6f8c80499b4f5d1035976e73dbed3ab175983
4
- data.tar.gz: dfc17e48483cc59aa02039c5355debc7a6ac4cc3
3
+ metadata.gz: 0a17fec18c90bd8bcc707ece5b24581641daa7ca
4
+ data.tar.gz: 1e2fe6ea713bb57924f91cc76e957b6c6e8be4a7
5
5
  SHA512:
6
- metadata.gz: 95e7f3548ba2b4d8e0dea2ddd86020dac3f7cfcccdbd0454ed874520a79140f0fb85bb9d821e575036e8ca67f9a9cca7f1665e36a5dd8506b5503fcfb11029c8
7
- data.tar.gz: 27470dc7107882b981585d3ffd4ea5277e2463af28b713f8cee6390886222e4908ba83b7fbad0bb46f84acfbe754b644f69f706dac7a1cfafe5a6c824982c4fc
6
+ metadata.gz: 5c04b42547a32d00cfaca0e83a719b4adfaf9ad83ce2ad4ccffbcc3c926c70fe22e5f4fa40f9ea3cc92f21eb879697eed1a9d030168cd3002a98d152cf49a5ac
7
+ data.tar.gz: 4c2b7efe0e5d18f531ab515e89308c4718f248831b37572eb1a1efb29f397f7927c160f4e8cf0071832b12061f8f3cf2209646c7567e61abb185783384d6e8f0
data/.gitignore CHANGED
@@ -3,6 +3,7 @@
3
3
  .bundle
4
4
  .config
5
5
  .yardoc
6
+ .idea
6
7
  Gemfile.lock
7
8
  InstalledFiles
8
9
  _yardoc
data/.travis.yml CHANGED
@@ -1,7 +1,14 @@
1
- sudo: false
2
- language: ruby
3
1
  rvm:
4
- - 2.2
5
- - 2.1
6
- - 2.0.0
7
- - 1.9.3
2
+ - 2.0.*
3
+ - 2.1.*
4
+ - 2.2.*
5
+ - 2.3.0
6
+ gemfile:
7
+ - Gemfile
8
+ - Gemfile.fluentd.0.12
9
+ - Gemfile.fluentd.0.10
10
+ matrix:
11
+ exclude:
12
+ - rvm: 2.0.*
13
+ gemfile: Gemfile
14
+ before_install: gem update bundler
data/Gemfile CHANGED
@@ -3,5 +3,5 @@ source 'https://rubygems.org'
3
3
  gemspec
4
4
 
5
5
  group :test do
6
- gem "codeclimate-test-reporter", require: nil
6
+ gem 'codeclimate-test-reporter', require: nil
7
7
  end
data/Gemfile.fluentd.0.10 ADDED
@@ -0,0 +1,9 @@
1
+ source 'http://rubygems.org'
2
+
3
+ gem 'fluentd', '~> 0.10.43'
4
+
5
+ gemspec
6
+
7
+ group :test do
8
+ gem 'codeclimate-test-reporter', require: nil
9
+ end
data/Gemfile.fluentd.0.12 ADDED
@@ -0,0 +1,9 @@
1
+ source 'http://rubygems.org'
2
+
3
+ gem 'fluentd', '~> 0.12.0'
4
+
5
+ gemspec
6
+
7
+ group :test do
8
+ gem 'codeclimate-test-reporter', require: nil
9
+ end
data/README.md CHANGED
@@ -24,12 +24,15 @@ Example:
24
24
  aws_key_id YOUR_AWS_KEY_ID
25
25
  aws_sec_key YOUR_AWS_SECRET/KEY
26
26
 
27
+ metadata_refresh_seconds 300 # Optional, default 300 seconds
28
+
27
29
  output_tag ${instance_id}.${tag}
28
30
  <record>
29
31
  hostname ${tagset_name}
30
32
  instance_id ${instance_id}
31
33
  instance_type ${instance_type}
32
34
  az ${availability_zone}
35
+ private_ip ${private_ip}
33
36
  vpc_id ${vpc_id}
34
37
  ami_id ${image_id}
35
38
  account_id ${account_id}
@@ -50,6 +53,7 @@ i-28b5ee77.foo.bar {
50
53
  "instance_id" : "i-28b5ee77",
51
54
  "instance_type" : "m1.large",
52
55
  "az" : "us-west-1b",
56
+ "private_ip : "10.21.34.200",
53
57
  "vpc_id" : "vpc-25dab194",
54
58
  "account_id" : "123456789",
55
59
  "image_id" : "ami-123456",
@@ -65,10 +69,13 @@ Or you can use filter version:
65
69
  aws_key_id YOUR_AWS_KEY_ID
66
70
  aws_sec_key YOUR_AWS_SECRET/KEY
67
71
 
72
+ metadata_refresh_seconds 300 # Optional, default 300 seconds
73
+
68
74
  <record>
69
75
  hostname ${tagset_name}
70
76
  instance_id ${instance_id}
71
77
  instance_type ${instance_type}
78
+ private_ip ${private_ip}
72
79
  az ${availability_zone}
73
80
  vpc_id ${vpc_id}
74
81
  ami_id ${image_id}
@@ -86,6 +93,7 @@ The following placeholders are always available:
86
93
  * ${instance_type} instance type
87
94
  * ${availability_zone} availability zone
88
95
  * ${region} region
96
+ * ${private_ip} private ip
89
97
  * ${mac} MAC address
90
98
  * ${vpc_id} vpc id
91
99
  * ${subnet_id} subnet id
data/Rakefile CHANGED
@@ -1,9 +1,17 @@
1
1
  require "bundler/gem_tasks"
2
+ require "fluent/version"
3
+ require "rake/testtask"
2
4
 
3
- require 'rake/testtask'
4
5
  Rake::TestTask.new(:test) do |test|
5
6
  test.libs << 'lib' << 'test'
6
- test.pattern = 'test/**/test_*.rb'
7
+
8
+ test.test_files = case
9
+ when Fluent::VERSION.start_with?("0.10.")
10
+ ["test/plugin/test_out_ec2_metadata.rb"]
11
+ else
12
+ Dir['test/**/test_*.rb']
13
+ end
14
+
7
15
  test.verbose = true
8
16
  end
9
17
 
data/fluent-plugin-ec2-metadata.gemspec CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-ec2-metadata"
7
- spec.version = "0.0.10"
7
+ spec.version = "0.0.11"
8
8
  spec.authors = ["SAKAMOTO Takumi"]
9
9
  spec.email = ["takumi.saka@gmail.com"]
10
10
  spec.description = %q{Fluentd output plugin to add ec2 metadata fields to a event record}
@@ -17,11 +17,12 @@ Gem::Specification.new do |spec|
17
17
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
18
  spec.require_paths = ["lib"]
19
19
 
20
+ spec.add_runtime_dependency "fluentd"
21
+ spec.add_runtime_dependency "oj"
22
+ spec.add_runtime_dependency "aws-sdk"
23
+
20
24
  spec.add_development_dependency "rake"
21
25
  spec.add_development_dependency "vcr"
22
26
  spec.add_development_dependency "webmock"
23
27
  spec.add_development_dependency "test-unit", ">= 3.1.0"
24
- spec.add_runtime_dependency "fluentd"
25
- spec.add_runtime_dependency "oj"
26
- spec.add_runtime_dependency "aws-sdk"
27
28
  end
data/lib/fluent/plugin/ec2_metadata.rb CHANGED
@@ -22,37 +22,46 @@ module Fluent
22
22
 
23
23
  @placeholder_expander = PlaceholderExpander.new
24
24
 
25
- set_metadata
25
+ # get metadata first and then setup a refresh thread
26
+ set_metadata
26
27
  set_tag
28
+ @refresh_thread = Thread.new {
29
+ while true
30
+ sleep @metadata_refresh_seconds
31
+ set_metadata
32
+ set_tag
33
+ end
34
+ }
27
35
  end
28
36
 
29
37
  private
30
38
 
31
39
  def set_metadata()
32
- @ec2_metadata = {}
33
-
40
+ ec2_metadata = {}
34
41
 
35
42
  instance_identity = Oj.load(get_dynamic_data("instance-identity/document"))
36
- @ec2_metadata['account_id'] = instance_identity["accountId"]
37
- @ec2_metadata['image_id'] = instance_identity["imageId"]
38
-
39
- @ec2_metadata['instance_id'] = get_metadata('instance-id')
40
- @ec2_metadata['instance_type'] = get_metadata('instance-type')
41
- @ec2_metadata['availability_zone'] = get_metadata('placement/availability-zone')
42
- @ec2_metadata['region'] = @ec2_metadata['availability_zone'].chop
43
- @ec2_metadata['mac'] = get_metadata('mac')
43
+ ec2_metadata['account_id'] = instance_identity["accountId"]
44
+ ec2_metadata['image_id'] = instance_identity["imageId"]
45
+
46
+ ec2_metadata['instance_id'] = get_metadata('instance-id')
47
+ ec2_metadata['instance_type'] = get_metadata('instance-type')
48
+ ec2_metadata['availability_zone'] = get_metadata('placement/availability-zone')
49
+ ec2_metadata['region'] = ec2_metadata['availability_zone'].chop
50
+ ec2_metadata['private_ip'] = get_metadata('local-ipv4')
51
+ ec2_metadata['mac'] = get_metadata('mac')
44
52
  begin
45
- @ec2_metadata['vpc_id'] = get_metadata("network/interfaces/macs/#{@ec2_metadata['mac']}/vpc-id")
53
+ ec2_metadata['vpc_id'] = get_metadata("network/interfaces/macs/#{ec2_metadata['mac']}/vpc-id")
46
54
  rescue
47
- @ec2_metadata['vpc_id'] = nil
48
- $log.info "ec2-metadata: 'vpc_id' is undefined #{@ec2_metadata['instance_id']} is not in VPC}"
55
+ ec2_metadata['vpc_id'] = nil
56
+ $log.info "ec2-metadata: 'vpc_id' is undefined #{ec2_metadata['instance_id']} is not in VPC}"
49
57
  end
50
58
  begin
51
- @ec2_metadata['subnet_id'] = get_metadata("network/interfaces/macs/#{@ec2_metadata['mac']}/subnet-id")
59
+ ec2_metadata['subnet_id'] = get_metadata("network/interfaces/macs/#{ec2_metadata['mac']}/subnet-id")
52
60
  rescue
53
- @ec2_metadata['subnet_id'] = nil
54
- $log.info "ec2-metadata: 'subnet_id' is undefined because #{@ec2_metadata['instance_id']} is not in VPC}"
61
+ ec2_metadata['subnet_id'] = nil
62
+ $log.info "ec2-metadata: 'subnet_id' is undefined because #{ec2_metadata['instance_id']} is not in VPC}"
55
63
  end
64
+ @ec2_metadata=ec2_metadata
56
65
  end
57
66
 
58
67
  def get_dynamic_data(f)
data/lib/fluent/plugin/filter_ec2_metadata.rb CHANGED
@@ -13,6 +13,7 @@ module Fluent
13
13
 
14
14
  config_param :aws_key_id, :string, :default => ENV['AWS_ACCESS_KEY_ID'], :secret => true
15
15
  config_param :aws_sec_key, :string, :default => ENV['AWS_SECRET_ACCESS_KEY'], :secret => true
16
+ config_param :metadata_refresh_seconds, :integer, :default => 300
16
17
 
17
18
  attr_reader :ec2_metadata
18
19
 
data/lib/fluent/plugin/out_ec2_metadata.rb CHANGED
@@ -14,6 +14,7 @@ module Fluent
14
14
  config_param :output_tag, :string
15
15
  config_param :aws_key_id, :string, :default => ENV['AWS_ACCESS_KEY_ID'], :secret => true
16
16
  config_param :aws_sec_key, :string, :default => ENV['AWS_SECRET_ACCESS_KEY'], :secret => true
17
+ config_param :metadata_refresh_seconds, :integer, :default => 300
17
18
 
18
19
  attr_reader :ec2_metadata
19
20
 
data/test/cassettes/ec2-classic.yml CHANGED
@@ -117,6 +117,45 @@ http_interactions:
117
117
  string: ap-northeast-1b
118
118
  http_version:
119
119
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
120
+ - request:
121
+ method: get
122
+ uri: http://169.254.169.254/latest/meta-data/local-ipv4
123
+ body:
124
+ encoding: US-ASCII
125
+ string: ''
126
+ headers:
127
+ Accept-Encoding:
128
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
129
+ Accept:
130
+ - '*/*'
131
+ User-Agent:
132
+ - Ruby
133
+ response:
134
+ status:
135
+ code: 200
136
+ message: OK
137
+ headers:
138
+ Content-Type:
139
+ - text/plain
140
+ Accept-Ranges:
141
+ - bytes
142
+ Etag:
143
+ - '"1870317889"'
144
+ Last-Modified:
145
+ - Thu, 02 Jul 2015 12:49:17 GMT
146
+ Content-Length:
147
+ - '15'
148
+ Connection:
149
+ - close
150
+ Date:
151
+ - Sun, 13 Dec 2015 06:37:00 GMT
152
+ Server:
153
+ - EC2ws
154
+ body:
155
+ encoding: UTF-8
156
+ string: 10.21.34.200
157
+ http_version:
158
+ recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
120
159
  - request:
121
160
  method: get
122
161
  uri: http://169.254.169.254/latest/meta-data/mac
data/test/cassettes/ec2-vpc.yml CHANGED
@@ -117,6 +117,45 @@ http_interactions:
117
117
  string: ap-northeast-1b
118
118
  http_version:
119
119
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
120
+ - request:
121
+ method: get
122
+ uri: http://169.254.169.254/latest/meta-data/local-ipv4
123
+ body:
124
+ encoding: US-ASCII
125
+ string: ''
126
+ headers:
127
+ Accept-Encoding:
128
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
129
+ Accept:
130
+ - '*/*'
131
+ User-Agent:
132
+ - Ruby
133
+ response:
134
+ status:
135
+ code: 200
136
+ message: OK
137
+ headers:
138
+ Content-Type:
139
+ - text/plain
140
+ Accept-Ranges:
141
+ - bytes
142
+ Etag:
143
+ - '"1870317889"'
144
+ Last-Modified:
145
+ - Thu, 02 Jul 2015 12:49:17 GMT
146
+ Content-Length:
147
+ - '15'
148
+ Connection:
149
+ - close
150
+ Date:
151
+ - Sun, 13 Dec 2015 06:37:00 GMT
152
+ Server:
153
+ - EC2ws
154
+ body:
155
+ encoding: UTF-8
156
+ string: 10.21.34.200
157
+ http_version:
158
+ recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
120
159
  - request:
121
160
  method: get
122
161
  uri: http://169.254.169.254/latest/meta-data/mac
data/test/helper.rb CHANGED
@@ -33,8 +33,5 @@ VCR.configure do |config|
33
33
  config.ignore_hosts 'codeclimate.com'
34
34
  end
35
35
 
36
- require 'fluent/plugin/out_ec2_metadata'
37
- require 'fluent/plugin/filter_ec2_metadata'
38
-
39
36
  class Test::Unit::TestCase
40
37
  end
data/test/plugin/test_filter_ec2_metadata.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require 'helper'
2
+ require 'fluent/plugin/filter_ec2_metadata'
2
3
 
3
4
  require 'webmock/test_unit'
4
5
  WebMock.disable_net_connect!
data/test/plugin/test_out_ec2_metadata.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require 'helper'
2
+ require 'fluent/plugin/out_ec2_metadata'
2
3
 
3
4
  require 'webmock/test_unit'
4
5
  WebMock.disable_net_connect!
@@ -43,6 +44,7 @@ class EC2MetadataOutputTest < Test::Unit::TestCase
43
44
 
44
45
  assert_equal("ami-123456", d.instance.ec2_metadata['image_id'])
45
46
  assert_equal("123456789", d.instance.ec2_metadata['account_id'])
47
+ assert_equal("10.21.34.200", d.instance.ec2_metadata['private_ip'])
46
48
 
47
49
  assert_equal("i-0c0c0000", d.instance.ec2_metadata['instance_id'])
48
50
  assert_equal("m3.large", d.instance.ec2_metadata['instance_type'])
metadata CHANGED
@@ -1,23 +1,23 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-ec2-metadata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.10
4
+ version: 0.0.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - SAKAMOTO Takumi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-05-17 00:00:00.000000000 Z
11
+ date: 2016-07-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: rake
14
+ name: fluentd
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
19
  version: '0'
20
- type: :development
20
+ type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
@@ -25,13 +25,13 @@ dependencies:
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
- name: vcr
28
+ name: oj
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
33
  version: '0'
34
- type: :development
34
+ type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
@@ -39,13 +39,13 @@ dependencies:
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
- name: webmock
42
+ name: aws-sdk
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0'
48
- type: :development
48
+ type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
@@ -53,27 +53,27 @@ dependencies:
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
- name: test-unit
56
+ name: rake
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - ">="
60
60
  - !ruby/object:Gem::Version
61
- version: 3.1.0
61
+ version: '0'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - ">="
67
67
  - !ruby/object:Gem::Version
68
- version: 3.1.0
68
+ version: '0'
69
69
  - !ruby/object:Gem::Dependency
70
- name: fluentd
70
+ name: vcr
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - ">="
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
- type: :runtime
76
+ type: :development
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
@@ -81,13 +81,13 @@ dependencies:
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
- name: oj
84
+ name: webmock
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
87
  - - ">="
88
88
  - !ruby/object:Gem::Version
89
89
  version: '0'
90
- type: :runtime
90
+ type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
@@ -95,19 +95,19 @@ dependencies:
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
97
  - !ruby/object:Gem::Dependency
98
- name: aws-sdk
98
+ name: test-unit
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
101
  - - ">="
102
102
  - !ruby/object:Gem::Version
103
- version: '0'
104
- type: :runtime
103
+ version: 3.1.0
104
+ type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - ">="
109
109
  - !ruby/object:Gem::Version
110
- version: '0'
110
+ version: 3.1.0
111
111
  description: Fluentd output plugin to add ec2 metadata fields to a event record
112
112
  email:
113
113
  - takumi.saka@gmail.com
@@ -118,6 +118,8 @@ files:
118
118
  - ".gitignore"
119
119
  - ".travis.yml"
120
120
  - Gemfile
121
+ - Gemfile.fluentd.0.10
122
+ - Gemfile.fluentd.0.12
121
123
  - LICENSE.txt
122
124
  - README.md
123
125
  - Rakefile