fluent-plugin-ec2-metadata 0.0.9 → 0.0.10

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6a0cd615badf1c5b2a61d1911c8f51cbb902fe2c
4
- data.tar.gz: 57b1f0dc6af6988d3be54a8880e2d3f162615f7c
3
+ metadata.gz: dbc6f8c80499b4f5d1035976e73dbed3ab175983
4
+ data.tar.gz: dfc17e48483cc59aa02039c5355debc7a6ac4cc3
5
5
  SHA512:
6
- metadata.gz: 61e1f818cc83363166125539aca48585e6f0b34a454f423c8b559c95730f36b8adf8babf15360eb5983d21895b06631de3eef04feed6a09d1c97a74e4238fa38
7
- data.tar.gz: 6d4418525ae1c495dee94d07dda681166c65921287729eda4d9f752e8450d540aaa1945fcd3388735b7998a8b7c85b156fe6173e428eb99ae2557d48a6638da6
6
+ metadata.gz: 95e7f3548ba2b4d8e0dea2ddd86020dac3f7cfcccdbd0454ed874520a79140f0fb85bb9d821e575036e8ca67f9a9cca7f1665e36a5dd8506b5503fcfb11029c8
7
+ data.tar.gz: 27470dc7107882b981585d3ffd4ea5277e2463af28b713f8cee6390886222e4908ba83b7fbad0bb46f84acfbe754b644f69f706dac7a1cfafe5a6c824982c4fc
data/.travis.yml CHANGED
@@ -1,9 +1,7 @@
1
+ sudo: false
1
2
  language: ruby
2
3
  rvm:
3
4
  - 2.2
4
5
  - 2.1
5
6
  - 2.0.0
6
7
  - 1.9.3
7
- script:
8
- - bundle exec rake test
9
- - bundle exec rake build
data/README.md CHANGED
@@ -2,8 +2,10 @@
2
2
 
3
3
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-ec2-metadata.svg)](http://badge.fury.io/rb/fluent-plugin-ec2-metadata)
4
4
  [![Build Status](https://travis-ci.org/takus/fluent-plugin-ec2-metadata.svg?branch=master)](https://travis-ci.org/takus/fluent-plugin-ec2-metadata)
5
+ [![Dependency Status](https://gemnasium.com/takus/fluent-plugin-ec2-metadata.svg)](https://gemnasium.com/takus/fluent-plugin-ec2-metadata)
5
6
  [![Test Coverage](https://codeclimate.com/github/takus/fluent-plugin-ec2-metadata/badges/coverage.svg)](https://codeclimate.com/github/takus/fluent-plugin-ec2-metadata/coverage)
6
7
  [![Code Climate](https://codeclimate.com/github/takus/fluent-plugin-ec2-metadata/badges/gpa.svg)](https://codeclimate.com/github/takus/fluent-plugin-ec2-metadata)
8
+ [![Codacy Badge](https://api.codacy.com/project/badge/grade/16f6786edb554f1ea7462353808011d6)](https://www.codacy.com/app/takus/fluent-plugin-ec2-metadata)
7
9
 
8
10
  [Fluentd](http://fluentd.org) plugin to add ec2 metadata fields to a event record
9
11
 
@@ -29,28 +31,51 @@ Example:
29
31
  instance_type ${instance_type}
30
32
  az ${availability_zone}
31
33
  vpc_id ${vpc_id}
34
+ ami_id ${image_id}
35
+ account_id ${account_id}
32
36
  </record>
33
37
  </match>
34
38
 
35
39
  Assume following input is coming:
36
40
 
37
- ```js
41
+ ```
38
42
  foo.bar {"message":"hello ec2!"}
39
43
  ```
40
44
 
41
45
  then output becomes as below (indented):
42
46
 
43
- ```js
47
+ ```
44
48
  i-28b5ee77.foo.bar {
45
49
  "hostname" : "web0001",
46
50
  "instance_id" : "i-28b5ee77",
47
51
  "instance_type" : "m1.large",
48
52
  "az" : "us-west-1b",
49
53
  "vpc_id" : "vpc-25dab194",
54
+ "account_id" : "123456789",
55
+ "image_id" : "ami-123456",
50
56
  "message" : "hello ec2!"
51
57
  }
52
58
  ```
53
59
 
60
+ Or you can use filter version:
61
+
62
+ <filter foo.**>
63
+ type ec2_metadata
64
+
65
+ aws_key_id YOUR_AWS_KEY_ID
66
+ aws_sec_key YOUR_AWS_SECRET/KEY
67
+
68
+ <record>
69
+ hostname ${tagset_name}
70
+ instance_id ${instance_id}
71
+ instance_type ${instance_type}
72
+ az ${availability_zone}
73
+ vpc_id ${vpc_id}
74
+ ami_id ${image_id}
75
+ account_id ${account_id}
76
+ </record>
77
+ </filter>
78
+
54
79
  ### Placeholders
55
80
 
56
81
  The following placeholders are always available:
@@ -64,6 +89,8 @@ The following placeholders are always available:
64
89
  * ${mac} MAC address
65
90
  * ${vpc_id} vpc id
66
91
  * ${subnet_id} subnet id
92
+ * ${account_id} account id
93
+ * ${image_id} ami image id
67
94
 
68
95
  The followings are available when you define `aws_key_id` and `aws_sec_key`(or define IAM Policy):
69
96
 
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-ec2-metadata"
7
- spec.version = "0.0.9"
7
+ spec.version = "0.0.10"
8
8
  spec.authors = ["SAKAMOTO Takumi"]
9
9
  spec.email = ["takumi.saka@gmail.com"]
10
10
  spec.description = %q{Fluentd output plugin to add ec2 metadata fields to a event record}
@@ -22,5 +22,6 @@ Gem::Specification.new do |spec|
22
22
  spec.add_development_dependency "webmock"
23
23
  spec.add_development_dependency "test-unit", ">= 3.1.0"
24
24
  spec.add_runtime_dependency "fluentd"
25
+ spec.add_runtime_dependency "oj"
25
26
  spec.add_runtime_dependency "aws-sdk"
26
27
  end
@@ -0,0 +1,144 @@
1
+ module Fluent
2
+ module EC2Metadata
3
+
4
+ def initialize
5
+ super
6
+ require 'net/http'
7
+ require 'aws-sdk'
8
+ require 'oj'
9
+ end
10
+
11
+ def configure(conf)
12
+ super
13
+
14
+ # <record></record> directive
15
+ @map = {}
16
+ conf.elements.select { |element| element.name == 'record' }.each { |element|
17
+ element.each_pair { |k, v|
18
+ element.has_key?(k)
19
+ @map[k] = v
20
+ }
21
+ }
22
+
23
+ @placeholder_expander = PlaceholderExpander.new
24
+
25
+ set_metadata
26
+ set_tag
27
+ end
28
+
29
+ private
30
+
31
+ def set_metadata()
32
+ @ec2_metadata = {}
33
+
34
+
35
+ instance_identity = Oj.load(get_dynamic_data("instance-identity/document"))
36
+ @ec2_metadata['account_id'] = instance_identity["accountId"]
37
+ @ec2_metadata['image_id'] = instance_identity["imageId"]
38
+
39
+ @ec2_metadata['instance_id'] = get_metadata('instance-id')
40
+ @ec2_metadata['instance_type'] = get_metadata('instance-type')
41
+ @ec2_metadata['availability_zone'] = get_metadata('placement/availability-zone')
42
+ @ec2_metadata['region'] = @ec2_metadata['availability_zone'].chop
43
+ @ec2_metadata['mac'] = get_metadata('mac')
44
+ begin
45
+ @ec2_metadata['vpc_id'] = get_metadata("network/interfaces/macs/#{@ec2_metadata['mac']}/vpc-id")
46
+ rescue
47
+ @ec2_metadata['vpc_id'] = nil
48
+ $log.info "ec2-metadata: 'vpc_id' is undefined #{@ec2_metadata['instance_id']} is not in VPC}"
49
+ end
50
+ begin
51
+ @ec2_metadata['subnet_id'] = get_metadata("network/interfaces/macs/#{@ec2_metadata['mac']}/subnet-id")
52
+ rescue
53
+ @ec2_metadata['subnet_id'] = nil
54
+ $log.info "ec2-metadata: 'subnet_id' is undefined because #{@ec2_metadata['instance_id']} is not in VPC}"
55
+ end
56
+ end
57
+
58
+ def get_dynamic_data(f)
59
+ res = Net::HTTP.get_response("169.254.169.254", "/latest/dynamic/#{f}")
60
+ raise Fluent::ConfigError, "ec2-dynamic-data: failed to get #{f}" unless res.is_a?(Net::HTTPSuccess)
61
+ res.body
62
+ end
63
+
64
+ def get_metadata(f)
65
+ res = Net::HTTP.get_response("169.254.169.254", "/latest/meta-data/#{f}")
66
+ raise Fluent::ConfigError, "ec2-metadata: failed to get #{f}" unless res.is_a?(Net::HTTPSuccess)
67
+ res.body
68
+ end
69
+
70
+ def set_tag()
71
+ if @map.values.any? { |v| v.match(/^\${tagset_/) } || @output_tag =~ /\${tagset_/
72
+
73
+ if @aws_key_id and @aws_sec_key
74
+ ec2 = Aws::EC2::Client.new(
75
+ region: @ec2_metadata['region'],
76
+ access_key_id: @aws_key_id,
77
+ secret_access_key: @aws_sec_key,
78
+ )
79
+ else
80
+ ec2 = Aws::EC2::Client.new(
81
+ region: @ec2_metadata['region'],
82
+ )
83
+ end
84
+
85
+ response = ec2.describe_instances({ :instance_ids => [@ec2_metadata['instance_id']] })
86
+ instance = response.reservations[0].instances[0]
87
+ raise Fluent::ConfigError, "ec2-metadata: failed to get instance data #{response.pretty_inspect}" if instance.nil?
88
+
89
+ instance.tags.each { |tag|
90
+ @ec2_metadata["tagset_#{tag.key.downcase}"] = tag.value
91
+ }
92
+ end
93
+ end
94
+
95
+ def modify_record(record, tag, tag_parts)
96
+ @placeholder_expander.prepare_placeholders(record, tag, tag_parts, @ec2_metadata)
97
+ new_record = record.dup
98
+ @map.each_pair { |k, v| new_record[k] = @placeholder_expander.expand(v) }
99
+ new_record
100
+ end
101
+
102
+ def modify(output_tag, record, tag, tag_parts)
103
+ @placeholder_expander.prepare_placeholders(record, tag, tag_parts, @ec2_metadata)
104
+
105
+ new_tag = @placeholder_expander.expand(output_tag)
106
+
107
+ new_record = record.dup
108
+ @map.each_pair { |k, v| new_record[k] = @placeholder_expander.expand(v) }
109
+
110
+ [new_tag, new_record]
111
+ end
112
+
113
+ class PlaceholderExpander
114
+ # referenced https://github.com/fluent/fluent-plugin-rewrite-tag-filter
115
+ # referenced https://github.com/sonots/fluent-plugin-record-reformer
116
+ attr_reader :placeholders
117
+
118
+ def prepare_placeholders(_record, tag, tag_parts, ec2_metadata)
119
+ placeholders = {
120
+ '${tag}' => tag,
121
+ }
122
+
123
+ size = tag_parts.size
124
+ tag_parts.each_with_index { |t, idx|
125
+ placeholders.store("${tag_parts[#{idx}]}", t)
126
+ placeholders.store("${tag_parts[#{idx-size}]}", t) # support tag_parts[-1]
127
+ }
128
+
129
+ ec2_metadata.each { |k, v|
130
+ placeholders.store("${#{k}}", v)
131
+ }
132
+
133
+ @placeholders = placeholders
134
+ end
135
+
136
+ def expand(str)
137
+ str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
138
+ $log.warn "ec2-metadata: unknown placeholder `#{$1}` found in a tag `#{@placeholders['${tag}']}`" unless @placeholders.include?($1)
139
+ @placeholders[$1]
140
+ }
141
+ end
142
+ end
143
+ end
144
+ end
@@ -0,0 +1,26 @@
1
+ require_relative 'ec2_metadata'
2
+
3
+ module Fluent
4
+ class EC2MetadataFilter < Filter
5
+ include Fluent::EC2Metadata
6
+
7
+ Fluent::Plugin.register_filter('ec2_metadata', self)
8
+
9
+ # Define `router` method of v0.12 to support v0.10 or earlier
10
+ unless method_defined?(:router)
11
+ define_method("router") { Fluent::Engine }
12
+ end
13
+
14
+ config_param :aws_key_id, :string, :default => ENV['AWS_ACCESS_KEY_ID'], :secret => true
15
+ config_param :aws_sec_key, :string, :default => ENV['AWS_SECRET_ACCESS_KEY'], :secret => true
16
+
17
+ attr_reader :ec2_metadata
18
+
19
+ def filter(tag, time, record)
20
+ tag_parts = tag.split('.')
21
+ modify_record(record, tag, tag_parts)
22
+ rescue => e
23
+ $log.warn "ec2-metadata: #{e.class} #{e.message} #{e.backtrace.join(', ')}"
24
+ end
25
+ end
26
+ end
@@ -1,5 +1,9 @@
1
+ require_relative 'ec2_metadata'
2
+
1
3
  module Fluent
2
4
  class EC2MetadataOutput < Output
5
+ include EC2Metadata
6
+
3
7
  Fluent::Plugin.register_output('ec2_metadata', self)
4
8
 
5
9
  # Define `router` method of v0.12 to support v0.10 or earlier
@@ -7,74 +11,11 @@ module Fluent
7
11
  define_method("router") { Fluent::Engine }
8
12
  end
9
13
 
10
- def initialize
11
- super
12
- require 'net/http'
13
- end
14
-
15
14
  config_param :output_tag, :string
16
15
  config_param :aws_key_id, :string, :default => ENV['AWS_ACCESS_KEY_ID'], :secret => true
17
16
  config_param :aws_sec_key, :string, :default => ENV['AWS_SECRET_ACCESS_KEY'], :secret => true
18
17
 
19
- def configure(conf)
20
- super
21
-
22
- # <record></record> directive
23
- @map = {}
24
- conf.elements.select { |element| element.name == 'record' }.each { |element|
25
- element.each_pair { |k, v|
26
- element.has_key?(k)
27
- @map[k] = v
28
- }
29
- }
30
-
31
- @placeholder_expander = PlaceholderExpander.new
32
-
33
- # get ec2 metadata
34
- @ec2_metadata = {}
35
- @ec2_metadata['instance_id'] = get_metadata('instance-id')
36
- @ec2_metadata['instance_type'] = get_metadata('instance-type')
37
- @ec2_metadata['availability_zone'] = get_metadata('placement/availability-zone')
38
- @ec2_metadata['region'] = @ec2_metadata['availability_zone'].chop
39
- @ec2_metadata['mac'] = get_metadata('mac')
40
- begin
41
- @ec2_metadata['vpc_id'] = get_metadata("network/interfaces/macs/#{@ec2_metadata['mac']}/vpc-id")
42
- rescue
43
- @ec2_metadata['vpc_id'] = nil
44
- $log.info "ec2-metadata: 'vpc_id' is undefined #{@ec2_metadata['instance_id']} is not in VPC}"
45
- end
46
- begin
47
- @ec2_metadata['subnet_id'] = get_metadata("network/interfaces/macs/#{@ec2_metadata['mac']}/subnet-id")
48
- rescue
49
- @ec2_metadata['subnet_id'] = nil
50
- $log.info "ec2-metadata: 'subnet_id' is undefined because #{@ec2_metadata['instance_id']} is not in VPC}"
51
- end
52
-
53
- # get tags
54
- if @map.values.any? { |v| v.match(/^\${tagset_/) } || @output_tag =~ /\${tagset_/
55
- require 'aws-sdk'
56
-
57
- if @aws_key_id and @aws_sec_key then
58
- ec2 = Aws::EC2::Client.new(
59
- region: @ec2_metadata['region'],
60
- access_key_id: @aws_key_id,
61
- secret_access_key: @aws_sec_key,
62
- )
63
- else
64
- ec2 = Aws::EC2::Client.new(
65
- region: @ec2_metadata['region'],
66
- )
67
- end
68
-
69
- response = ec2.describe_instances({ :instance_ids => [@ec2_metadata['instance_id']] })
70
- instance = response.reservations[0].instances[0]
71
- raise Fluent::ConfigError, "ec2-metadata: failed to get instance data #{response.pretty_inspect}" if instance.nil?
72
-
73
- instance.tags.each { |tag|
74
- @ec2_metadata["tagset_#{tag.key.downcase}"] = tag.value
75
- }
76
- end
77
- end
18
+ attr_reader :ec2_metadata
78
19
 
79
20
  def emit(tag, es, chain)
80
21
  tag_parts = tag.split('.')
@@ -86,56 +27,5 @@ module Fluent
86
27
  rescue => e
87
28
  $log.warn "ec2-metadata: #{e.class} #{e.message} #{e.backtrace.join(', ')}"
88
29
  end
89
-
90
- private
91
-
92
- def get_metadata(f)
93
- res = Net::HTTP.get_response("169.254.169.254", "/latest/meta-data/#{f}")
94
- raise Fluent::ConfigError, "ec2-metadata: failed to get #{f}" unless res.is_a?(Net::HTTPSuccess)
95
- res.body
96
- end
97
-
98
- def modify(output_tag, record, tag, tag_parts)
99
- @placeholder_expander.prepare_placeholders(record, tag, tag_parts, @ec2_metadata)
100
-
101
- new_tag = @placeholder_expander.expand(output_tag)
102
-
103
- new_record = record.dup
104
- @map.each_pair { |k, v| new_record[k] = @placeholder_expander.expand(v) }
105
-
106
- [new_tag, new_record]
107
- end
108
-
109
- class PlaceholderExpander
110
- # referenced https://github.com/fluent/fluent-plugin-rewrite-tag-filter
111
- # referenced https://github.com/sonots/fluent-plugin-record-reformer
112
- attr_reader :placeholders
113
-
114
- def prepare_placeholders(record, tag, tag_parts, ec2_metadata)
115
- placeholders = {
116
- '${tag}' => tag,
117
- }
118
-
119
- size = tag_parts.size
120
- tag_parts.each_with_index { |t, idx|
121
- placeholders.store("${tag_parts[#{idx}]}", t)
122
- placeholders.store("${tag_parts[#{idx-size}]}", t) # support tag_parts[-1]
123
- }
124
-
125
- ec2_metadata.each { |k, v|
126
- placeholders.store("${#{k}}", v)
127
- }
128
-
129
- @placeholders = placeholders
130
- end
131
-
132
- def expand(str)
133
- str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
134
- $log.warn "ec2-metadata: unknown placeholder `#{$1}` found in a tag `#{@placeholders['${tag}']}`" unless @placeholders.include?($1)
135
- @placeholders[$1]
136
- }
137
- end
138
- end
139
-
140
30
  end
141
31
  end
@@ -0,0 +1,302 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://169.254.169.254/latest/meta-data/instance-id
6
+ body:
7
+ encoding: US-ASCII
8
+ string: ''
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - '*/*'
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Content-Type:
22
+ - text/plain
23
+ Accept-Ranges:
24
+ - bytes
25
+ Etag:
26
+ - '"1971254081"'
27
+ Last-Modified:
28
+ - Thu, 02 Jul 2015 12:49:17 GMT
29
+ Content-Length:
30
+ - '10'
31
+ Connection:
32
+ - close
33
+ Date:
34
+ - Sun, 13 Dec 2015 06:37:00 GMT
35
+ Server:
36
+ - EC2ws
37
+ body:
38
+ encoding: UTF-8
39
+ string: i-0c0c0000
40
+ http_version:
41
+ recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
42
+ - request:
43
+ method: get
44
+ uri: http://169.254.169.254/latest/meta-data/instance-type
45
+ body:
46
+ encoding: US-ASCII
47
+ string: ''
48
+ headers:
49
+ Accept-Encoding:
50
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
51
+ Accept:
52
+ - '*/*'
53
+ User-Agent:
54
+ - Ruby
55
+ response:
56
+ status:
57
+ code: 200
58
+ message: OK
59
+ headers:
60
+ Content-Type:
61
+ - text/plain
62
+ Accept-Ranges:
63
+ - bytes
64
+ Etag:
65
+ - '"765393446"'
66
+ Last-Modified:
67
+ - Thu, 02 Jul 2015 12:49:17 GMT
68
+ Content-Length:
69
+ - '8'
70
+ Connection:
71
+ - close
72
+ Date:
73
+ - Sun, 13 Dec 2015 06:37:00 GMT
74
+ Server:
75
+ - EC2ws
76
+ body:
77
+ encoding: UTF-8
78
+ string: m3.large
79
+ http_version:
80
+ recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
81
+ - request:
82
+ method: get
83
+ uri: http://169.254.169.254/latest/meta-data/placement/availability-zone
84
+ body:
85
+ encoding: US-ASCII
86
+ string: ''
87
+ headers:
88
+ Accept-Encoding:
89
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
90
+ Accept:
91
+ - '*/*'
92
+ User-Agent:
93
+ - Ruby
94
+ response:
95
+ status:
96
+ code: 200
97
+ message: OK
98
+ headers:
99
+ Content-Type:
100
+ - text/plain
101
+ Accept-Ranges:
102
+ - bytes
103
+ Etag:
104
+ - '"1870317889"'
105
+ Last-Modified:
106
+ - Thu, 02 Jul 2015 12:49:17 GMT
107
+ Content-Length:
108
+ - '15'
109
+ Connection:
110
+ - close
111
+ Date:
112
+ - Sun, 13 Dec 2015 06:37:00 GMT
113
+ Server:
114
+ - EC2ws
115
+ body:
116
+ encoding: UTF-8
117
+ string: ap-northeast-1b
118
+ http_version:
119
+ recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
120
+ - request:
121
+ method: get
122
+ uri: http://169.254.169.254/latest/meta-data/mac
123
+ body:
124
+ encoding: US-ASCII
125
+ string: ''
126
+ headers:
127
+ Accept-Encoding:
128
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
129
+ Accept:
130
+ - '*/*'
131
+ User-Agent:
132
+ - Ruby
133
+ response:
134
+ status:
135
+ code: 200
136
+ message: OK
137
+ headers:
138
+ Content-Type:
139
+ - text/plain
140
+ Accept-Ranges:
141
+ - bytes
142
+ Etag:
143
+ - '"1925384845"'
144
+ Last-Modified:
145
+ - Fri, 27 Feb 2015 04:49:13 GMT
146
+ Content-Length:
147
+ - '17'
148
+ Connection:
149
+ - close
150
+ Date:
151
+ - Wed, 16 Dec 2015 17:11:22 GMT
152
+ Server:
153
+ - EC2ws
154
+ body:
155
+ encoding: UTF-8
156
+ string: 00:00:0A:AA:0A:0A
157
+ http_version:
158
+ recorded_at: Wed, 16 Dec 2015 17:11:22 GMT
159
+ - request:
160
+ method: get
161
+ uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:00:0A:AA:0A:0A/vpc-id
162
+ body:
163
+ encoding: US-ASCII
164
+ string: ''
165
+ headers:
166
+ Accept-Encoding:
167
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
168
+ Accept:
169
+ - '*/*'
170
+ User-Agent:
171
+ - Ruby
172
+ response:
173
+ status:
174
+ code: 404
175
+ message: Not Found
176
+ headers:
177
+ Content-Type:
178
+ - text/html
179
+ Content-Length:
180
+ - '345'
181
+ Connection:
182
+ - close
183
+ Date:
184
+ - Wed, 16 Dec 2015 17:11:22 GMT
185
+ Server:
186
+ - EC2ws
187
+ body:
188
+ encoding: UTF-8
189
+ string: |
190
+ <?xml version="1.0" encoding="iso-8859-1"?>
191
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
192
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
193
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
194
+ <head>
195
+ <title>404 - Not Found</title>
196
+ </head>
197
+ <body>
198
+ <h1>404 - Not Found</h1>
199
+ </body>
200
+ </html>
201
+ http_version:
202
+ recorded_at: Wed, 16 Dec 2015 17:11:22 GMT
203
+ - request:
204
+ method: get
205
+ uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:00:0A:AA:0A:0A/subnet-id
206
+ body:
207
+ encoding: US-ASCII
208
+ string: ''
209
+ headers:
210
+ Accept-Encoding:
211
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
212
+ Accept:
213
+ - '*/*'
214
+ User-Agent:
215
+ - Ruby
216
+ response:
217
+ status:
218
+ code: 404
219
+ message: Not Found
220
+ headers:
221
+ Content-Type:
222
+ - text/html
223
+ Content-Length:
224
+ - '345'
225
+ Connection:
226
+ - close
227
+ Date:
228
+ - Wed, 16 Dec 2015 17:11:22 GMT
229
+ Server:
230
+ - EC2ws
231
+ body:
232
+ encoding: UTF-8
233
+ string: |
234
+ <?xml version="1.0" encoding="iso-8859-1"?>
235
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
236
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
237
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
238
+ <head>
239
+ <title>404 - Not Found</title>
240
+ </head>
241
+ <body>
242
+ <h1>404 - Not Found</h1>
243
+ </body>
244
+ </html>
245
+ http_version:
246
+ recorded_at: Wed, 16 Dec 2015 17:11:22 GMT
247
+ - request:
248
+ method: get
249
+ uri: http://169.254.169.254/latest/dynamic/instance-identity/document
250
+ body:
251
+ encoding: US-ASCII
252
+ string: ''
253
+ headers:
254
+ Accept-Encoding:
255
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
256
+ Accept:
257
+ - '*/*'
258
+ User-Agent:
259
+ - Ruby
260
+ response:
261
+ status:
262
+ code: 200
263
+ message: OK
264
+ headers:
265
+ Content-Type:
266
+ - text/plain
267
+ Accept-Ranges:
268
+ - bytes
269
+ Etag:
270
+ - '"2871764318"'
271
+ Last-Modified:
272
+ - Wed, 11 May 2016 20:52:55 GMT
273
+ Content-Length:
274
+ - '422'
275
+ Connection:
276
+ - close
277
+ Date:
278
+ - Wed, 11 May 2016 21:05:40 GMT
279
+ Server:
280
+ - EC2ws
281
+ body:
282
+ encoding: UTF-8
283
+ string: |-
284
+ {
285
+ "privateIp" : "172.31.1.232",
286
+ "devpayProductCodes" : null,
287
+ "availabilityZone" : "us-west-1c",
288
+ "version" : "2010-08-31",
289
+ "instanceId" : "i-123456",
290
+ "billingProducts" : null,
291
+ "instanceType" : "t2.micro",
292
+ "accountId" : "123456789",
293
+ "imageId" : "ami-123456",
294
+ "pendingTime" : "2016-05-11T20:52:25Z",
295
+ "architecture" : "x86_64",
296
+ "kernelId" : null,
297
+ "ramdiskId" : null,
298
+ "region" : "us-west-1"
299
+ }
300
+ http_version:
301
+ recorded_at: Wed, 11 May 2016 21:05:40 GMT
302
+ recorded_with: VCR 3.0.1
@@ -10,7 +10,7 @@ http_interactions:
10
10
  Accept-Encoding:
11
11
  - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
12
  Accept:
13
- - "*/*"
13
+ - '*/*'
14
14
  User-Agent:
15
15
  - Ruby
16
16
  response:
@@ -37,7 +37,7 @@ http_interactions:
37
37
  body:
38
38
  encoding: UTF-8
39
39
  string: i-0c0c0000
40
- http_version:
40
+ http_version:
41
41
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
42
42
  - request:
43
43
  method: get
@@ -49,7 +49,7 @@ http_interactions:
49
49
  Accept-Encoding:
50
50
  - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
51
51
  Accept:
52
- - "*/*"
52
+ - '*/*'
53
53
  User-Agent:
54
54
  - Ruby
55
55
  response:
@@ -76,7 +76,7 @@ http_interactions:
76
76
  body:
77
77
  encoding: UTF-8
78
78
  string: m3.large
79
- http_version:
79
+ http_version:
80
80
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
81
81
  - request:
82
82
  method: get
@@ -88,7 +88,7 @@ http_interactions:
88
88
  Accept-Encoding:
89
89
  - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
90
90
  Accept:
91
- - "*/*"
91
+ - '*/*'
92
92
  User-Agent:
93
93
  - Ruby
94
94
  response:
@@ -115,7 +115,7 @@ http_interactions:
115
115
  body:
116
116
  encoding: UTF-8
117
117
  string: ap-northeast-1b
118
- http_version:
118
+ http_version:
119
119
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
120
120
  - request:
121
121
  method: get
@@ -127,7 +127,7 @@ http_interactions:
127
127
  Accept-Encoding:
128
128
  - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
129
129
  Accept:
130
- - "*/*"
130
+ - '*/*'
131
131
  User-Agent:
132
132
  - Ruby
133
133
  response:
@@ -153,12 +153,12 @@ http_interactions:
153
153
  - EC2ws
154
154
  body:
155
155
  encoding: UTF-8
156
- string: 00:c0:00:0c:cc:00
157
- http_version:
156
+ string: 00:A0:00:0A:AA:00
157
+ http_version:
158
158
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
159
159
  - request:
160
160
  method: get
161
- uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:c0:00:0c:cc:00/vpc-id
161
+ uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:A0:00:0A:AA:00/vpc-id
162
162
  body:
163
163
  encoding: US-ASCII
164
164
  string: ''
@@ -166,7 +166,7 @@ http_interactions:
166
166
  Accept-Encoding:
167
167
  - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
168
168
  Accept:
169
- - "*/*"
169
+ - '*/*'
170
170
  User-Agent:
171
171
  - Ruby
172
172
  response:
@@ -193,11 +193,11 @@ http_interactions:
193
193
  body:
194
194
  encoding: UTF-8
195
195
  string: vpc-00000000
196
- http_version:
196
+ http_version:
197
197
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
198
198
  - request:
199
199
  method: get
200
- uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:c0:00:0c:cc:00/subnet-id
200
+ uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:A0:00:0A:AA:00/subnet-id
201
201
  body:
202
202
  encoding: US-ASCII
203
203
  string: ''
@@ -205,7 +205,7 @@ http_interactions:
205
205
  Accept-Encoding:
206
206
  - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
207
207
  Accept:
208
- - "*/*"
208
+ - '*/*'
209
209
  User-Agent:
210
210
  - Ruby
211
211
  response:
@@ -232,8 +232,96 @@ http_interactions:
232
232
  body:
233
233
  encoding: UTF-8
234
234
  string: subnet-00000000
235
- http_version:
235
+ http_version:
236
236
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
237
+ - request:
238
+ method: get
239
+ uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:00:0A:AA:0A:0A/vpc-id
240
+ body:
241
+ encoding: US-ASCII
242
+ string: ''
243
+ headers:
244
+ Accept-Encoding:
245
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
246
+ Accept:
247
+ - '*/*'
248
+ User-Agent:
249
+ - Ruby
250
+ response:
251
+ status:
252
+ code: 404
253
+ message: Not Found
254
+ headers:
255
+ Content-Type:
256
+ - text/html
257
+ Content-Length:
258
+ - '345'
259
+ Connection:
260
+ - close
261
+ Date:
262
+ - Wed, 16 Dec 2015 17:11:22 GMT
263
+ Server:
264
+ - EC2ws
265
+ body:
266
+ encoding: UTF-8
267
+ string: |
268
+ <?xml version="1.0" encoding="iso-8859-1"?>
269
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
270
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
271
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
272
+ <head>
273
+ <title>404 - Not Found</title>
274
+ </head>
275
+ <body>
276
+ <h1>404 - Not Found</h1>
277
+ </body>
278
+ </html>
279
+ http_version:
280
+ recorded_at: Wed, 16 Dec 2015 17:11:22 GMT
281
+ - request:
282
+ method: get
283
+ uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:00:0A:AA:0A:0A/subnet-id
284
+ body:
285
+ encoding: US-ASCII
286
+ string: ''
287
+ headers:
288
+ Accept-Encoding:
289
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
290
+ Accept:
291
+ - '*/*'
292
+ User-Agent:
293
+ - Ruby
294
+ response:
295
+ status:
296
+ code: 404
297
+ message: Not Found
298
+ headers:
299
+ Content-Type:
300
+ - text/html
301
+ Content-Length:
302
+ - '345'
303
+ Connection:
304
+ - close
305
+ Date:
306
+ - Wed, 16 Dec 2015 17:11:22 GMT
307
+ Server:
308
+ - EC2ws
309
+ body:
310
+ encoding: UTF-8
311
+ string: |
312
+ <?xml version="1.0" encoding="iso-8859-1"?>
313
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
314
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
315
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
316
+ <head>
317
+ <title>404 - Not Found</title>
318
+ </head>
319
+ <body>
320
+ <h1>404 - Not Found</h1>
321
+ </body>
322
+ </html>
323
+ http_version:
324
+ recorded_at: Wed, 16 Dec 2015 17:11:22 GMT
237
325
  - request:
238
326
  method: post
239
327
  uri: https://ec2.ap-northeast-1.amazonaws.com/
@@ -254,7 +342,7 @@ http_interactions:
254
342
  Content-Length:
255
343
  - '67'
256
344
  Accept:
257
- - "*/*"
345
+ - '*/*'
258
346
  response:
259
347
  status:
260
348
  code: 200
@@ -295,6 +383,61 @@ http_interactions:
295
383
  </item>
296
384
  </reservationSet>
297
385
  </DescribeInstancesResponse>
298
- http_version:
386
+ http_version:
299
387
  recorded_at: Sun, 13 Dec 2015 10:06:07 GMT
300
- recorded_with: VCR 3.0.0
388
+ - request:
389
+ method: get
390
+ uri: http://169.254.169.254/latest/dynamic/instance-identity/document
391
+ body:
392
+ encoding: US-ASCII
393
+ string: ''
394
+ headers:
395
+ Accept-Encoding:
396
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
397
+ Accept:
398
+ - '*/*'
399
+ User-Agent:
400
+ - Ruby
401
+ response:
402
+ status:
403
+ code: 200
404
+ message: OK
405
+ headers:
406
+ Content-Type:
407
+ - text/plain
408
+ Accept-Ranges:
409
+ - bytes
410
+ Etag:
411
+ - '"2871764318"'
412
+ Last-Modified:
413
+ - Wed, 11 May 2016 20:52:55 GMT
414
+ Content-Length:
415
+ - '422'
416
+ Connection:
417
+ - close
418
+ Date:
419
+ - Wed, 11 May 2016 21:05:40 GMT
420
+ Server:
421
+ - EC2ws
422
+ body:
423
+ encoding: UTF-8
424
+ string: |-
425
+ {
426
+ "privateIp" : "172.31.1.232",
427
+ "devpayProductCodes" : null,
428
+ "availabilityZone" : "us-west-1c",
429
+ "version" : "2010-08-31",
430
+ "instanceId" : "i-123456",
431
+ "billingProducts" : null,
432
+ "instanceType" : "t2.micro",
433
+ "accountId" : "123456789",
434
+ "imageId" : "ami-123456",
435
+ "pendingTime" : "2016-05-11T20:52:25Z",
436
+ "architecture" : "x86_64",
437
+ "kernelId" : null,
438
+ "ramdiskId" : null,
439
+ "region" : "us-west-1"
440
+ }
441
+ http_version:
442
+ recorded_at: Wed, 11 May 2016 21:05:40 GMT
443
+ recorded_with: VCR 3.0.1
data/test/helper.rb CHANGED
@@ -34,6 +34,7 @@ VCR.configure do |config|
34
34
  end
35
35
 
36
36
  require 'fluent/plugin/out_ec2_metadata'
37
+ require 'fluent/plugin/filter_ec2_metadata'
37
38
 
38
39
  class Test::Unit::TestCase
39
40
  end
@@ -0,0 +1,73 @@
1
+ require 'helper'
2
+
3
+ require 'webmock/test_unit'
4
+ WebMock.disable_net_connect!
5
+
6
+ class EC2MetadataFilterTest < Test::Unit::TestCase
7
+
8
+ CONFIG = %[
9
+ aws_key_id aws_key
10
+ aws_sec_key aws_sec
11
+ <record>
12
+ name ${tagset_name}
13
+ instance_id ${instance_id}
14
+ az ${availability_zone}
15
+ </record>
16
+ ]
17
+
18
+ def setup
19
+ Fluent::Test.setup
20
+ @time = Fluent::Engine.now
21
+ end
22
+
23
+ def create_driver(conf=CONFIG, tag='test')
24
+ Fluent::Test::FilterTestDriver.new(Fluent::EC2MetadataFilter, tag).configure(conf)
25
+ end
26
+
27
+ test 'configure-vpc' do
28
+ VCR.use_cassette('ec2-vpc') do
29
+ c = %[
30
+ aws_key_id aws_key
31
+ aws_sec_key aws_sec
32
+ <record>
33
+ name ${tagset_name}
34
+ </record>
35
+ ]
36
+ d = create_driver(conf=c)
37
+
38
+ assert_equal("aws_key", d.instance.aws_key_id)
39
+ assert_equal("aws_sec", d.instance.aws_sec_key)
40
+
41
+ assert_equal("ami-123456", d.instance.ec2_metadata['image_id'])
42
+ assert_equal("123456789", d.instance.ec2_metadata['account_id'])
43
+
44
+ assert_equal("i-0c0c0000", d.instance.ec2_metadata['instance_id'])
45
+ assert_equal("m3.large", d.instance.ec2_metadata['instance_type'])
46
+ assert_equal("ap-northeast-1", d.instance.ec2_metadata['region'])
47
+ assert_equal("ap-northeast-1b", d.instance.ec2_metadata['availability_zone'])
48
+ assert_equal("00:A0:00:0A:AA:00", d.instance.ec2_metadata['mac'])
49
+ assert_equal("vpc-00000000", d.instance.ec2_metadata['vpc_id'])
50
+ assert_equal("subnet-00000000", d.instance.ec2_metadata['subnet_id'])
51
+
52
+ assert_equal("instance-name", d.instance.ec2_metadata['tagset_name'])
53
+ end
54
+ end
55
+
56
+ test 'configure-classic' do
57
+ VCR.use_cassette('ec2-classic') do
58
+ c = %[
59
+ <record>
60
+ instance_id ${instance_id}
61
+ </record>
62
+ ]
63
+ d = create_driver(conf=c)
64
+
65
+ assert_equal(nil, d.instance.aws_key_id)
66
+ assert_equal(nil, d.instance.aws_sec_key)
67
+
68
+ assert_equal("00:00:0A:AA:0A:0A", d.instance.ec2_metadata['mac'])
69
+ assert_equal(nil, d.instance.ec2_metadata['vpc_id'])
70
+ assert_equal(nil, d.instance.ec2_metadata['subnet_id'])
71
+ end
72
+ end
73
+ end
@@ -25,17 +25,59 @@ class EC2MetadataOutputTest < Test::Unit::TestCase
25
25
  Fluent::Test::OutputTestDriver.new(Fluent::EC2MetadataOutput, tag).configure(conf)
26
26
  end
27
27
 
28
- test 'configure' do
29
- VCR.use_cassette('ec2') do
30
- d = create_driver
28
+ test 'configure-vpc' do
29
+ VCR.use_cassette('ec2-vpc') do
30
+ c = %[
31
+ output_tag ${instance_id}.${tag}
32
+ aws_key_id aws_key
33
+ aws_sec_key aws_sec
34
+ <record>
35
+ name ${tagset_name}
36
+ </record>
37
+ ]
38
+ d = create_driver(conf=c)
39
+
31
40
  assert_equal("${instance_id}.${tag}", d.instance.output_tag)
32
41
  assert_equal("aws_key", d.instance.aws_key_id)
33
42
  assert_equal("aws_sec", d.instance.aws_sec_key)
43
+
44
+ assert_equal("ami-123456", d.instance.ec2_metadata['image_id'])
45
+ assert_equal("123456789", d.instance.ec2_metadata['account_id'])
46
+
47
+ assert_equal("i-0c0c0000", d.instance.ec2_metadata['instance_id'])
48
+ assert_equal("m3.large", d.instance.ec2_metadata['instance_type'])
49
+ assert_equal("ap-northeast-1", d.instance.ec2_metadata['region'])
50
+ assert_equal("ap-northeast-1b", d.instance.ec2_metadata['availability_zone'])
51
+ assert_equal("00:A0:00:0A:AA:00", d.instance.ec2_metadata['mac'])
52
+ assert_equal("vpc-00000000", d.instance.ec2_metadata['vpc_id'])
53
+ assert_equal("subnet-00000000", d.instance.ec2_metadata['subnet_id'])
54
+
55
+ assert_equal("instance-name", d.instance.ec2_metadata['tagset_name'])
56
+ end
57
+ end
58
+
59
+ test 'configure-classic' do
60
+ VCR.use_cassette('ec2-classic') do
61
+ c = %[
62
+ output_tag test
63
+ <record>
64
+ instance_id ${instance_id}
65
+ </record>
66
+ ]
67
+ d = create_driver(conf=c)
68
+
69
+ assert_equal("test", d.instance.output_tag)
70
+ assert_equal(nil, d.instance.aws_key_id)
71
+ assert_equal(nil, d.instance.aws_sec_key)
72
+
73
+ assert_equal("00:00:0A:AA:0A:0A", d.instance.ec2_metadata['mac'])
74
+ assert_equal(nil, d.instance.ec2_metadata['vpc_id'])
75
+ assert_equal(nil, d.instance.ec2_metadata['subnet_id'])
34
76
  end
35
77
  end
36
78
 
37
79
  test 'emit' do
38
- VCR.use_cassette('ec2') do
80
+ VCR.use_cassette('ec2-vpc') do
39
81
  d = create_driver
40
82
 
41
83
  d.run do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-ec2-metadata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.9
4
+ version: 0.0.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - SAKAMOTO Takumi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-12-13 00:00:00.000000000 Z
11
+ date: 2016-05-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -80,6 +80,20 @@ dependencies:
80
80
  - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: oj
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
83
97
  - !ruby/object:Gem::Dependency
84
98
  name: aws-sdk
85
99
  requirement: !ruby/object:Gem::Requirement
@@ -108,9 +122,13 @@ files:
108
122
  - README.md
109
123
  - Rakefile
110
124
  - fluent-plugin-ec2-metadata.gemspec
125
+ - lib/fluent/plugin/ec2_metadata.rb
126
+ - lib/fluent/plugin/filter_ec2_metadata.rb
111
127
  - lib/fluent/plugin/out_ec2_metadata.rb
112
- - test/cassettes/ec2.yml
128
+ - test/cassettes/ec2-classic.yml
129
+ - test/cassettes/ec2-vpc.yml
113
130
  - test/helper.rb
131
+ - test/plugin/test_filter_ec2_metadata.rb
114
132
  - test/plugin/test_out_ec2_metadata.rb
115
133
  homepage: https://github.com/takus/fluent-plugin-ec2-metadata
116
134
  licenses:
@@ -137,6 +155,8 @@ signing_key:
137
155
  specification_version: 4
138
156
  summary: Fluentd output plugin to add ec2 metadata fields to a event record
139
157
  test_files:
140
- - test/cassettes/ec2.yml
158
+ - test/cassettes/ec2-classic.yml
159
+ - test/cassettes/ec2-vpc.yml
141
160
  - test/helper.rb
161
+ - test/plugin/test_filter_ec2_metadata.rb
142
162
  - test/plugin/test_out_ec2_metadata.rb