fluent-plugin-ec2-metadata 0.0.9 → 0.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6a0cd615badf1c5b2a61d1911c8f51cbb902fe2c
4
- data.tar.gz: 57b1f0dc6af6988d3be54a8880e2d3f162615f7c
3
+ metadata.gz: dbc6f8c80499b4f5d1035976e73dbed3ab175983
4
+ data.tar.gz: dfc17e48483cc59aa02039c5355debc7a6ac4cc3
5
5
  SHA512:
6
- metadata.gz: 61e1f818cc83363166125539aca48585e6f0b34a454f423c8b559c95730f36b8adf8babf15360eb5983d21895b06631de3eef04feed6a09d1c97a74e4238fa38
7
- data.tar.gz: 6d4418525ae1c495dee94d07dda681166c65921287729eda4d9f752e8450d540aaa1945fcd3388735b7998a8b7c85b156fe6173e428eb99ae2557d48a6638da6
6
+ metadata.gz: 95e7f3548ba2b4d8e0dea2ddd86020dac3f7cfcccdbd0454ed874520a79140f0fb85bb9d821e575036e8ca67f9a9cca7f1665e36a5dd8506b5503fcfb11029c8
7
+ data.tar.gz: 27470dc7107882b981585d3ffd4ea5277e2463af28b713f8cee6390886222e4908ba83b7fbad0bb46f84acfbe754b644f69f706dac7a1cfafe5a6c824982c4fc
data/.travis.yml CHANGED
@@ -1,9 +1,7 @@
1
+ sudo: false
1
2
  language: ruby
2
3
  rvm:
3
4
  - 2.2
4
5
  - 2.1
5
6
  - 2.0.0
6
7
  - 1.9.3
7
- script:
8
- - bundle exec rake test
9
- - bundle exec rake build
data/README.md CHANGED
@@ -2,8 +2,10 @@
2
2
 
3
3
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-ec2-metadata.svg)](http://badge.fury.io/rb/fluent-plugin-ec2-metadata)
4
4
  [![Build Status](https://travis-ci.org/takus/fluent-plugin-ec2-metadata.svg?branch=master)](https://travis-ci.org/takus/fluent-plugin-ec2-metadata)
5
+ [![Dependency Status](https://gemnasium.com/takus/fluent-plugin-ec2-metadata.svg)](https://gemnasium.com/takus/fluent-plugin-ec2-metadata)
5
6
  [![Test Coverage](https://codeclimate.com/github/takus/fluent-plugin-ec2-metadata/badges/coverage.svg)](https://codeclimate.com/github/takus/fluent-plugin-ec2-metadata/coverage)
6
7
  [![Code Climate](https://codeclimate.com/github/takus/fluent-plugin-ec2-metadata/badges/gpa.svg)](https://codeclimate.com/github/takus/fluent-plugin-ec2-metadata)
8
+ [![Codacy Badge](https://api.codacy.com/project/badge/grade/16f6786edb554f1ea7462353808011d6)](https://www.codacy.com/app/takus/fluent-plugin-ec2-metadata)
7
9
 
8
10
  [Fluentd](http://fluentd.org) plugin to add ec2 metadata fields to a event record
9
11
 
@@ -29,28 +31,51 @@ Example:
29
31
  instance_type ${instance_type}
30
32
  az ${availability_zone}
31
33
  vpc_id ${vpc_id}
34
+ ami_id ${image_id}
35
+ account_id ${account_id}
32
36
  </record>
33
37
  </match>
34
38
 
35
39
  Assume following input is coming:
36
40
 
37
- ```js
41
+ ```
38
42
  foo.bar {"message":"hello ec2!"}
39
43
  ```
40
44
 
41
45
  then output becomes as below (indented):
42
46
 
43
- ```js
47
+ ```
44
48
  i-28b5ee77.foo.bar {
45
49
  "hostname" : "web0001",
46
50
  "instance_id" : "i-28b5ee77",
47
51
  "instance_type" : "m1.large",
48
52
  "az" : "us-west-1b",
49
53
  "vpc_id" : "vpc-25dab194",
54
+ "account_id" : "123456789",
55
+ "image_id" : "ami-123456",
50
56
  "message" : "hello ec2!"
51
57
  }
52
58
  ```
53
59
 
60
+ Or you can use filter version:
61
+
62
+ <filter foo.**>
63
+ type ec2_metadata
64
+
65
+ aws_key_id YOUR_AWS_KEY_ID
66
+ aws_sec_key YOUR_AWS_SECRET/KEY
67
+
68
+ <record>
69
+ hostname ${tagset_name}
70
+ instance_id ${instance_id}
71
+ instance_type ${instance_type}
72
+ az ${availability_zone}
73
+ vpc_id ${vpc_id}
74
+ ami_id ${image_id}
75
+ account_id ${account_id}
76
+ </record>
77
+ </filter>
78
+
54
79
  ### Placeholders
55
80
 
56
81
  The following placeholders are always available:
@@ -64,6 +89,8 @@ The following placeholders are always available:
64
89
  * ${mac} MAC address
65
90
  * ${vpc_id} vpc id
66
91
  * ${subnet_id} subnet id
92
+ * ${account_id} account id
93
+ * ${image_id} ami image id
67
94
 
68
95
  The followings are available when you define `aws_key_id` and `aws_sec_key`(or define IAM Policy):
69
96
 
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-ec2-metadata"
7
- spec.version = "0.0.9"
7
+ spec.version = "0.0.10"
8
8
  spec.authors = ["SAKAMOTO Takumi"]
9
9
  spec.email = ["takumi.saka@gmail.com"]
10
10
  spec.description = %q{Fluentd output plugin to add ec2 metadata fields to a event record}
@@ -22,5 +22,6 @@ Gem::Specification.new do |spec|
22
22
  spec.add_development_dependency "webmock"
23
23
  spec.add_development_dependency "test-unit", ">= 3.1.0"
24
24
  spec.add_runtime_dependency "fluentd"
25
+ spec.add_runtime_dependency "oj"
25
26
  spec.add_runtime_dependency "aws-sdk"
26
27
  end
@@ -0,0 +1,144 @@
1
+ module Fluent
2
+ module EC2Metadata
3
+
4
+ def initialize
5
+ super
6
+ require 'net/http'
7
+ require 'aws-sdk'
8
+ require 'oj'
9
+ end
10
+
11
+ def configure(conf)
12
+ super
13
+
14
+ # <record></record> directive
15
+ @map = {}
16
+ conf.elements.select { |element| element.name == 'record' }.each { |element|
17
+ element.each_pair { |k, v|
18
+ element.has_key?(k)
19
+ @map[k] = v
20
+ }
21
+ }
22
+
23
+ @placeholder_expander = PlaceholderExpander.new
24
+
25
+ set_metadata
26
+ set_tag
27
+ end
28
+
29
+ private
30
+
31
+ def set_metadata()
32
+ @ec2_metadata = {}
33
+
34
+
35
+ instance_identity = Oj.load(get_dynamic_data("instance-identity/document"))
36
+ @ec2_metadata['account_id'] = instance_identity["accountId"]
37
+ @ec2_metadata['image_id'] = instance_identity["imageId"]
38
+
39
+ @ec2_metadata['instance_id'] = get_metadata('instance-id')
40
+ @ec2_metadata['instance_type'] = get_metadata('instance-type')
41
+ @ec2_metadata['availability_zone'] = get_metadata('placement/availability-zone')
42
+ @ec2_metadata['region'] = @ec2_metadata['availability_zone'].chop
43
+ @ec2_metadata['mac'] = get_metadata('mac')
44
+ begin
45
+ @ec2_metadata['vpc_id'] = get_metadata("network/interfaces/macs/#{@ec2_metadata['mac']}/vpc-id")
46
+ rescue
47
+ @ec2_metadata['vpc_id'] = nil
48
+ $log.info "ec2-metadata: 'vpc_id' is undefined #{@ec2_metadata['instance_id']} is not in VPC}"
49
+ end
50
+ begin
51
+ @ec2_metadata['subnet_id'] = get_metadata("network/interfaces/macs/#{@ec2_metadata['mac']}/subnet-id")
52
+ rescue
53
+ @ec2_metadata['subnet_id'] = nil
54
+ $log.info "ec2-metadata: 'subnet_id' is undefined because #{@ec2_metadata['instance_id']} is not in VPC}"
55
+ end
56
+ end
57
+
58
+ def get_dynamic_data(f)
59
+ res = Net::HTTP.get_response("169.254.169.254", "/latest/dynamic/#{f}")
60
+ raise Fluent::ConfigError, "ec2-dynamic-data: failed to get #{f}" unless res.is_a?(Net::HTTPSuccess)
61
+ res.body
62
+ end
63
+
64
+ def get_metadata(f)
65
+ res = Net::HTTP.get_response("169.254.169.254", "/latest/meta-data/#{f}")
66
+ raise Fluent::ConfigError, "ec2-metadata: failed to get #{f}" unless res.is_a?(Net::HTTPSuccess)
67
+ res.body
68
+ end
69
+
70
+ def set_tag()
71
+ if @map.values.any? { |v| v.match(/^\${tagset_/) } || @output_tag =~ /\${tagset_/
72
+
73
+ if @aws_key_id and @aws_sec_key
74
+ ec2 = Aws::EC2::Client.new(
75
+ region: @ec2_metadata['region'],
76
+ access_key_id: @aws_key_id,
77
+ secret_access_key: @aws_sec_key,
78
+ )
79
+ else
80
+ ec2 = Aws::EC2::Client.new(
81
+ region: @ec2_metadata['region'],
82
+ )
83
+ end
84
+
85
+ response = ec2.describe_instances({ :instance_ids => [@ec2_metadata['instance_id']] })
86
+ instance = response.reservations[0].instances[0]
87
+ raise Fluent::ConfigError, "ec2-metadata: failed to get instance data #{response.pretty_inspect}" if instance.nil?
88
+
89
+ instance.tags.each { |tag|
90
+ @ec2_metadata["tagset_#{tag.key.downcase}"] = tag.value
91
+ }
92
+ end
93
+ end
94
+
95
+ def modify_record(record, tag, tag_parts)
96
+ @placeholder_expander.prepare_placeholders(record, tag, tag_parts, @ec2_metadata)
97
+ new_record = record.dup
98
+ @map.each_pair { |k, v| new_record[k] = @placeholder_expander.expand(v) }
99
+ new_record
100
+ end
101
+
102
+ def modify(output_tag, record, tag, tag_parts)
103
+ @placeholder_expander.prepare_placeholders(record, tag, tag_parts, @ec2_metadata)
104
+
105
+ new_tag = @placeholder_expander.expand(output_tag)
106
+
107
+ new_record = record.dup
108
+ @map.each_pair { |k, v| new_record[k] = @placeholder_expander.expand(v) }
109
+
110
+ [new_tag, new_record]
111
+ end
112
+
113
+ class PlaceholderExpander
114
+ # referenced https://github.com/fluent/fluent-plugin-rewrite-tag-filter
115
+ # referenced https://github.com/sonots/fluent-plugin-record-reformer
116
+ attr_reader :placeholders
117
+
118
+ def prepare_placeholders(_record, tag, tag_parts, ec2_metadata)
119
+ placeholders = {
120
+ '${tag}' => tag,
121
+ }
122
+
123
+ size = tag_parts.size
124
+ tag_parts.each_with_index { |t, idx|
125
+ placeholders.store("${tag_parts[#{idx}]}", t)
126
+ placeholders.store("${tag_parts[#{idx-size}]}", t) # support tag_parts[-1]
127
+ }
128
+
129
+ ec2_metadata.each { |k, v|
130
+ placeholders.store("${#{k}}", v)
131
+ }
132
+
133
+ @placeholders = placeholders
134
+ end
135
+
136
+ def expand(str)
137
+ str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
138
+ $log.warn "ec2-metadata: unknown placeholder `#{$1}` found in a tag `#{@placeholders['${tag}']}`" unless @placeholders.include?($1)
139
+ @placeholders[$1]
140
+ }
141
+ end
142
+ end
143
+ end
144
+ end
@@ -0,0 +1,26 @@
1
+ require_relative 'ec2_metadata'
2
+
3
+ module Fluent
4
+ class EC2MetadataFilter < Filter
5
+ include Fluent::EC2Metadata
6
+
7
+ Fluent::Plugin.register_filter('ec2_metadata', self)
8
+
9
+ # Define `router` method of v0.12 to support v0.10 or earlier
10
+ unless method_defined?(:router)
11
+ define_method("router") { Fluent::Engine }
12
+ end
13
+
14
+ config_param :aws_key_id, :string, :default => ENV['AWS_ACCESS_KEY_ID'], :secret => true
15
+ config_param :aws_sec_key, :string, :default => ENV['AWS_SECRET_ACCESS_KEY'], :secret => true
16
+
17
+ attr_reader :ec2_metadata
18
+
19
+ def filter(tag, time, record)
20
+ tag_parts = tag.split('.')
21
+ modify_record(record, tag, tag_parts)
22
+ rescue => e
23
+ $log.warn "ec2-metadata: #{e.class} #{e.message} #{e.backtrace.join(', ')}"
24
+ end
25
+ end
26
+ end
@@ -1,5 +1,9 @@
1
+ require_relative 'ec2_metadata'
2
+
1
3
  module Fluent
2
4
  class EC2MetadataOutput < Output
5
+ include EC2Metadata
6
+
3
7
  Fluent::Plugin.register_output('ec2_metadata', self)
4
8
 
5
9
  # Define `router` method of v0.12 to support v0.10 or earlier
@@ -7,74 +11,11 @@ module Fluent
7
11
  define_method("router") { Fluent::Engine }
8
12
  end
9
13
 
10
- def initialize
11
- super
12
- require 'net/http'
13
- end
14
-
15
14
  config_param :output_tag, :string
16
15
  config_param :aws_key_id, :string, :default => ENV['AWS_ACCESS_KEY_ID'], :secret => true
17
16
  config_param :aws_sec_key, :string, :default => ENV['AWS_SECRET_ACCESS_KEY'], :secret => true
18
17
 
19
- def configure(conf)
20
- super
21
-
22
- # <record></record> directive
23
- @map = {}
24
- conf.elements.select { |element| element.name == 'record' }.each { |element|
25
- element.each_pair { |k, v|
26
- element.has_key?(k)
27
- @map[k] = v
28
- }
29
- }
30
-
31
- @placeholder_expander = PlaceholderExpander.new
32
-
33
- # get ec2 metadata
34
- @ec2_metadata = {}
35
- @ec2_metadata['instance_id'] = get_metadata('instance-id')
36
- @ec2_metadata['instance_type'] = get_metadata('instance-type')
37
- @ec2_metadata['availability_zone'] = get_metadata('placement/availability-zone')
38
- @ec2_metadata['region'] = @ec2_metadata['availability_zone'].chop
39
- @ec2_metadata['mac'] = get_metadata('mac')
40
- begin
41
- @ec2_metadata['vpc_id'] = get_metadata("network/interfaces/macs/#{@ec2_metadata['mac']}/vpc-id")
42
- rescue
43
- @ec2_metadata['vpc_id'] = nil
44
- $log.info "ec2-metadata: 'vpc_id' is undefined #{@ec2_metadata['instance_id']} is not in VPC}"
45
- end
46
- begin
47
- @ec2_metadata['subnet_id'] = get_metadata("network/interfaces/macs/#{@ec2_metadata['mac']}/subnet-id")
48
- rescue
49
- @ec2_metadata['subnet_id'] = nil
50
- $log.info "ec2-metadata: 'subnet_id' is undefined because #{@ec2_metadata['instance_id']} is not in VPC}"
51
- end
52
-
53
- # get tags
54
- if @map.values.any? { |v| v.match(/^\${tagset_/) } || @output_tag =~ /\${tagset_/
55
- require 'aws-sdk'
56
-
57
- if @aws_key_id and @aws_sec_key then
58
- ec2 = Aws::EC2::Client.new(
59
- region: @ec2_metadata['region'],
60
- access_key_id: @aws_key_id,
61
- secret_access_key: @aws_sec_key,
62
- )
63
- else
64
- ec2 = Aws::EC2::Client.new(
65
- region: @ec2_metadata['region'],
66
- )
67
- end
68
-
69
- response = ec2.describe_instances({ :instance_ids => [@ec2_metadata['instance_id']] })
70
- instance = response.reservations[0].instances[0]
71
- raise Fluent::ConfigError, "ec2-metadata: failed to get instance data #{response.pretty_inspect}" if instance.nil?
72
-
73
- instance.tags.each { |tag|
74
- @ec2_metadata["tagset_#{tag.key.downcase}"] = tag.value
75
- }
76
- end
77
- end
18
+ attr_reader :ec2_metadata
78
19
 
79
20
  def emit(tag, es, chain)
80
21
  tag_parts = tag.split('.')
@@ -86,56 +27,5 @@ module Fluent
86
27
  rescue => e
87
28
  $log.warn "ec2-metadata: #{e.class} #{e.message} #{e.backtrace.join(', ')}"
88
29
  end
89
-
90
- private
91
-
92
- def get_metadata(f)
93
- res = Net::HTTP.get_response("169.254.169.254", "/latest/meta-data/#{f}")
94
- raise Fluent::ConfigError, "ec2-metadata: failed to get #{f}" unless res.is_a?(Net::HTTPSuccess)
95
- res.body
96
- end
97
-
98
- def modify(output_tag, record, tag, tag_parts)
99
- @placeholder_expander.prepare_placeholders(record, tag, tag_parts, @ec2_metadata)
100
-
101
- new_tag = @placeholder_expander.expand(output_tag)
102
-
103
- new_record = record.dup
104
- @map.each_pair { |k, v| new_record[k] = @placeholder_expander.expand(v) }
105
-
106
- [new_tag, new_record]
107
- end
108
-
109
- class PlaceholderExpander
110
- # referenced https://github.com/fluent/fluent-plugin-rewrite-tag-filter
111
- # referenced https://github.com/sonots/fluent-plugin-record-reformer
112
- attr_reader :placeholders
113
-
114
- def prepare_placeholders(record, tag, tag_parts, ec2_metadata)
115
- placeholders = {
116
- '${tag}' => tag,
117
- }
118
-
119
- size = tag_parts.size
120
- tag_parts.each_with_index { |t, idx|
121
- placeholders.store("${tag_parts[#{idx}]}", t)
122
- placeholders.store("${tag_parts[#{idx-size}]}", t) # support tag_parts[-1]
123
- }
124
-
125
- ec2_metadata.each { |k, v|
126
- placeholders.store("${#{k}}", v)
127
- }
128
-
129
- @placeholders = placeholders
130
- end
131
-
132
- def expand(str)
133
- str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
134
- $log.warn "ec2-metadata: unknown placeholder `#{$1}` found in a tag `#{@placeholders['${tag}']}`" unless @placeholders.include?($1)
135
- @placeholders[$1]
136
- }
137
- end
138
- end
139
-
140
30
  end
141
31
  end
@@ -0,0 +1,302 @@
1
+ ---
2
+ http_interactions:
3
+ - request:
4
+ method: get
5
+ uri: http://169.254.169.254/latest/meta-data/instance-id
6
+ body:
7
+ encoding: US-ASCII
8
+ string: ''
9
+ headers:
10
+ Accept-Encoding:
11
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
+ Accept:
13
+ - '*/*'
14
+ User-Agent:
15
+ - Ruby
16
+ response:
17
+ status:
18
+ code: 200
19
+ message: OK
20
+ headers:
21
+ Content-Type:
22
+ - text/plain
23
+ Accept-Ranges:
24
+ - bytes
25
+ Etag:
26
+ - '"1971254081"'
27
+ Last-Modified:
28
+ - Thu, 02 Jul 2015 12:49:17 GMT
29
+ Content-Length:
30
+ - '10'
31
+ Connection:
32
+ - close
33
+ Date:
34
+ - Sun, 13 Dec 2015 06:37:00 GMT
35
+ Server:
36
+ - EC2ws
37
+ body:
38
+ encoding: UTF-8
39
+ string: i-0c0c0000
40
+ http_version:
41
+ recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
42
+ - request:
43
+ method: get
44
+ uri: http://169.254.169.254/latest/meta-data/instance-type
45
+ body:
46
+ encoding: US-ASCII
47
+ string: ''
48
+ headers:
49
+ Accept-Encoding:
50
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
51
+ Accept:
52
+ - '*/*'
53
+ User-Agent:
54
+ - Ruby
55
+ response:
56
+ status:
57
+ code: 200
58
+ message: OK
59
+ headers:
60
+ Content-Type:
61
+ - text/plain
62
+ Accept-Ranges:
63
+ - bytes
64
+ Etag:
65
+ - '"765393446"'
66
+ Last-Modified:
67
+ - Thu, 02 Jul 2015 12:49:17 GMT
68
+ Content-Length:
69
+ - '8'
70
+ Connection:
71
+ - close
72
+ Date:
73
+ - Sun, 13 Dec 2015 06:37:00 GMT
74
+ Server:
75
+ - EC2ws
76
+ body:
77
+ encoding: UTF-8
78
+ string: m3.large
79
+ http_version:
80
+ recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
81
+ - request:
82
+ method: get
83
+ uri: http://169.254.169.254/latest/meta-data/placement/availability-zone
84
+ body:
85
+ encoding: US-ASCII
86
+ string: ''
87
+ headers:
88
+ Accept-Encoding:
89
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
90
+ Accept:
91
+ - '*/*'
92
+ User-Agent:
93
+ - Ruby
94
+ response:
95
+ status:
96
+ code: 200
97
+ message: OK
98
+ headers:
99
+ Content-Type:
100
+ - text/plain
101
+ Accept-Ranges:
102
+ - bytes
103
+ Etag:
104
+ - '"1870317889"'
105
+ Last-Modified:
106
+ - Thu, 02 Jul 2015 12:49:17 GMT
107
+ Content-Length:
108
+ - '15'
109
+ Connection:
110
+ - close
111
+ Date:
112
+ - Sun, 13 Dec 2015 06:37:00 GMT
113
+ Server:
114
+ - EC2ws
115
+ body:
116
+ encoding: UTF-8
117
+ string: ap-northeast-1b
118
+ http_version:
119
+ recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
120
+ - request:
121
+ method: get
122
+ uri: http://169.254.169.254/latest/meta-data/mac
123
+ body:
124
+ encoding: US-ASCII
125
+ string: ''
126
+ headers:
127
+ Accept-Encoding:
128
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
129
+ Accept:
130
+ - '*/*'
131
+ User-Agent:
132
+ - Ruby
133
+ response:
134
+ status:
135
+ code: 200
136
+ message: OK
137
+ headers:
138
+ Content-Type:
139
+ - text/plain
140
+ Accept-Ranges:
141
+ - bytes
142
+ Etag:
143
+ - '"1925384845"'
144
+ Last-Modified:
145
+ - Fri, 27 Feb 2015 04:49:13 GMT
146
+ Content-Length:
147
+ - '17'
148
+ Connection:
149
+ - close
150
+ Date:
151
+ - Wed, 16 Dec 2015 17:11:22 GMT
152
+ Server:
153
+ - EC2ws
154
+ body:
155
+ encoding: UTF-8
156
+ string: 00:00:0A:AA:0A:0A
157
+ http_version:
158
+ recorded_at: Wed, 16 Dec 2015 17:11:22 GMT
159
+ - request:
160
+ method: get
161
+ uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:00:0A:AA:0A:0A/vpc-id
162
+ body:
163
+ encoding: US-ASCII
164
+ string: ''
165
+ headers:
166
+ Accept-Encoding:
167
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
168
+ Accept:
169
+ - '*/*'
170
+ User-Agent:
171
+ - Ruby
172
+ response:
173
+ status:
174
+ code: 404
175
+ message: Not Found
176
+ headers:
177
+ Content-Type:
178
+ - text/html
179
+ Content-Length:
180
+ - '345'
181
+ Connection:
182
+ - close
183
+ Date:
184
+ - Wed, 16 Dec 2015 17:11:22 GMT
185
+ Server:
186
+ - EC2ws
187
+ body:
188
+ encoding: UTF-8
189
+ string: |
190
+ <?xml version="1.0" encoding="iso-8859-1"?>
191
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
192
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
193
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
194
+ <head>
195
+ <title>404 - Not Found</title>
196
+ </head>
197
+ <body>
198
+ <h1>404 - Not Found</h1>
199
+ </body>
200
+ </html>
201
+ http_version:
202
+ recorded_at: Wed, 16 Dec 2015 17:11:22 GMT
203
+ - request:
204
+ method: get
205
+ uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:00:0A:AA:0A:0A/subnet-id
206
+ body:
207
+ encoding: US-ASCII
208
+ string: ''
209
+ headers:
210
+ Accept-Encoding:
211
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
212
+ Accept:
213
+ - '*/*'
214
+ User-Agent:
215
+ - Ruby
216
+ response:
217
+ status:
218
+ code: 404
219
+ message: Not Found
220
+ headers:
221
+ Content-Type:
222
+ - text/html
223
+ Content-Length:
224
+ - '345'
225
+ Connection:
226
+ - close
227
+ Date:
228
+ - Wed, 16 Dec 2015 17:11:22 GMT
229
+ Server:
230
+ - EC2ws
231
+ body:
232
+ encoding: UTF-8
233
+ string: |
234
+ <?xml version="1.0" encoding="iso-8859-1"?>
235
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
236
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
237
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
238
+ <head>
239
+ <title>404 - Not Found</title>
240
+ </head>
241
+ <body>
242
+ <h1>404 - Not Found</h1>
243
+ </body>
244
+ </html>
245
+ http_version:
246
+ recorded_at: Wed, 16 Dec 2015 17:11:22 GMT
247
+ - request:
248
+ method: get
249
+ uri: http://169.254.169.254/latest/dynamic/instance-identity/document
250
+ body:
251
+ encoding: US-ASCII
252
+ string: ''
253
+ headers:
254
+ Accept-Encoding:
255
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
256
+ Accept:
257
+ - '*/*'
258
+ User-Agent:
259
+ - Ruby
260
+ response:
261
+ status:
262
+ code: 200
263
+ message: OK
264
+ headers:
265
+ Content-Type:
266
+ - text/plain
267
+ Accept-Ranges:
268
+ - bytes
269
+ Etag:
270
+ - '"2871764318"'
271
+ Last-Modified:
272
+ - Wed, 11 May 2016 20:52:55 GMT
273
+ Content-Length:
274
+ - '422'
275
+ Connection:
276
+ - close
277
+ Date:
278
+ - Wed, 11 May 2016 21:05:40 GMT
279
+ Server:
280
+ - EC2ws
281
+ body:
282
+ encoding: UTF-8
283
+ string: |-
284
+ {
285
+ "privateIp" : "172.31.1.232",
286
+ "devpayProductCodes" : null,
287
+ "availabilityZone" : "us-west-1c",
288
+ "version" : "2010-08-31",
289
+ "instanceId" : "i-123456",
290
+ "billingProducts" : null,
291
+ "instanceType" : "t2.micro",
292
+ "accountId" : "123456789",
293
+ "imageId" : "ami-123456",
294
+ "pendingTime" : "2016-05-11T20:52:25Z",
295
+ "architecture" : "x86_64",
296
+ "kernelId" : null,
297
+ "ramdiskId" : null,
298
+ "region" : "us-west-1"
299
+ }
300
+ http_version:
301
+ recorded_at: Wed, 11 May 2016 21:05:40 GMT
302
+ recorded_with: VCR 3.0.1
@@ -10,7 +10,7 @@ http_interactions:
10
10
  Accept-Encoding:
11
11
  - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
12
12
  Accept:
13
- - "*/*"
13
+ - '*/*'
14
14
  User-Agent:
15
15
  - Ruby
16
16
  response:
@@ -37,7 +37,7 @@ http_interactions:
37
37
  body:
38
38
  encoding: UTF-8
39
39
  string: i-0c0c0000
40
- http_version:
40
+ http_version:
41
41
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
42
42
  - request:
43
43
  method: get
@@ -49,7 +49,7 @@ http_interactions:
49
49
  Accept-Encoding:
50
50
  - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
51
51
  Accept:
52
- - "*/*"
52
+ - '*/*'
53
53
  User-Agent:
54
54
  - Ruby
55
55
  response:
@@ -76,7 +76,7 @@ http_interactions:
76
76
  body:
77
77
  encoding: UTF-8
78
78
  string: m3.large
79
- http_version:
79
+ http_version:
80
80
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
81
81
  - request:
82
82
  method: get
@@ -88,7 +88,7 @@ http_interactions:
88
88
  Accept-Encoding:
89
89
  - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
90
90
  Accept:
91
- - "*/*"
91
+ - '*/*'
92
92
  User-Agent:
93
93
  - Ruby
94
94
  response:
@@ -115,7 +115,7 @@ http_interactions:
115
115
  body:
116
116
  encoding: UTF-8
117
117
  string: ap-northeast-1b
118
- http_version:
118
+ http_version:
119
119
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
120
120
  - request:
121
121
  method: get
@@ -127,7 +127,7 @@ http_interactions:
127
127
  Accept-Encoding:
128
128
  - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
129
129
  Accept:
130
- - "*/*"
130
+ - '*/*'
131
131
  User-Agent:
132
132
  - Ruby
133
133
  response:
@@ -153,12 +153,12 @@ http_interactions:
153
153
  - EC2ws
154
154
  body:
155
155
  encoding: UTF-8
156
- string: 00:c0:00:0c:cc:00
157
- http_version:
156
+ string: 00:A0:00:0A:AA:00
157
+ http_version:
158
158
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
159
159
  - request:
160
160
  method: get
161
- uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:c0:00:0c:cc:00/vpc-id
161
+ uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:A0:00:0A:AA:00/vpc-id
162
162
  body:
163
163
  encoding: US-ASCII
164
164
  string: ''
@@ -166,7 +166,7 @@ http_interactions:
166
166
  Accept-Encoding:
167
167
  - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
168
168
  Accept:
169
- - "*/*"
169
+ - '*/*'
170
170
  User-Agent:
171
171
  - Ruby
172
172
  response:
@@ -193,11 +193,11 @@ http_interactions:
193
193
  body:
194
194
  encoding: UTF-8
195
195
  string: vpc-00000000
196
- http_version:
196
+ http_version:
197
197
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
198
198
  - request:
199
199
  method: get
200
- uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:c0:00:0c:cc:00/subnet-id
200
+ uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:A0:00:0A:AA:00/subnet-id
201
201
  body:
202
202
  encoding: US-ASCII
203
203
  string: ''
@@ -205,7 +205,7 @@ http_interactions:
205
205
  Accept-Encoding:
206
206
  - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
207
207
  Accept:
208
- - "*/*"
208
+ - '*/*'
209
209
  User-Agent:
210
210
  - Ruby
211
211
  response:
@@ -232,8 +232,96 @@ http_interactions:
232
232
  body:
233
233
  encoding: UTF-8
234
234
  string: subnet-00000000
235
- http_version:
235
+ http_version:
236
236
  recorded_at: Sun, 13 Dec 2015 06:37:00 GMT
237
+ - request:
238
+ method: get
239
+ uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:00:0A:AA:0A:0A/vpc-id
240
+ body:
241
+ encoding: US-ASCII
242
+ string: ''
243
+ headers:
244
+ Accept-Encoding:
245
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
246
+ Accept:
247
+ - '*/*'
248
+ User-Agent:
249
+ - Ruby
250
+ response:
251
+ status:
252
+ code: 404
253
+ message: Not Found
254
+ headers:
255
+ Content-Type:
256
+ - text/html
257
+ Content-Length:
258
+ - '345'
259
+ Connection:
260
+ - close
261
+ Date:
262
+ - Wed, 16 Dec 2015 17:11:22 GMT
263
+ Server:
264
+ - EC2ws
265
+ body:
266
+ encoding: UTF-8
267
+ string: |
268
+ <?xml version="1.0" encoding="iso-8859-1"?>
269
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
270
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
271
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
272
+ <head>
273
+ <title>404 - Not Found</title>
274
+ </head>
275
+ <body>
276
+ <h1>404 - Not Found</h1>
277
+ </body>
278
+ </html>
279
+ http_version:
280
+ recorded_at: Wed, 16 Dec 2015 17:11:22 GMT
281
+ - request:
282
+ method: get
283
+ uri: http://169.254.169.254/latest/meta-data/network/interfaces/macs/00:00:0A:AA:0A:0A/subnet-id
284
+ body:
285
+ encoding: US-ASCII
286
+ string: ''
287
+ headers:
288
+ Accept-Encoding:
289
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
290
+ Accept:
291
+ - '*/*'
292
+ User-Agent:
293
+ - Ruby
294
+ response:
295
+ status:
296
+ code: 404
297
+ message: Not Found
298
+ headers:
299
+ Content-Type:
300
+ - text/html
301
+ Content-Length:
302
+ - '345'
303
+ Connection:
304
+ - close
305
+ Date:
306
+ - Wed, 16 Dec 2015 17:11:22 GMT
307
+ Server:
308
+ - EC2ws
309
+ body:
310
+ encoding: UTF-8
311
+ string: |
312
+ <?xml version="1.0" encoding="iso-8859-1"?>
313
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
314
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
315
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
316
+ <head>
317
+ <title>404 - Not Found</title>
318
+ </head>
319
+ <body>
320
+ <h1>404 - Not Found</h1>
321
+ </body>
322
+ </html>
323
+ http_version:
324
+ recorded_at: Wed, 16 Dec 2015 17:11:22 GMT
237
325
  - request:
238
326
  method: post
239
327
  uri: https://ec2.ap-northeast-1.amazonaws.com/
@@ -254,7 +342,7 @@ http_interactions:
254
342
  Content-Length:
255
343
  - '67'
256
344
  Accept:
257
- - "*/*"
345
+ - '*/*'
258
346
  response:
259
347
  status:
260
348
  code: 200
@@ -295,6 +383,61 @@ http_interactions:
295
383
  </item>
296
384
  </reservationSet>
297
385
  </DescribeInstancesResponse>
298
- http_version:
386
+ http_version:
299
387
  recorded_at: Sun, 13 Dec 2015 10:06:07 GMT
300
- recorded_with: VCR 3.0.0
388
+ - request:
389
+ method: get
390
+ uri: http://169.254.169.254/latest/dynamic/instance-identity/document
391
+ body:
392
+ encoding: US-ASCII
393
+ string: ''
394
+ headers:
395
+ Accept-Encoding:
396
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
397
+ Accept:
398
+ - '*/*'
399
+ User-Agent:
400
+ - Ruby
401
+ response:
402
+ status:
403
+ code: 200
404
+ message: OK
405
+ headers:
406
+ Content-Type:
407
+ - text/plain
408
+ Accept-Ranges:
409
+ - bytes
410
+ Etag:
411
+ - '"2871764318"'
412
+ Last-Modified:
413
+ - Wed, 11 May 2016 20:52:55 GMT
414
+ Content-Length:
415
+ - '422'
416
+ Connection:
417
+ - close
418
+ Date:
419
+ - Wed, 11 May 2016 21:05:40 GMT
420
+ Server:
421
+ - EC2ws
422
+ body:
423
+ encoding: UTF-8
424
+ string: |-
425
+ {
426
+ "privateIp" : "172.31.1.232",
427
+ "devpayProductCodes" : null,
428
+ "availabilityZone" : "us-west-1c",
429
+ "version" : "2010-08-31",
430
+ "instanceId" : "i-123456",
431
+ "billingProducts" : null,
432
+ "instanceType" : "t2.micro",
433
+ "accountId" : "123456789",
434
+ "imageId" : "ami-123456",
435
+ "pendingTime" : "2016-05-11T20:52:25Z",
436
+ "architecture" : "x86_64",
437
+ "kernelId" : null,
438
+ "ramdiskId" : null,
439
+ "region" : "us-west-1"
440
+ }
441
+ http_version:
442
+ recorded_at: Wed, 11 May 2016 21:05:40 GMT
443
+ recorded_with: VCR 3.0.1
data/test/helper.rb CHANGED
@@ -34,6 +34,7 @@ VCR.configure do |config|
34
34
  end
35
35
 
36
36
  require 'fluent/plugin/out_ec2_metadata'
37
+ require 'fluent/plugin/filter_ec2_metadata'
37
38
 
38
39
  class Test::Unit::TestCase
39
40
  end
@@ -0,0 +1,73 @@
1
+ require 'helper'
2
+
3
+ require 'webmock/test_unit'
4
+ WebMock.disable_net_connect!
5
+
6
+ class EC2MetadataFilterTest < Test::Unit::TestCase
7
+
8
+ CONFIG = %[
9
+ aws_key_id aws_key
10
+ aws_sec_key aws_sec
11
+ <record>
12
+ name ${tagset_name}
13
+ instance_id ${instance_id}
14
+ az ${availability_zone}
15
+ </record>
16
+ ]
17
+
18
+ def setup
19
+ Fluent::Test.setup
20
+ @time = Fluent::Engine.now
21
+ end
22
+
23
+ def create_driver(conf=CONFIG, tag='test')
24
+ Fluent::Test::FilterTestDriver.new(Fluent::EC2MetadataFilter, tag).configure(conf)
25
+ end
26
+
27
+ test 'configure-vpc' do
28
+ VCR.use_cassette('ec2-vpc') do
29
+ c = %[
30
+ aws_key_id aws_key
31
+ aws_sec_key aws_sec
32
+ <record>
33
+ name ${tagset_name}
34
+ </record>
35
+ ]
36
+ d = create_driver(conf=c)
37
+
38
+ assert_equal("aws_key", d.instance.aws_key_id)
39
+ assert_equal("aws_sec", d.instance.aws_sec_key)
40
+
41
+ assert_equal("ami-123456", d.instance.ec2_metadata['image_id'])
42
+ assert_equal("123456789", d.instance.ec2_metadata['account_id'])
43
+
44
+ assert_equal("i-0c0c0000", d.instance.ec2_metadata['instance_id'])
45
+ assert_equal("m3.large", d.instance.ec2_metadata['instance_type'])
46
+ assert_equal("ap-northeast-1", d.instance.ec2_metadata['region'])
47
+ assert_equal("ap-northeast-1b", d.instance.ec2_metadata['availability_zone'])
48
+ assert_equal("00:A0:00:0A:AA:00", d.instance.ec2_metadata['mac'])
49
+ assert_equal("vpc-00000000", d.instance.ec2_metadata['vpc_id'])
50
+ assert_equal("subnet-00000000", d.instance.ec2_metadata['subnet_id'])
51
+
52
+ assert_equal("instance-name", d.instance.ec2_metadata['tagset_name'])
53
+ end
54
+ end
55
+
56
+ test 'configure-classic' do
57
+ VCR.use_cassette('ec2-classic') do
58
+ c = %[
59
+ <record>
60
+ instance_id ${instance_id}
61
+ </record>
62
+ ]
63
+ d = create_driver(conf=c)
64
+
65
+ assert_equal(nil, d.instance.aws_key_id)
66
+ assert_equal(nil, d.instance.aws_sec_key)
67
+
68
+ assert_equal("00:00:0A:AA:0A:0A", d.instance.ec2_metadata['mac'])
69
+ assert_equal(nil, d.instance.ec2_metadata['vpc_id'])
70
+ assert_equal(nil, d.instance.ec2_metadata['subnet_id'])
71
+ end
72
+ end
73
+ end
@@ -25,17 +25,59 @@ class EC2MetadataOutputTest < Test::Unit::TestCase
25
25
  Fluent::Test::OutputTestDriver.new(Fluent::EC2MetadataOutput, tag).configure(conf)
26
26
  end
27
27
 
28
- test 'configure' do
29
- VCR.use_cassette('ec2') do
30
- d = create_driver
28
+ test 'configure-vpc' do
29
+ VCR.use_cassette('ec2-vpc') do
30
+ c = %[
31
+ output_tag ${instance_id}.${tag}
32
+ aws_key_id aws_key
33
+ aws_sec_key aws_sec
34
+ <record>
35
+ name ${tagset_name}
36
+ </record>
37
+ ]
38
+ d = create_driver(conf=c)
39
+
31
40
  assert_equal("${instance_id}.${tag}", d.instance.output_tag)
32
41
  assert_equal("aws_key", d.instance.aws_key_id)
33
42
  assert_equal("aws_sec", d.instance.aws_sec_key)
43
+
44
+ assert_equal("ami-123456", d.instance.ec2_metadata['image_id'])
45
+ assert_equal("123456789", d.instance.ec2_metadata['account_id'])
46
+
47
+ assert_equal("i-0c0c0000", d.instance.ec2_metadata['instance_id'])
48
+ assert_equal("m3.large", d.instance.ec2_metadata['instance_type'])
49
+ assert_equal("ap-northeast-1", d.instance.ec2_metadata['region'])
50
+ assert_equal("ap-northeast-1b", d.instance.ec2_metadata['availability_zone'])
51
+ assert_equal("00:A0:00:0A:AA:00", d.instance.ec2_metadata['mac'])
52
+ assert_equal("vpc-00000000", d.instance.ec2_metadata['vpc_id'])
53
+ assert_equal("subnet-00000000", d.instance.ec2_metadata['subnet_id'])
54
+
55
+ assert_equal("instance-name", d.instance.ec2_metadata['tagset_name'])
56
+ end
57
+ end
58
+
59
+ test 'configure-classic' do
60
+ VCR.use_cassette('ec2-classic') do
61
+ c = %[
62
+ output_tag test
63
+ <record>
64
+ instance_id ${instance_id}
65
+ </record>
66
+ ]
67
+ d = create_driver(conf=c)
68
+
69
+ assert_equal("test", d.instance.output_tag)
70
+ assert_equal(nil, d.instance.aws_key_id)
71
+ assert_equal(nil, d.instance.aws_sec_key)
72
+
73
+ assert_equal("00:00:0A:AA:0A:0A", d.instance.ec2_metadata['mac'])
74
+ assert_equal(nil, d.instance.ec2_metadata['vpc_id'])
75
+ assert_equal(nil, d.instance.ec2_metadata['subnet_id'])
34
76
  end
35
77
  end
36
78
 
37
79
  test 'emit' do
38
- VCR.use_cassette('ec2') do
80
+ VCR.use_cassette('ec2-vpc') do
39
81
  d = create_driver
40
82
 
41
83
  d.run do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-ec2-metadata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.9
4
+ version: 0.0.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - SAKAMOTO Takumi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-12-13 00:00:00.000000000 Z
11
+ date: 2016-05-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -80,6 +80,20 @@ dependencies:
80
80
  - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: oj
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
83
97
  - !ruby/object:Gem::Dependency
84
98
  name: aws-sdk
85
99
  requirement: !ruby/object:Gem::Requirement
@@ -108,9 +122,13 @@ files:
108
122
  - README.md
109
123
  - Rakefile
110
124
  - fluent-plugin-ec2-metadata.gemspec
125
+ - lib/fluent/plugin/ec2_metadata.rb
126
+ - lib/fluent/plugin/filter_ec2_metadata.rb
111
127
  - lib/fluent/plugin/out_ec2_metadata.rb
112
- - test/cassettes/ec2.yml
128
+ - test/cassettes/ec2-classic.yml
129
+ - test/cassettes/ec2-vpc.yml
113
130
  - test/helper.rb
131
+ - test/plugin/test_filter_ec2_metadata.rb
114
132
  - test/plugin/test_out_ec2_metadata.rb
115
133
  homepage: https://github.com/takus/fluent-plugin-ec2-metadata
116
134
  licenses:
@@ -137,6 +155,8 @@ signing_key:
137
155
  specification_version: 4
138
156
  summary: Fluentd output plugin to add ec2 metadata fields to a event record
139
157
  test_files:
140
- - test/cassettes/ec2.yml
158
+ - test/cassettes/ec2-classic.yml
159
+ - test/cassettes/ec2-vpc.yml
141
160
  - test/helper.rb
161
+ - test/plugin/test_filter_ec2_metadata.rb
142
162
  - test/plugin/test_out_ec2_metadata.rb