fluent-plugin-ec2-metadata 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 622067156fe604a54288fc1c8a3322c96a1b2927
-  data.tar.gz: efb9a9f3c6415ae0899d8d90e60811ac0bb67918
+  metadata.gz: e775b4cdc0695590d02f3933c3ae964da6075b2a
+  data.tar.gz: c00a2053b3cefd22f1526486eb380a5807a4b05e
 SHA512:
-  metadata.gz: cbeb6506a22bb9e54ba7e27910d6c7f018ccd9f95f4166e32396aea2299117d5cdd85b4db8f9d7942cd2477a6dbfa7108aad8445d8fe35a698c0a662977217ee
-  data.tar.gz: bfc3acf3973c78dfb32a4a16602283546f605a941a24e7c5fd599e891a5dd027aa173597329a471326dc931d74b5a913e3cc9ef39a13066b68147691eb8aa6ac
+  metadata.gz: 6a1bc7aef728a460b519d695641f1683dd6b738bbc66ba948df8b3a21aca1d0b79e951ee6d9f0577773c078995f6a276bc76fa524bcfc80d2a33e2163f63d80d
+  data.tar.gz: 778cc3e7bf5233862971030ee5023264fc7f42057199ab296b5d091469eebd3452a971accdb25826ee825dbbb75cda7d677f0bc479461818daa454d84bb2fa4b

data/README.md

@@ -12,11 +12,18 @@ Use RubyGems:
 Example:
 
     <match foo.**>
-      type ec2-metadata
-      output_tag ${instance_id}.${tag}
+      type ec2_metadata
+
+      aws_key_id  YOUR_AWS_KEY_ID
+      aws_sec_key YOUR_AWS_SECRET/KEY
 
+      output_tag ${instance_id}.${tag}
       <record>
-        hostname ${instance_id}
+        hostname      ${tagset_name}
+        instance_id   ${instance_id}
+        instance_type ${instance_type}
+        az            ${availability_zone}
+        vpc_id        ${vpc_id}
       </record>
     </match>
 
@@ -30,18 +37,31 @@ then output becomes as below (indented):
 
 ```js
 i-28b5ee77.foo.bar {
-  "hostname" : "i-28b5ee77",
-  "message"  : "hello ec2!"
+  "hostname"      : "web0001",
+  "instance_id"   : "i-28b5ee77",
+  "instance_type" : "m1.large",
+  "az"            : "us-west-1b",
+  "vpc_id"        : "vpc-25dab194",
+  "message"       : "hello ec2!"
 }
 ```
 
 ### Placeholders
 
-The following placeholders are available:
+The following placeholders are always available:
 
-* ${instance_id} instance id
 * ${tag} input tag
 * ${tag_parts} input tag splitted by '.'. you can use it like `${tag_parts[0]}` or `${tag_parts[-1]}`
+* ${instance_id} instance id
+* ${instance_type} instance type
+* ${availability_zone} availability zone
+* ${region} region
+
+The followings are available when you define `aws_key_id` and `aws_sec_key`:
+
+* ${vpc_id} vpc id
+* ${subnet_id} subnet id
+* ${tagset_xxx} EC2 tag (e.g. tagset_name is replaced by the value of Key = Name)
 
 ## Contributing
 

data/fluent-plugin-ec2-metadata.gemspec

@@ -4,11 +4,11 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-ec2-metadata"
-  spec.version       = "0.0.2"
+  spec.version       = "0.0.3"
   spec.authors       = ["SAKAMOTO Takumi"]
   spec.email         = ["takumi.saka@gmail.com"]
-  spec.description   = %q{Output plugin to add ec2 metadata into messages}
-  spec.summary       = %q{Output plugin to add ec2 metadata into messages}
+  spec.description   = %q{Fluentd output plugin to add ec2 metadata fields to a event record}
+  spec.summary       = %q{Fluentd output plugin to add ec2 metadata fields to a event record}
   spec.homepage      = "https://github.com/takus/fluent-plugin-ec2-metadata"
   spec.license       = "APLv2"
 
@@ -19,4 +19,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "rake"
   spec.add_runtime_dependency     "fluentd"
+  spec.add_runtime_dependency     "aws-sdk"
 end

data/lib/fluent/plugin/out_ec2_metadata.rb

@@ -7,7 +7,10 @@ module Fluent
       require 'net/http'
     end
 
-    config_param :output_tag, :string
+    config_param :output_tag,  :string
+
+    config_param :aws_key_id,  :string, :default => ENV['AWS_ACCESS_KEY_ID']
+    config_param :aws_sec_key, :string, :default => ENV['AWS_SECRET_ACCESS_KEY']
 
     def configure(conf)
       super
@@ -25,7 +28,27 @@ module Fluent
 
       # get ec2 metadata
       @ec2_metadata = {}
-      @ec2_metadata['instance_id'] = get_instance_id()
+      @ec2_metadata['instance_id']       = get_metadata('instance-id')
+      @ec2_metadata['instance_type']     = get_metadata('instance-type')
+      @ec2_metadata['availability_zone'] = get_metadata('placement/availability-zone')
+      @ec2_metadata['region']            = @ec2_metadata['availability_zone'].chop
+
+      if @aws_key_id and @aws_sec_key then
+        require 'aws-sdk'
+
+        AWS.config(access_key_id: @aws_key_id, secret_access_key: @aws_sec_key, region: @ec2_metadata['region'])
+
+        response = AWS.ec2.client.describe_instances( { :instance_ids => [ @ec2_metadata['instance_id'] ] })
+        instance = response[:reservation_set].first[:instances_set].first
+        raise Fluent::ConfigError, "ec2-metadata: failed to get instance data #{response.pretty_inspect}" if instance.nil?
+
+        @ec2_metadata['vpc_id']    = instance[:vpc_id]
+        @ec2_metadata['subnet_id'] = instance[:subnet_id]
+
+        instance[:tag_set].each { |tag|
+            @ec2_metadata["tagset_#{tag[:key].downcase}"] = tag[:value]
+        }
+      end
     end
 
     def emit(tag, es, chain)
@@ -41,10 +64,9 @@ module Fluent
 
     private
 
-    def get_instance_id
-        res = Net::HTTP.get_response("169.254.169.254", "/latest/meta-data/instance-id")
-        raise Fluent::ConfigError, "ec2-metadata: failed to get instance-id" unless res.is_a?(Net::HTTPSuccess)
-        raise Fluent::ConfigError, "ec2-metadata: invalid instance-id #{res.body}" unless res.body =~ /^i-\h{8}$/
+    def get_metadata(f)
+        res = Net::HTTP.get_response("169.254.169.254", "/latest/meta-data/#{f}")
+        raise Fluent::ConfigError, "ec2-metadata: failed to get #{f}" unless res.is_a?(Net::HTTPSuccess)
         res.body
     end
 
@@ -83,8 +105,7 @@ module Fluent
       end
 
       def expand(str)
-        str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
-          $log.warn "ec2-metadata: unknown placeholder `#{$1}` found in a tag `#{tag}`" unless @placeholders.include?($1)
+        str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) { $log.warn "ec2-metadata: unknown placeholder `#{$1}` found in a tag `#{tag}`" unless @placeholders.include?($1)
           @placeholders[$1]
         }
       end

data/test/plugin/test_out_ec2_metadata.rb

@@ -9,6 +9,7 @@ class EC2MetadataOutputTest < Test::Unit::TestCase
     output_tag ${instance_id}.${tag}
     <record>
       instance_id ${instance_id}
+      az          ${availability_zone}
     </record>
   ]
 
@@ -20,6 +21,10 @@ class EC2MetadataOutputTest < Test::Unit::TestCase
     Net::HTTP.get_response('169.254.169.254', '/latest/meta-data/instance-id').body
   end
 
+  def get_avaivality_zone
+    Net::HTTP.get_response('169.254.169.254', '/latest/meta-data/placement/availability-zone').body
+  end
+
   def test_emit
     d = create_driver(CONFIG, 'foo.bar')
 
@@ -29,13 +34,14 @@ class EC2MetadataOutputTest < Test::Unit::TestCase
     end
 
     instance_id = get_instance_id
+    availability_zone = get_avaivality_zone
 
     # tag
     assert_equal "#{instance_id}.foo.bar", d.emits[0][0]
     assert_equal "#{instance_id}.foo.bar", d.emits[1][0]
 
     # record
-    mapped = { 'instance_id' => instance_id }
+    mapped = { 'instance_id' => instance_id, 'az' => availability_zone }
     assert_equal [
       {"a" => 1}.merge(mapped),
       {"a" => 2}.merge(mapped),

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-ec2-metadata
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - SAKAMOTO Takumi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-17 00:00:00.000000000 Z
+date: 2014-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -38,7 +38,21 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Output plugin to add ec2 metadata into messages
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd output plugin to add ec2 metadata fields to a event record
 email:
 - takumi.saka@gmail.com
 executables: []
@@ -78,7 +92,7 @@ rubyforge_project:
 rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
-summary: Output plugin to add ec2 metadata into messages
+summary: Fluentd output plugin to add ec2 metadata fields to a event record
 test_files:
 - test/helper.rb
 - test/plugin/test_out_ec2_metadata.rb