fluent-plugin-google-cloud 0.3.1 → 0.4.0

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.3.0)
+    fluent-plugin-google-cloud (0.3.1)
       fluentd (>= 0.10)
       google-api-client (>= 0.8)
       googleauth (~> 0.4)
+      json (~> 1.8.3)
 
 GEM
   remote: https://rubygems.org/
@@ -20,13 +21,13 @@ GEM
       addressable (>= 2.3.1)
       extlib (>= 0.9.15)
       multi_json (>= 1.0.0)
-    cool.io (1.3.0)
+    cool.io (1.3.1)
     crack (0.4.2)
       safe_yaml (~> 1.0.0)
     extlib (0.9.16)
     faraday (0.9.1)
       multipart-post (>= 1.2, < 3)
-    fluentd (0.12.8)
+    fluentd (0.12.11)
       cool.io (>= 1.2.2, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       json (>= 1.4.3)
@@ -56,8 +57,8 @@ GEM
       signet (~> 0.6)
     http_parser.rb (0.6.0)
     i18n (0.7.0)
-    json (1.8.2)
-    jwt (1.4.1)
+    json (1.8.3)
+    jwt (1.5.0)
     launchy (2.4.3)
       addressable (~> 2.3)
     little-plugger (1.1.3)
@@ -66,22 +67,22 @@ GEM
       multi_json (~> 1.10)
     memoist (0.12.0)
     metaclass (0.0.4)
-    minitest (5.6.1)
+    minitest (5.7.0)
     mocha (1.1.0)
       metaclass (~> 0.0.1)
-    msgpack (0.5.11)
+    msgpack (0.5.12)
     multi_json (1.11.0)
     multipart-post (2.0.0)
     power_assert (0.2.3)
     rake (10.4.2)
     retriable (1.4.1)
     safe_yaml (1.0.4)
-    sigdump (0.2.2)
-    signet (0.6.0)
+    sigdump (0.2.3)
+    signet (0.6.1)
       addressable (~> 2.3)
       extlib (~> 0.9)
       faraday (~> 0.9)
-      jwt (~> 1.0)
+      jwt (~> 1.5)
       multi_json (~> 1.10)
     string-scrub (0.0.5)
     test-unit (3.0.9)

data/fluent-plugin-google-cloud.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{Fluentd plugin to stream logs to the Google Cloud Platform's logging API}
   gem.homepage      = 'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache 2.0'
-  gem.version       = '0.3.1'
+  gem.version       = '0.4.0'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
 
@@ -15,6 +15,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency 'fluentd', '>= 0.10'
   gem.add_runtime_dependency 'google-api-client', '>= 0.8'
   gem.add_runtime_dependency 'googleauth', '~> 0.4'
+  gem.add_runtime_dependency 'json', '~> 1.8.3'
   gem.add_development_dependency "rake", '>= 10.3.2'
   gem.add_development_dependency "webmock", '>= 1.17.0'
   gem.add_development_dependency "test-unit", "~> 3.0.2"

data/lib/fluent/plugin/out_google_cloud.rb

@@ -16,14 +16,18 @@ module Fluent
   class GoogleCloudOutput < BufferedOutput
     Fluent::Plugin.register_output('google_cloud', self)
 
-    # Constants for Google service names.
+    # Constants for service names.
     APPENGINE_SERVICE = 'appengine.googleapis.com'
     COMPUTE_SERVICE = 'compute.googleapis.com'
     DATAFLOW_SERVICE = 'dataflow.googleapis.com'
+    EC2_SERVICE = 'ec2.amazonaws.com'
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
 
+    # Address of the metadata service.
+    METADATA_SERVICE_ADDR = '169.254.169.254'
+
     # DEPRECATED: auth_method (and support for 'private_key') is deprecated in
     # favor of Google Application Default Credentials as documented at:
     # https://developers.google.com/identity/protocols/application-default-credentials
@@ -36,10 +40,16 @@ module Fluent
     config_param :private_key_path, :string, :default => nil
     config_param :private_key_passphrase, :string, :default => 'notasecret'
 
-    # If use_metadata_service is set to true, we obtain the project_id, zone,
-    # and vm_id from the GCE metadata service.  Otherwise, those parameters
-    # must be specified in the config file explicitly.
+    # Specify project/instance metadata.
+    #
+    # project_id, zone, and vm_id are required to have valid values, which
+    # can be obtained from the metadata service or set explicitly.
+    # Otherwise, the plugin will fail to initialize.
+    #
+    # Whether to attempt to obtain metadata from the local metadata service.
+    # It is safe to specify 'true' even on platforms with no metadata service.
     config_param :use_metadata_service, :bool, :default => true
+    # These parameters override any values obtained from the metadata service.
     config_param :project_id, :string, :default => nil
     config_param :zone, :string, :default => nil
     config_param :vm_id, :string, :default => nil
@@ -63,6 +73,7 @@ module Fluent
       require 'google/api_client'
       require 'google/api_client/auth/compute_service_account'
       require 'googleauth'
+      require 'json'
       require 'open-uri'
     end
 
@@ -86,11 +97,95 @@ module Fluent
         end
       end
 
-      unless @use_metadata_service
-        unless @project_id && @zone && @vm_id
-          raise Fluent::ConfigError,
-              ('Please specify "project_id", "zone" and "vm_id" if you set "use_metadata_service" to false')
+      # TODO: Send instance tags and/or hostname as labels as well?
+      @common_labels = {}
+
+      # set attributes from metadata (unless overriden by static config)
+      @platform = detect_platform
+      case @platform
+      when Platform::GCE
+        if @project_id.nil?
+          @project_id = fetch_gce_metadata('project/project-id')
+        end
+        if @zone.nil?
+          # this returns "projects/<number>/zones/<zone>"; we only want
+          # the part after the final slash.
+          fully_qualified_zone = fetch_gce_metadata('instance/zone')
+          @zone = fully_qualified_zone.rpartition('/')[2]
+        end
+        if @vm_id.nil?
+          @vm_id = fetch_gce_metadata('instance/id')
+        end
+      when Platform::EC2
+        metadata = fetch_ec2_metadata
+        if @zone.nil? && metadata.has_key?('availabilityZone')
+          @zone = 'aws:' + metadata['availabilityZone']
         end
+        if @vm_id.nil? && metadata.has_key?('instanceId')
+          @vm_id = metadata['instanceId']
+        end
+        if metadata.has_key?('accountId')
+          common_labels["#{EC2_SERVICE}/account_id"] = metadata['accountId']
+        end
+      when Platform::OTHER
+        # do nothing
+      else
+        raise Fluent::ConfigError, 'Unknown platform ' + @platform
+      end
+
+      # all metadata parameters must now be set
+      unless @project_id && @zone && @vm_id
+        missing = []
+        missing << "project_id" unless @project_id
+        missing << "zone" unless @zone
+        missing << "vm_id" unless @vm_id
+        raise Fluent::ConfigError,
+          ('Unable to obtain metadata parameters: ' + missing.join(' '))
+      end
+
+      # Default this to false; it is only overwritten if we detect Managed VM.
+      @running_on_managed_vm = false
+
+      # Set labels, etc. based on the config
+      case @platform
+      when Platform::GCE
+        @service_name = COMPUTE_SERVICE
+        # Check for specialized GCE environments (Managed VM or Dataflow).
+        # TODO: Add config options for these to allow for running outside GCE?
+        attributes = fetch_gce_metadata('instance/attributes/').split
+        if (attributes.include?('gae_backend_name') &&
+            attributes.include?('gae_backend_version'))
+          # Managed VM
+          @running_on_managed_vm = true
+          @gae_backend_name =
+              fetch_gce_metadata('instance/attributes/gae_backend_name')
+          @gae_backend_version =
+              fetch_gce_metadata('instance/attributes/gae_backend_version')
+          @service_name = APPENGINE_SERVICE
+          common_labels["#{APPENGINE_SERVICE}/module_id"] = @gae_backend_name
+          common_labels["#{APPENGINE_SERVICE}/version_id"] =
+            @gae_backend_version
+        elsif (attributes.include?('job_id'))
+          # Dataflow
+          @service_name = DATAFLOW_SERVICE
+          @dataflow_job_id = fetch_gce_metadata('instance/attributes/job_id')
+          common_labels["#{DATAFLOW_SERVICE}/job_id"] = @dataflow_job_id
+        end
+        # include GCE labels unless we're running on dataflow, which
+        # uses their own labels exclusively.
+        if (@service_name != DATAFLOW_SERVICE)
+          common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
+          common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
+        end
+      when Platform::EC2
+        @service_name = EC2_SERVICE
+        common_labels["#{EC2_SERVICE}/resource_type"] = 'instance'
+        common_labels["#{EC2_SERVICE}/resource_id"] = @vm_id
+      when Platform::OTHER
+        # Use COMPUTE_SERVICE as the default environment.
+        @service_name = COMPUTE_SERVICE
+        common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
+        common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
       end
     end
 
@@ -101,47 +196,6 @@ module Fluent
 
       @successful_call = false
       @timenanos_warning = false
-
-      if @use_metadata_service
-        # Grab metadata about the Google Compute Engine instance that we're on.
-        @project_id = fetch_metadata('project/project-id')
-        fully_qualified_zone = fetch_metadata('instance/zone')
-        @zone = fully_qualified_zone.rpartition('/')[2]
-        @vm_id = fetch_metadata('instance/id')
-      end
-      # TODO: Send instance tags and/or hostname with the logs as well?
-      @common_labels = {}
-
-      # If this is running on a Managed VM, grab the relevant App Engine
-      # metadata as well.
-      # TODO: Add config options for these to allow for running outside GCE?
-      attributes_string = @use_metadata_service ?
-          fetch_metadata('instance/attributes/') : ""
-      attributes = attributes_string.split
-      if (attributes.include?('gae_backend_name') &&
-          attributes.include?('gae_backend_version'))
-        @running_on_managed_vm = true
-        @gae_backend_name =
-            fetch_metadata('instance/attributes/gae_backend_name')
-        @gae_backend_version =
-            fetch_metadata('instance/attributes/gae_backend_version')
-        @service_name = APPENGINE_SERVICE
-        common_labels["#{APPENGINE_SERVICE}/module_id"] = @gae_backend_name
-        common_labels["#{APPENGINE_SERVICE}/version_id"] = @gae_backend_version
-      elsif (attributes.include?('job_id'))
-        @running_on_managed_vm = false
-        @service_name = DATAFLOW_SERVICE
-        @dataflow_job_id = fetch_metadata('instance/attributes/job_id')
-        common_labels["#{DATAFLOW_SERVICE}/job_id"] = @dataflow_job_id
-      else
-        @running_on_managed_vm = false
-        @service_name = COMPUTE_SERVICE
-      end
-
-      if (@service_name != DATAFLOW_SERVICE)
-        common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
-        common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
-      end
     end
 
     def shutdown
@@ -284,17 +338,64 @@ module Fluent
     def log_write_failure(request, error)
       dropped = request['entries'].length
       $log.warn "Dropping #{dropped} log message(s)",
-        :error_class=>error.class.to_s, :error=>error.to_s
+        :error_class => error.class.to_s, :error => error.to_s
+    end
+
+    # "enum" of Platform values
+    module Platform
+      OTHER = 0  # Other/unkown platform
+      GCE = 1    # Google Compute Engine
+      EC2 = 2    # Amazon EC2
+    end
+
+    # Determine what platform we are running on by consulting the metadata
+    # service (unless the user has explicitly disabled using that).
+    def detect_platform
+      if !@use_metadata_service
+        $log.info "use_metadata_service is false; not detecting platform"
+        return Platform::OTHER
+      end
+
+      begin
+        open('http://' + METADATA_SERVICE_ADDR) do |f|
+          if (f.meta['metadata-flavor'] == 'Google')
+            $log.info 'Detected GCE platform'
+            return Platform::GCE
+          end
+          if (f.meta['server'] == 'EC2ws')
+            $log.info 'Detected EC2 platform'
+            return Platform::EC2
+          end
+        end
+      rescue Exception => e
+        $log.debug "Failed to access metadata service: ", :error => e
+      end
+
+      $log.info 'Unable to determine platform'
+      return Platform::OTHER
     end
 
-    def fetch_metadata(metadata_path)
-      # Fetch GCE metadata - see https://cloud.google.com/compute/docs/metadata
-      open('http://169.254.169.254/computeMetadata/v1/' + metadata_path,
-           {'Metadata-Flavor' => 'Google'}) do |f|
+    def fetch_gce_metadata(metadata_path)
+      raise "Called fetch_gce_metadata with platform=#{@platform}" unless
+        @platform == Platform::GCE
+      # See https://cloud.google.com/compute/docs/metadata
+      open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
+           metadata_path, {'Metadata-Flavor' => 'Google'}) do |f|
         f.read
       end
     end
 
+    def fetch_ec2_metadata
+      raise "Called fetch_ec2_metadata with platform=#{@platform}" unless
+        @platform == Platform::EC2
+      # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
+      open('http://' + METADATA_SERVICE_ADDR +
+           '/latest/dynamic/instance-identity/document') do |f|
+        contents = f.read()
+        return JSON.parse(contents)
+      end
+    end
+
     # Values permitted by the API for 'severity' (which is an enum).
     VALID_SEVERITIES = Set.new [
       'DEFAULT', 'DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL',
@@ -360,7 +461,7 @@ module Fluent
     def init_api_client
       @client = Google::APIClient.new(
         :application_name => 'Fluentd Google Cloud Logging plugin',
-        :application_version => '0.3.1',
+        :application_version => '0.4.0',
         :retries => 1)
 
       if @auth_method == 'private_key'

data/test/plugin/test_out_google_cloud.rb

@@ -25,22 +25,41 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     @logs_sent = []
   end
 
+  # attributes used for the GCE metadata service
   PROJECT_ID = 'test-project-id'
   ZONE = 'us-central1-b'
   FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
   VM_ID = '9876543210'
 
+  # attributes used for custom (overridden) configs
   CUSTOM_PROJECT_ID = 'test-custom-project-id'
   CUSTOM_ZONE = 'us-custom-central1-b'
   CUSTOM_FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
   CUSTOM_VM_ID = 'C9876543210'
 
+  # attributes used for the EC2 metadata service
+  EC2_PROJECT_ID = 'test-ec2-project-id'
+  EC2_ZONE = 'us-west-2b'
+  EC2_PREFIXED_ZONE = 'aws:' + EC2_ZONE
+  EC2_VM_ID = 'i-81c16767'
+  EC2_ACCOUNT_ID = '123456789012'
+
+  # The formatting here matches the format used on the VM.
+  EC2_IDENTITY_DOCUMENT = %[{
+  "accountId" : "#{EC2_ACCOUNT_ID}",
+  "availabilityZone" : "#{EC2_ZONE}",
+  "instanceId" : "#{EC2_VM_ID}"
+}]
+
+  # Managed VMs specific labels
   MANAGED_VM_BACKEND_NAME = 'default'
   MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
 
+  # Parameters used for authentication
   AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
   FAKE_AUTH_TOKEN = 'abc123'
 
+  # Configuration files for various test scenarios
   APPLICATION_DEFAULT_CONFIG = %[
   ]
 
@@ -50,29 +69,52 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
   ]
 
-  CUSTOM_METADATA_CONFIG = %[
+  NO_METADATA_SERVICE_CONFIG = %[
     use_metadata_service false
+  ]
+
+  CUSTOM_METADATA_CONFIG = %[
     project_id #{CUSTOM_PROJECT_ID}
     zone #{CUSTOM_ZONE}
     vm_id #{CUSTOM_VM_ID}
   ]
 
-  INVALID_CONFIG_MISSING_PRIVATE_KEY_PATH = %[
+  CONFIG_MISSING_PRIVATE_KEY_PATH = %[
     auth_method private_key
     private_key_email nobody@example.com
   ]
-  INVALID_CONFIG_MISSING_PRIVATE_KEY_EMAIL = %[
+  CONFIG_MISSING_PRIVATE_KEY_EMAIL = %[
     auth_method private_key
     private_key_path /fake/path/to/key
   ]
-  INVALID_CONFIG_MISSING_METADATA_VM_ID = %[
-    use_metadata_service false
+  CONFIG_MISSING_METADATA_PROJECT_ID = %[
+    zone #{CUSTOM_ZONE}
+    vm_id #{CUSTOM_VM_ID}
+  ]
+  CONFIG_MISSING_METADATA_ZONE = %[
+    project_id #{CUSTOM_PROJECT_ID}
+    vm_id #{CUSTOM_VM_ID}
+  ]
+  CONFIG_MISSING_METADATA_VM_ID = %[
     project_id #{CUSTOM_PROJECT_ID}
     zone #{CUSTOM_ZONE}
   ]
+  CONFIG_MISSING_METADATA_ALL = %[
+  ]
 
+  CONFIG_EC2_PROJECT_ID = %[
+    project_id #{EC2_PROJECT_ID}
+  ]
+
+  CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %[
+    project_id #{EC2_PROJECT_ID}
+    vm_id #{CUSTOM_VM_ID}
+  ]
+
+  # Service configurations for various services
   COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
   APPENGINE_SERVICE_NAME = 'appengine.googleapis.com'
+  EC2_SERVICE_NAME = 'ec2.amazonaws.com'
 
   COMPUTE_PARAMS = {
     'service_name' => COMPUTE_SERVICE_NAME,
@@ -109,6 +151,18 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     }
   }
 
+  EC2_PARAMS = {
+    'service_name' => EC2_SERVICE_NAME,
+    'log_name' => 'test',
+    'project_id' => EC2_PROJECT_ID,
+    'zone' => EC2_PREFIXED_ZONE,
+    'labels' => {
+      "#{EC2_SERVICE_NAME}/resource_type" => 'instance',
+      "#{EC2_SERVICE_NAME}/resource_id" => EC2_VM_ID,
+      "#{EC2_SERVICE_NAME}/account_id" => EC2_ACCOUNT_ID
+    }
+  }
+
   def create_driver(conf=APPLICATION_DEFAULT_CONFIG)
     Fluent::Test::BufferedOutputTestDriver.new(
         Fluent::GoogleCloudOutput).configure(conf)
@@ -127,16 +181,17 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   def test_configure_custom_metadata
+    setup_no_metadata_service_stubs
     d = create_driver(CUSTOM_METADATA_CONFIG)
     assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
     assert_equal CUSTOM_ZONE, d.instance.zone
     assert_equal CUSTOM_VM_ID, d.instance.vm_id
   end
 
-  def test_configure_invalid_configs
+  def test_configure_invalid_private_key_configs
     exception_count = 0
     begin
-      d = create_driver(INVALID_CONFIG_MISSING_PRIVATE_KEY_PATH)
+      d = create_driver(CONFIG_MISSING_PRIVATE_KEY_PATH)
     rescue Fluent::ConfigError => error
       assert error.message.include? 'private_key_path'
       exception_count += 1
@@ -145,18 +200,54 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
 
     exception_count = 0
     begin
-      d = create_driver(INVALID_CONFIG_MISSING_PRIVATE_KEY_EMAIL)
+      d = create_driver(CONFIG_MISSING_PRIVATE_KEY_EMAIL)
     rescue Fluent::ConfigError => error
       assert error.message.include? 'private_key_email'
       exception_count += 1
     end
     assert_equal 1, exception_count
+  end
+
+  def test_configure_invalid_metadata_configs_no_metadata_service
+    setup_no_metadata_service_stubs
+    exception_count = 0
+    begin
+      d = create_driver(CONFIG_MISSING_METADATA_PROJECT_ID)
+    rescue Fluent::ConfigError => error
+      assert error.message.include? 'Unable to obtain metadata parameters:'
+      assert error.message.include? 'project_id'
+      exception_count += 1
+    end
+    assert_equal 1, exception_count
+
+    exception_count = 0
+    begin
+      d = create_driver(CONFIG_MISSING_METADATA_ZONE)
+    rescue Fluent::ConfigError => error
+      assert error.message.include? 'Unable to obtain metadata parameters:'
+      assert error.message.include? 'zone'
+      exception_count += 1
+    end
+    assert_equal 1, exception_count
+
+    exception_count = 0
+    begin
+      d = create_driver(CONFIG_MISSING_METADATA_VM_ID)
+    rescue Fluent::ConfigError => error
+      assert error.message.include? 'Unable to obtain metadata parameters:'
+      assert error.message.include? 'vm_id'
+      exception_count += 1
+    end
+    assert_equal 1, exception_count
 
     exception_count = 0
     begin
-      d = create_driver(INVALID_CONFIG_MISSING_METADATA_VM_ID)
+      d = create_driver(CONFIG_MISSING_METADATA_ALL)
     rescue Fluent::ConfigError => error
-      assert error.message.include? 'use_metadata_service'
+      assert error.message.include? 'Unable to obtain metadata parameters:'
+      assert error.message.include? 'project_id'
+      assert error.message.include? 'zone'
+      assert error.message.include? 'vm_id'
       exception_count += 1
     end
     assert_equal 1, exception_count
@@ -187,6 +278,18 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
 
   def test_gce_metadata_does_not_load_when_use_metadata_service_is_false
     Fluent::GoogleCloudOutput.any_instance.expects(:fetch_metadata).never
+    d = create_driver(NO_METADATA_SERVICE_CONFIG + CUSTOM_METADATA_CONFIG)
+    d.run
+    assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
+    assert_equal CUSTOM_ZONE, d.instance.zone
+    assert_equal CUSTOM_VM_ID, d.instance.vm_id
+    assert_equal false, d.instance.running_on_managed_vm
+  end
+
+  def test_metadata_overrides_on_gce
+    # In this case we are overriding all configured parameters so we should
+    # see all "custom" values rather than the ones from the metadata server.
+    setup_gce_metadata_stubs
     d = create_driver(CUSTOM_METADATA_CONFIG)
     d.run
     assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
@@ -195,6 +298,51 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_equal false, d.instance.running_on_managed_vm
   end
 
+  def test_metadata_partial_overrides_on_gce
+    # Similar to above, but we are not overriding project_id in this config
+    # so we should see the metadata value for project_id and "custom" otherwise.
+    setup_gce_metadata_stubs
+    d = create_driver(CONFIG_MISSING_METADATA_PROJECT_ID)
+    d.run
+    assert_equal PROJECT_ID, d.instance.project_id
+    assert_equal CUSTOM_ZONE, d.instance.zone
+    assert_equal CUSTOM_VM_ID, d.instance.vm_id
+    assert_equal false, d.instance.running_on_managed_vm
+  end
+
+  def test_ec2_metadata_loading
+    setup_ec2_metadata_stubs
+    d = create_driver(CONFIG_EC2_PROJECT_ID)
+    d.run
+    assert_equal EC2_PROJECT_ID, d.instance.project_id
+    assert_equal EC2_PREFIXED_ZONE, d.instance.zone
+    assert_equal EC2_VM_ID, d.instance.vm_id
+    assert_equal false, d.instance.running_on_managed_vm
+  end
+
+  def test_ec2_metadata_partial_override
+    setup_ec2_metadata_stubs
+    d = create_driver(CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID)
+    d.run
+    assert_equal EC2_PROJECT_ID, d.instance.project_id
+    assert_equal EC2_PREFIXED_ZONE, d.instance.zone
+    assert_equal CUSTOM_VM_ID, d.instance.vm_id
+    assert_equal false, d.instance.running_on_managed_vm
+  end
+
+  def test_ec2_metadata_requires_project_id
+    setup_ec2_metadata_stubs
+    exception_count = 0
+    begin
+      d = create_driver()
+    rescue Fluent::ConfigError => error
+      assert error.message.include? 'Unable to obtain metadata parameters:'
+      assert error.message.include? 'project_id'
+      exception_count += 1
+    end
+    assert_equal 1, exception_count
+  end
+
   def test_one_log
     setup_gce_metadata_stubs
     setup_logging_stubs
@@ -240,15 +388,26 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   def test_one_log_custom_metadata
+    # don't set up any metadata stubs, so the test will fail if we try to
+    # fetch metadata (and explicitly check this as well).
     Fluent::GoogleCloudOutput.any_instance.expects(:fetch_metadata).never
     ENV['GOOGLE_APPLICATION_CREDENTIALS'] = 'test/plugin/data/credentials.json'
     setup_logging_stubs
-    d = create_driver(CUSTOM_METADATA_CONFIG)
+    d = create_driver(NO_METADATA_SERVICE_CONFIG + CUSTOM_METADATA_CONFIG)
     d.emit({'message' => log_entry(0)})
     d.run
     verify_log_entries(1, CUSTOM_PARAMS)
   end
 
+  def test_one_log_ec2
+    setup_ec2_metadata_stubs
+    setup_logging_stubs
+    d = create_driver(CONFIG_EC2_PROJECT_ID)
+    d.emit({'message' => log_entry(0)})
+    d.run
+    verify_log_entries(1, EC2_PARAMS)
+  end
+
   def test_struct_payload_log
     setup_gce_metadata_stubs
     setup_logging_stubs
@@ -534,7 +693,17 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
                 :headers => {'Content-Length' => response_body.length})
   end
 
+  def setup_no_metadata_service_stubs
+    # Simulate a machine with no metadata service present
+    stub_request(:any, /http:\/\/169.254.169.254\/.*/).
+      to_raise Errno::EHOSTUNREACH;
+  end
+
   def setup_gce_metadata_stubs
+    # Stub the root, used for platform detection by the plugin and 'googleauth'.
+    stub_request(:get, 'http://169.254.169.254').
+      to_return(:status => 200, :headers => {'Metadata-Flavor' => 'Google'})
+
     # Create stubs for all the GCE metadata lookups the agent needs to make.
     stub_metadata_request('project/project-id', PROJECT_ID)
     stub_metadata_request('instance/zone', FULLY_QUALIFIED_ZONE)
@@ -542,11 +711,6 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     stub_metadata_request('instance/attributes/',
                           "attribute1\nattribute2\nattribute3")
 
-    # Used by 'googleauth' to test whether we're running on GCE.
-    # It only cares about the request succeeding with Metdata-Flavor: Google.
-    stub_request(:get, 'http://169.254.169.254').
-      to_return(:status => 200, :headers => {'Metadata-Flavor' => 'Google'})
-
     # Used by 'googleauth' to fetch the default service account credentials.
     stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token').
       to_return(:body => "{\"access_token\": \"#{FAKE_AUTH_TOKEN}\"}",
@@ -555,8 +719,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
                              'Content-Type' => 'application/json' })
   end
 
+  def setup_ec2_metadata_stubs
+    # Stub the root, used for platform detection
+    stub_request(:get, 'http://169.254.169.254').
+      to_return(:status => 200, :headers => {'Server' => 'EC2ws'})
+
+    # Stub the identity document lookup made by the agent.
+    stub_request(:get, 'http://169.254.169.254/latest/dynamic/instance-identity/document').
+      to_return(:body => EC2_IDENTITY_DOCUMENT, :status => 200,
+                :headers => {'Content-Length' => EC2_IDENTITY_DOCUMENT.length})
+  end
+
   def setup_logging_stubs
-    [COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS].each do |params|
+    [COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS, EC2_PARAMS].
+        each do |params|
       stub_request(:post, uri_for_log(params)).to_return do |request|
         @logs_sent << JSON.parse(request.body)
         {:body => ''}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-19 00:00:00.000000000 Z
+date: 2015-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -60,6 +60,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.3
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement