fluent-plugin-google-cloud 0.2.3 → 0.2.4

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.2.3)
+    fluent-plugin-google-cloud (0.2.4)
       fluentd (>= 0.10)
       google-api-client (>= 0.8)
+      googleauth (~> 0.4)
 
 GEM
   remote: https://rubygems.org/
@@ -64,10 +65,14 @@ GEM
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
     memoist (0.12.0)
+    metaclass (0.0.4)
     minitest (5.6.1)
+    mocha (1.1.0)
+      metaclass (~> 0.0.1)
     msgpack (0.5.11)
     multi_json (1.11.0)
     multipart-post (2.0.0)
+    power_assert (0.2.3)
     rake (10.4.2)
     retriable (1.4.1)
     safe_yaml (1.0.4)
@@ -79,6 +84,8 @@ GEM
       jwt (~> 1.0)
       multi_json (~> 1.10)
     string-scrub (0.0.5)
+    test-unit (3.0.9)
+      power_assert
     thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
@@ -94,5 +101,7 @@ PLATFORMS
 
 DEPENDENCIES
   fluent-plugin-google-cloud!
+  mocha (~> 1.1)
   rake (>= 10.3.2)
+  test-unit (~> 3.0.2)
   webmock (>= 1.17.0)

data/fluent-plugin-google-cloud.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{Fluentd plugin to stream logs to the Google Cloud Platform's logging API}
   gem.homepage      = 'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache 2.0'
-  gem.version       = '0.2.3'
+  gem.version       = '0.2.4'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
 
@@ -14,6 +14,9 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency 'fluentd', '>= 0.10'
   gem.add_runtime_dependency 'google-api-client', '>= 0.8'
+  gem.add_runtime_dependency 'googleauth', '~> 0.4'
   gem.add_development_dependency "rake", '>= 10.3.2'
   gem.add_development_dependency "webmock", '>= 1.17.0'
+  gem.add_development_dependency "test-unit", "~> 3.0.2"
+  gem.add_development_dependency "mocha", "~> 1.1"
 end

data/lib/fluent/plugin/out_google_cloud.rb

@@ -21,21 +21,29 @@ module Fluent
     COMPUTE_SERVICE = 'compute.googleapis.com'
     DATAFLOW_SERVICE = 'dataflow.googleapis.com'
 
-    # Legal values:
-    # 'compute_engine_service_account' - Use the service account automatically
-    #   available on Google Compute Engine VMs. Note that this requires that
-    #   the logs.writeonly API scope is enabled on the VM, and scopes can
-    #   only be enabled at the time that a VM is created.
-    # 'private_key' - Use the service account credentials (email, private key
-    #   local file path, and file passphrase) provided below.
-    config_param :auth_method, :string,
-        :default => 'compute_engine_service_account'
-
-    # Parameters necessary to use the private_key auth_method.
+    # Name of the the Google cloud logging write scope.
+    LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
+
+    # DEPRECATED: auth_method (and support for 'private_key') is deprecated in
+    # favor of Google Application Default Credentials as documented at:
+    # https://developers.google.com/identity/protocols/application-default-credentials
+    # 'private_key' is still accepted to support existing users; any other
+    # value is ignored.
+    config_param :auth_method, :string, :default => nil
+
+    # DEPRECATED: Parameters necessary to use the private_key auth_method.
     config_param :private_key_email, :string, :default => nil
     config_param :private_key_path, :string, :default => nil
     config_param :private_key_passphrase, :string, :default => 'notasecret'
 
+    # If fetch_gce_metadata is set to true, we obtain the project_id, zone,
+    # and vm_id from the GCE metadata service.  Otherwise, those parameters
+    # must be specified in the config file explicitly.
+    config_param :fetch_gce_metadata, :bool, :default => true
+    config_param :project_id, :string, :default => nil
+    config_param :zone, :string, :default => nil
+    config_param :vm_id, :string, :default => nil
+
     # TODO: Add a log_name config option rather than just using the tag?
 
     # Expose attr_readers to make testing of metadata more direct than only
@@ -54,30 +62,35 @@ module Fluent
       require 'cgi'
       require 'google/api_client'
       require 'google/api_client/auth/compute_service_account'
+      require 'googleauth'
       require 'open-uri'
     end
 
     def configure(conf)
       super
 
-      case @auth_method
-      when 'private_key'
-        if !@private_key_email
-          raise Fluent::ConfigError, ('"private_key_email" must be ' +
-              'specified if auth_method is "private_key"')
-        elsif !@private_key_path
-          raise Fluent::ConfigError, ('"private_key_path" must be ' +
-              'specified if auth_method is "private_key"')
-        elsif !@private_key_passphrase
-          raise Fluent::ConfigError, ('"private_key_passphrase" must be ' +
-              'specified if auth_method is "private_key"')
+      if !@auth_method.nil?
+        $log.warn ('auth_method is deprecated; please migrate to using ' +
+                   'Application Default Credentials.')
+        if @auth_method == 'private_key'
+          if !@private_key_email
+            raise Fluent::ConfigError, ('"private_key_email" must be ' +
+                'specified if auth_method is "private_key"')
+          elsif !@private_key_path
+            raise Fluent::ConfigError, ('"private_key_path" must be ' +
+                'specified if auth_method is "private_key"')
+          elsif !@private_key_passphrase
+            raise Fluent::ConfigError, ('"private_key_passphrase" must be ' +
+                'specified if auth_method is "private_key"')
+          end
+        end
+      end
+
+      unless @fetch_gce_metadata
+        unless @project_id && @zone && @vm_id
+          raise Fluent::ConfigError,
+              ('Please specify "project_id", "zone" and "vm_id" if you set "fetch_gce_metadata" to false')
         end
-      when 'compute_engine_service_account'
-        # pass
-      else
-        raise Fluent::ConfigError,
-            ('Unrecognized "auth_method" parameter. Please specify either ' +
-             '"compute_engine_service_account" or "private_key".')
       end
     end
 
@@ -88,18 +101,21 @@ module Fluent
 
       @successful_call = false
 
-      # Grab metadata about the Google Compute Engine instance that we're on.
-      @project_id = fetch_metadata('project/project-id')
-      fully_qualified_zone = fetch_metadata('instance/zone')
-      @zone = fully_qualified_zone.rpartition('/')[2]
-      @vm_id = fetch_metadata('instance/id')
+      if @fetch_gce_metadata
+        # Grab metadata about the Google Compute Engine instance that we're on.
+        @project_id = fetch_metadata('project/project-id')
+        fully_qualified_zone = fetch_metadata('instance/zone')
+        @zone = fully_qualified_zone.rpartition('/')[2]
+        @vm_id = fetch_metadata('instance/id')
+      end
       # TODO: Send instance tags and/or hostname with the logs as well?
       @common_labels = {}
 
       # If this is running on a Managed VM, grab the relevant App Engine
       # metadata as well.
       # TODO: Add config options for these to allow for running outside GCE?
-      attributes_string = fetch_metadata('instance/attributes/')
+      attributes_string = @fetch_gce_metadata ?
+          fetch_metadata('instance/attributes/') : ""
       attributes = attributes_string.split
       if (attributes.include?('gae_backend_name') &&
           attributes.include?('gae_backend_version'))
@@ -319,19 +335,19 @@ module Fluent
     def init_api_client
       @client = Google::APIClient.new(
         :application_name => 'Fluentd Google Cloud Logging plugin',
-        :application_version => '0.2.3',
+        :application_version => '0.2.4',
         :retries => 1)
 
       if @auth_method == 'private_key'
         key = Google::APIClient::PKCS12.load_key(@private_key_path,
                                                  @private_key_passphrase)
         jwt_asserter = Google::APIClient::JWTAsserter.new(
-          @private_key_email, 'https://www.googleapis.com/auth/logging.write',
-          key)
+          @private_key_email, LOGGING_SCOPE, key)
         @client.authorization = jwt_asserter.to_authorization
         @client.authorization.expiry = 3600  # 3600s is the max allowed value
       else
-        @client.authorization = Google::APIClient::ComputeServiceAccount.new
+        @client.authorization = Google::Auth.get_application_default(
+            LOGGING_SCOPE)
       end
     end
 

data/test/plugin/data/credentials.json

@@ -0,0 +1,8 @@
+{
+  "private_key_id": "cbedb7568906086cab57859bbfc1748749cc46c4",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIICdwIBADANBgkqhkiG9w0BAQEcAASCAmEwggJdAgEAAoGBAKizy6B+aJ0Wua0e\njZ3pkHV0a2Ce1prJGhzGL5NpkbUjk6J11Kwp1yvPikTwALyy4PtUIZ+23D/unVRM\nHlKa2MkHIGjJg+mykX5Bd7eRJOxdJ0iu+eRWh7HiH+mdDntHwaz4xXihJBog71qS\n+9N+r2hy1hicybechchMiXHhmWPbAgMBAAECgYEAnSzeI4qCZxEcLtnPcXeBWpz7\nycpTAWUpycMvsjTiRxR9YRhM65YT3cJ//VhqJ2S1ThOcPCt/KqViuX4tpiKUo7qA\nH1AI9APbTo66wiGpgy+qG0wPJkKIQC8PpITNNcHqcbbAsIr3/XQduihsqxP2W2mT\na0nk5XJghs1Wa0xt28ECQQDgMqZjVDcDQyqM+bcBKJUUc/247KusjpdK70r6sx2o\nkZJGy/w9exlM5QrB6DLpw34/p5x4MoecZ7lS3yHdmaEhAkEAwKHsV4k5SXTUp4+J\nWK6GlQVvnwc+PQdX5gzt4/gWSY0Op5EQ+YD6cC7Lkz+GzXUzvmdp35c0ahS93D1/\nZLTZewJBAIjOc3cHMNadyr5BtulPEUE0ro+EY/GlBS8lu/QlDmkJg2AOI3qEvliM\nvza58S9yKny/U5yJAPVw2cZ3ABxQHeECQDyBX8PrBURuXvE2o5RoVTtvlqziAi3X\nJaPLwdkOLqnxlX3KkgNcoM0l1amtlYDpZcRVcSs0+9TqKOyJoH8YUwsCQA4cJmv3\n119xcijXPM2HZOB5cCxTHj59MRtQlLboNZ2witDCJ20eG9AC3ZcH7csS0H9dz8Jr\nXGEoQMPD2ck4T0U\u003d\n-----END PRIVATE KEY-----\n",
+  "client_email": "847859579879-q8ancssppuvtv8dac0i742pslde81jgl@developer.gserviceaccount.com",
+  "client_id": "847859579879-q8ancssppuvtv8dac0i742pslde81jgl.apps.googleusercontent.com",
+  "type": "service_account"
+}
+

data/test/plugin/data/invalid_credentials.json

@@ -0,0 +1,8 @@
+{
+  "private_key_id": "cbedb7568906086cab57859bbfc1748749cc46c4",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nCeci n'est pas une cle\n-----END PRIVATE KEY-----\n",
+  "client_email": "847859579879-q8ancssppuvtv8dac0i742pslde81jgl@developer.gserviceaccount.com",
+  "client_id": "847859579879-q8ancssppuvtv8dac0i742pslde81jgl.apps.googleusercontent.com",
+  "type": "service_account"
+}
+

data/test/plugin/test_out_google_cloud.rb

@@ -14,50 +14,34 @@
 
 require 'helper'
 require 'json'
+require 'mocha/test_unit'
 require 'webmock/test_unit'
 
 class GoogleCloudOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-
-    # Create stubs for all the GCE metadata lookups the agent needs to make.
-    stub_metadata_request('project/project-id', PROJECT_ID)
-    stub_metadata_request('instance/zone', FULLY_QUALIFIED_ZONE)
-    stub_metadata_request('instance/id', VM_ID)
-    stub_metadata_request('instance/attributes/',
-                          "attribute1\nattribute2\nattribute3")
-
-    stub_request(:post, 'https://accounts.google.com/o/oauth2/token').
-      with(:body => hash_including({:grant_type => AUTH_GRANT_TYPE})).
-      to_return(:body => "{\"access_token\": \"#{FAKE_AUTH_TOKEN}\"}",
-                :status => 200,
-                :headers => {'Content-Length' => FAKE_AUTH_TOKEN,
-                             'Content-Type' => 'application/json' })
-
+    ENV.delete('GOOGLE_APPLICATION_CREDENTIALS')
+    setup_auth_stubs
     @logs_sent = []
   end
 
-  def setup_logging_stubs
-    [COMPUTE_PARAMS, VMENGINE_PARAMS].each do |params|
-      stub_request(:post, uri_for_log(params)).to_return do |request|
-        @logs_sent << JSON.parse(request.body)
-        {:body => ''}
-      end
-    end
-  end
-
   PROJECT_ID = 'test-project-id'
   ZONE = 'us-central1-b'
   FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
   VM_ID = '9876543210'
 
+  CUSTOM_PROJECT_ID = 'test-custom-project-id'
+  CUSTOM_ZONE = 'us-custom-central1-b'
+  CUSTOM_FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
+  CUSTOM_VM_ID = 'C9876543210'
+
   MANAGED_VM_BACKEND_NAME = 'default'
   MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
 
   AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
   FAKE_AUTH_TOKEN = 'abc123'
 
-  COMPUTE_ENGINE_SERVICE_ACCOUNT_CONFIG = %[
+  APPLICATION_DEFAULT_CONFIG = %[
   ]
 
   PRIVATE_KEY_CONFIG = %[
@@ -66,16 +50,25 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
   ]
 
-  INVALID_CONFIG1 = %[
+  CUSTOM_METADATA_CONFIG = %[
+    fetch_gce_metadata false
+    project_id #{CUSTOM_PROJECT_ID}
+    zone #{CUSTOM_ZONE}
+    vm_id #{CUSTOM_VM_ID}
+  ]
+
+  INVALID_CONFIG_MISSING_PRIVATE_KEY_PATH = %[
     auth_method private_key
     private_key_email nobody@example.com
   ]
-  INVALID_CONFIG2 = %[
+  INVALID_CONFIG_MISSING_PRIVATE_KEY_EMAIL = %[
     auth_method private_key
     private_key_path /fake/path/to/key
   ]
-  INVALID_CONFIG3 = %[
-    auth_method service_account
+  INVALID_CONFIG_MISSING_METADATA_VM_ID = %[
+    fetch_gce_metadata false
+    project_id #{CUSTOM_PROJECT_ID}
+    zone #{CUSTOM_ZONE}
   ]
 
   COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
@@ -84,6 +77,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   COMPUTE_PARAMS = {
     'service_name' => COMPUTE_SERVICE_NAME,
     'log_name' => 'test',
+    'project_id' => PROJECT_ID,
+    'zone' => ZONE,
     'labels' => {
       "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
       "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID
@@ -93,6 +88,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   VMENGINE_PARAMS = {
     'service_name' => APPENGINE_SERVICE_NAME,
     'log_name' => "#{APPENGINE_SERVICE_NAME}%2Ftest",
+    'project_id' => PROJECT_ID,
+    'zone' => ZONE,
     'labels' => {
       "#{APPENGINE_SERVICE_NAME}/module_id" => MANAGED_VM_BACKEND_NAME,
       "#{APPENGINE_SERVICE_NAME}/version_id" => MANAGED_VM_BACKEND_VERSION,
@@ -101,44 +98,73 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     }
   }
 
-  def create_driver(conf=PRIVATE_KEY_CONFIG)
+  CUSTOM_PARAMS = {
+    'service_name' => COMPUTE_SERVICE_NAME,
+    'log_name' => 'test',
+    'project_id' => CUSTOM_PROJECT_ID,
+    'zone' => CUSTOM_ZONE,
+    'labels' => {
+      "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
+      "#{COMPUTE_SERVICE_NAME}/resource_id" => CUSTOM_VM_ID
+    }
+  }
+
+  def create_driver(conf=APPLICATION_DEFAULT_CONFIG)
     Fluent::Test::BufferedOutputTestDriver.new(
         Fluent::GoogleCloudOutput).configure(conf)
   end
 
-  def test_configure_service_account
-    d = create_driver(COMPUTE_ENGINE_SERVICE_ACCOUNT_CONFIG)
-    assert_equal 'compute_engine_service_account', d.instance.auth_method
+  def test_configure_service_account_application_default
+    setup_gce_metadata_stubs
+    d = create_driver(APPLICATION_DEFAULT_CONFIG)
+    assert d.instance.auth_method.nil?
   end
 
-  def test_configure_service_account
+  def test_configure_service_account_private_key
+    setup_gce_metadata_stubs
     d = create_driver(PRIVATE_KEY_CONFIG)
     assert_equal 'private_key', d.instance.auth_method
   end
 
+  def test_configure_custom_metadata
+    d = create_driver(CUSTOM_METADATA_CONFIG)
+    assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
+    assert_equal CUSTOM_ZONE, d.instance.zone
+    assert_equal CUSTOM_VM_ID, d.instance.vm_id
+  end
+
   def test_configure_invalid_configs
+    exception_count = 0
     begin
-      d = create_driver(INVALID_CONFIG1)
-      assert false
+      d = create_driver(INVALID_CONFIG_MISSING_PRIVATE_KEY_PATH)
     rescue Fluent::ConfigError => error
       assert error.message.include? 'private_key_path'
+      exception_count += 1
     end
+    assert_equal 1, exception_count
+
+    exception_count = 0
     begin
-      d = create_driver(INVALID_CONFIG2)
-      assert false
+      d = create_driver(INVALID_CONFIG_MISSING_PRIVATE_KEY_EMAIL)
     rescue Fluent::ConfigError => error
       assert error.message.include? 'private_key_email'
+      exception_count += 1
     end
+    assert_equal 1, exception_count
+
+    exception_count = 0
     begin
-      d = create_driver(INVALID_CONFIG3)
-      assert false
+      d = create_driver(INVALID_CONFIG_MISSING_METADATA_VM_ID)
     rescue Fluent::ConfigError => error
-      assert error.message.include? 'auth_method'
+      assert error.message.include? 'fetch_gce_metadata'
+      exception_count += 1
     end
+    assert_equal 1, exception_count
   end
 
   def test_metadata_loading
-    d = create_driver(PRIVATE_KEY_CONFIG)
+    setup_gce_metadata_stubs
+    d = create_driver()
     d.run
     assert_equal PROJECT_ID, d.instance.project_id
     assert_equal ZONE, d.instance.zone
@@ -147,8 +173,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   def test_managed_vm_metadata_loading
+    setup_gce_metadata_stubs
     setup_managed_vm_metadata_stubs
-    d = create_driver(PRIVATE_KEY_CONFIG)
+    d = create_driver()
     d.run
     assert_equal PROJECT_ID, d.instance.project_id
     assert_equal ZONE, d.instance.zone
@@ -158,7 +185,53 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_equal MANAGED_VM_BACKEND_VERSION, d.instance.gae_backend_version
   end
 
+  def test_gce_metadata_does_not_load_when_fetch_gce_metadata_is_false
+    Fluent::GoogleCloudOutput.any_instance.expects(:fetch_metadata).never
+    d = create_driver(CUSTOM_METADATA_CONFIG)
+    d.run
+    assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
+    assert_equal CUSTOM_ZONE, d.instance.zone
+    assert_equal CUSTOM_VM_ID, d.instance.vm_id
+    assert_equal false, d.instance.running_on_managed_vm
+  end
+
   def test_one_log
+    setup_gce_metadata_stubs
+    setup_logging_stubs
+    d = create_driver()
+    d.emit({'message' => log_entry(0)})
+    d.run
+    verify_log_entries(1, COMPUTE_PARAMS)
+  end
+
+  def test_one_log_with_json_credentials
+    setup_gce_metadata_stubs
+    setup_logging_stubs
+    ENV['GOOGLE_APPLICATION_CREDENTIALS'] = 'test/plugin/data/credentials.json'
+    d = create_driver()
+    d.emit({'message' => log_entry(0)})
+    d.run
+    verify_log_entries(1, COMPUTE_PARAMS)
+  end
+
+  def test_one_log_with_invalid_json_credentials
+    setup_gce_metadata_stubs
+    setup_logging_stubs
+    ENV['GOOGLE_APPLICATION_CREDENTIALS'] = 'test/plugin/data/invalid_credentials.json'
+    d = create_driver()
+    d.emit({'message' => log_entry(0)})
+    exception_count = 0
+    begin
+      d.run
+    rescue RuntimeError => error
+      assert error.message.include? 'Unable to read the credential file'
+      exception_count += 1
+    end
+    assert_equal 1, exception_count
+  end
+
+  def test_one_log_private_key
+    setup_gce_metadata_stubs
     setup_logging_stubs
     d = create_driver(PRIVATE_KEY_CONFIG)
     d.emit({'message' => log_entry(0)})
@@ -166,9 +239,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     verify_log_entries(1, COMPUTE_PARAMS)
   end
 
+  def test_one_log_custom_metadata
+    Fluent::GoogleCloudOutput.any_instance.expects(:fetch_metadata).never
+    ENV['GOOGLE_APPLICATION_CREDENTIALS'] = 'test/plugin/data/credentials.json'
+    setup_logging_stubs
+    d = create_driver(CUSTOM_METADATA_CONFIG)
+    d.emit({'message' => log_entry(0)})
+    d.run
+    verify_log_entries(1, CUSTOM_PARAMS)
+  end
+
   def test_struct_payload_log
+    setup_gce_metadata_stubs
     setup_logging_stubs
-    d = create_driver(PRIVATE_KEY_CONFIG)
+    d = create_driver()
     d.emit({'msg' => log_entry(0), 'tag2' => 'test', 'data' => 5000})
     d.run
     verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
@@ -180,8 +264,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   def test_timestamps
+    setup_gce_metadata_stubs
     setup_logging_stubs
-    d = create_driver(PRIVATE_KEY_CONFIG)
+    d = create_driver()
     expected_ts = []
     emit_index = 0
     [Time.at(123456.789), Time.at(0), Time.now].each do |ts|
@@ -208,8 +293,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   def test_severities
+    setup_gce_metadata_stubs
     setup_logging_stubs
-    d = create_driver(PRIVATE_KEY_CONFIG)
+    d = create_driver()
     expected_severity = []
     emit_index = 0
     # Array of pairs of [parsed_severity, expected_severity]
@@ -229,8 +315,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   def test_multiple_logs
+    setup_gce_metadata_stubs
     setup_logging_stubs
-    d = create_driver(PRIVATE_KEY_CONFIG)
+    d = create_driver()
     # Only test a few values because otherwise the test can take minutes.
     [2, 3, 5, 11, 50].each do |n|
       # The test driver doesn't clear its buffer of entries after running, so
@@ -244,11 +331,12 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   def test_client_error
+    setup_gce_metadata_stubs
     # The API Client should not retry this and the plugin should consume
     # the exception.
     stub_request(:post, uri_for_log(COMPUTE_PARAMS)).to_return(
         :status => 400, :body => "Bad Request")
-    d = create_driver(PRIVATE_KEY_CONFIG)
+    d = create_driver()
     d.emit({'message' => log_entry(0)})
     d.run
     assert_requested(:post, uri_for_log(COMPUTE_PARAMS), :times => 1)
@@ -256,9 +344,10 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
 
   # helper for the ClientError retriable special cases below.
   def client_error_helper(message)
+    setup_gce_metadata_stubs
     stub_request(:post, uri_for_log(COMPUTE_PARAMS)).to_return(
         :status => 401, :body => message)
-    d = create_driver(PRIVATE_KEY_CONFIG)
+    d = create_driver()
     d.emit({'message' => log_entry(0)})
     exception_count = 0
     begin
@@ -292,11 +381,12 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   def test_server_error
+    setup_gce_metadata_stubs
     # The API client should retry this once, then throw an exception which
     # gets propagated through the plugin.
     stub_request(:post, uri_for_log(COMPUTE_PARAMS)).to_return(
         :status => 500, :body => "Server Error")
-    d = create_driver(PRIVATE_KEY_CONFIG)
+    d = create_driver()
     d.emit({'message' => log_entry(0)})
     exception_count = 0
     begin
@@ -310,18 +400,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   def test_one_managed_vm_log
+    setup_gce_metadata_stubs
     setup_managed_vm_metadata_stubs
     setup_logging_stubs
-    d = create_driver(PRIVATE_KEY_CONFIG)
+    d = create_driver()
     d.emit({'message' => log_entry(0)})
     d.run
     verify_log_entries(1, VMENGINE_PARAMS)
   end
 
   def test_multiple_managed_vm_logs
+    setup_gce_metadata_stubs
     setup_managed_vm_metadata_stubs
     setup_logging_stubs
-    d = create_driver(PRIVATE_KEY_CONFIG)
+    d = create_driver()
     [2, 3, 5, 11, 50].each do |n|
       # The test driver doesn't clear its buffer of entries after running, so
       # do it manually here.
@@ -414,7 +506,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   private
 
   def uri_for_log(config)
-    'https://logging.googleapis.com/v1beta3/projects/' + PROJECT_ID +
+    'https://logging.googleapis.com/v1beta3/projects/' + config['project_id'] +
         '/logs/' + config['log_name'] + '/entries:write'
   end
 
@@ -424,6 +516,53 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
                 :headers => {'Content-Length' => response_body.length})
   end
 
+  def setup_gce_metadata_stubs
+    # Create stubs for all the GCE metadata lookups the agent needs to make.
+    stub_metadata_request('project/project-id', PROJECT_ID)
+    stub_metadata_request('instance/zone', FULLY_QUALIFIED_ZONE)
+    stub_metadata_request('instance/id', VM_ID)
+    stub_metadata_request('instance/attributes/',
+                          "attribute1\nattribute2\nattribute3")
+
+    # Used by 'googleauth' to test whether we're running on GCE.
+    # It only cares about the request succeeding with Metdata-Flavor: Google.
+    stub_request(:get, 'http://169.254.169.254').
+      to_return(:status => 200, :headers => {'Metadata-Flavor' => 'Google'})
+
+    # Used by 'googleauth' to fetch the default service account credentials.
+    stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token').
+      to_return(:body => "{\"access_token\": \"#{FAKE_AUTH_TOKEN}\"}",
+                :status => 200,
+                :headers => {'Content-Length' => FAKE_AUTH_TOKEN.length,
+                             'Content-Type' => 'application/json' })
+  end
+
+  def setup_logging_stubs
+    [COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS].each do |params|
+      stub_request(:post, uri_for_log(params)).to_return do |request|
+        @logs_sent << JSON.parse(request.body)
+        {:body => ''}
+      end
+    end
+  end
+
+  def setup_auth_stubs
+    # Used when loading credentials from a JSON file.
+    stub_request(:post, 'https://www.googleapis.com/oauth2/v3/token').
+      with(:body => hash_including({:grant_type => AUTH_GRANT_TYPE})).
+      to_return(:body => "{\"access_token\": \"#{FAKE_AUTH_TOKEN}\"}",
+                :status => 200,
+                :headers => {'Content-Length' => FAKE_AUTH_TOKEN.length,
+                             'Content-Type' => 'application/json' })
+    # Used for 'private_key' auth.
+    stub_request(:post, 'https://accounts.google.com/o/oauth2/token').
+      with(:body => hash_including({:grant_type => AUTH_GRANT_TYPE})).
+      to_return(:body => "{\"access_token\": \"#{FAKE_AUTH_TOKEN}\"}",
+                :status => 200,
+                :headers => {'Content-Length' => FAKE_AUTH_TOKEN.length,
+                             'Content-Type' => 'application/json' })
+  end
+
   def setup_managed_vm_metadata_stubs
     stub_metadata_request(
       'instance/attributes/',
@@ -462,7 +601,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
           assert_equal "test log entry #{i}", entry['textPayload'], batch
         end
 
-        assert_equal ZONE, entry['metadata']['zone']
+        assert_equal params['zone'], entry['metadata']['zone']
         assert_equal params['service_name'], entry['metadata']['serviceName']
         check_labels entry, batch['commonLabels'], params['labels']
         if (block_given?)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
   prerelease: 
 platform: ruby
 authors:
@@ -44,6 +44,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: googleauth
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -76,6 +92,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.17.0
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.2
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
 description: Fluentd plugin to stream logs to the Google Cloud Platform's logging
   API, which will make them viewable in the Developer Console's log viewer and can
   optionally store them in Google Cloud Storage and/or BigQuery. This is an official
@@ -86,17 +134,19 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/fluent/plugin/out_google_cloud.rb
-- Gemfile.lock
-- CONTRIBUTING
-- README.rdoc
 - test/helper.rb
-- test/plugin/test_out_google_cloud.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
+- test/plugin/data/credentials.json
+- test/plugin/data/invalid_credentials.json
+- test/plugin/test_out_google_cloud.rb
 - LICENSE
 - Rakefile
-- Gemfile
 - fluent-plugin-google-cloud.gemspec
+- lib/fluent/plugin/out_google_cloud.rb
+- CONTRIBUTING
+- Gemfile.lock
+- Gemfile
+- README.rdoc
 homepage: https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
 licenses:
 - Apache 2.0
@@ -124,5 +174,7 @@ specification_version: 3
 summary: Fluentd plugin to stream logs to the Google Cloud Platform's logging API
 test_files:
 - test/helper.rb
-- test/plugin/test_out_google_cloud.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
+- test/plugin/data/credentials.json
+- test/plugin/data/invalid_credentials.json
+- test/plugin/test_out_google_cloud.rb