fluent-plugin-google-cloud 0.6.13 → 0.6.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9d51deb1fd15ce338a6165a33e853704f7244051
-  data.tar.gz: 111bc6001f81947c1fd46e50a1d138487d864bc6
+  metadata.gz: f370253a680adefd979f61cee26d87eedb36e3f8
+  data.tar.gz: bfd12ef42e11ce1cd51bee4bf356f9014ad07a67
 SHA512:
-  metadata.gz: 7490512b67ea3bcdd2dcf60c12e29f8a2aa2f7232532a5d7f5a3e878c15913122bb7ee03b8c82b352d64b3224a90d900af3e2db94415ab694d5dce08eddd0193
-  data.tar.gz: 63e06d74b7148efc7c2ec31272ac15a999b921556ac08ef8f46807b110c22de0609f9a17573bb850a8ab2d6da7f88f90db2a1f6d9adb0cb64afd9ad70f07930b
+  metadata.gz: 671183d082caca469f7ded93eb737d07658beed0cce472806efeac2f303e810bd86049461a5ec5daef1d6d68436355bc4af81056ca0ee0c30a10f2a0047d86b9
+  data.tar.gz: d99f68afa553f93a280d0c648a24565c16bc1449af6d045cb31efc666e98c6c2d3406c1cf6879d77957a0513906fb87eb53a67c6133d37df523b81652aba87d9

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.6.13)
+    fluent-plugin-google-cloud (0.6.14)
       fluentd (~> 0.10)
-      google-api-client (~> 0.14)
-      google-cloud-logging (~> 1.2.3)
+      google-api-client (~> 0.17)
+      google-cloud-logging (~> 1.3, >= 1.3.2)
       googleapis-common-protos (~> 1.3)
-      googleauth (~> 0.5)
-      grpc (~> 1.2.5)
+      googleauth (~> 0.6)
+      grpc (~> 1.0)
       json (~> 1.8)
 
 GEM
@@ -15,13 +15,13 @@ GEM
   specs:
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
-    ast (2.3.0)
+    ast (2.4.0)
     cool.io (1.5.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     declarative (0.0.10)
     declarative-option (0.1.0)
-    faraday (0.13.1)
+    faraday (0.14.0)
       multipart-post (>= 1.2, < 3)
     fluentd (0.14.25)
       cool.io (>= 1.4.5, < 2.0.0)
@@ -34,7 +34,7 @@ GEM
       tzinfo (~> 1.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
-    google-api-client (0.19.3)
+    google-api-client (0.19.4)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (>= 0.5, < 0.7.0)
       httpclient (>= 2.8.1, < 3.0)
@@ -45,15 +45,15 @@ GEM
       google-cloud-env (~> 1.0)
     google-cloud-env (1.0.1)
       faraday (~> 0.11)
-    google-cloud-logging (1.2.3)
-      google-cloud-core (~> 1.0)
-      google-gax (~> 0.8.0)
+    google-cloud-logging (1.4.0)
+      google-cloud-core (~> 1.1)
+      google-gax (~> 1.0)
       stackdriver-core (~> 1.2)
-    google-gax (0.8.10)
+    google-gax (1.0.1)
       google-protobuf (~> 3.2)
-      googleapis-common-protos (~> 1.3.5)
-      googleauth (~> 0.5.1)
-      grpc (~> 1.0)
+      googleapis-common-protos (>= 1.3.5, < 2.0)
+      googleauth (~> 0.6.2)
+      grpc (>= 1.7.2, < 2.0)
       rly (~> 0.2.3)
     google-protobuf (3.5.1.1)
     googleapis-common-protos (1.3.7)
@@ -62,22 +62,23 @@ GEM
       grpc (~> 1.0)
     googleapis-common-protos-types (1.0.1)
       google-protobuf (~> 3.0)
-    googleauth (0.5.3)
+    googleauth (0.6.2)
       faraday (~> 0.12)
-      jwt (~> 1.4)
+      jwt (>= 1.4, < 3.0)
       logging (~> 2.0)
       memoist (~> 0.12)
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
-    grpc (1.2.5)
+    grpc (1.8.3)
       google-protobuf (~> 3.1)
-      googleauth (~> 0.5.1)
+      googleapis-common-protos-types (~> 1.0.0)
+      googleauth (>= 0.5.1, < 0.7)
     hashdiff (0.3.7)
     http_parser.rb (0.6.0)
     httpclient (2.8.3)
     json (1.8.6)
-    jwt (1.5.6)
+    jwt (2.1.0)
     little-plugger (1.1.4)
     logging (2.2.2)
       little-plugger (~> 1.1)
@@ -134,7 +135,7 @@ GEM
     thread_safe (0.3.6)
     tzinfo (1.2.4)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2017.3)
+    tzinfo-data (1.2018.3)
       tzinfo (>= 1.0.0)
     uber (0.1.0)
     unicode-display_width (1.3.0)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.13'
+  gem.version       = '0.6.14'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -21,10 +21,10 @@ eos
 
   gem.add_runtime_dependency 'fluentd', '~> 0.10'
   gem.add_runtime_dependency 'googleapis-common-protos', '~> 1.3'
-  gem.add_runtime_dependency 'google-api-client', '~> 0.14'
-  gem.add_runtime_dependency 'google-cloud-logging', '~> 1.2.3'
-  gem.add_runtime_dependency 'googleauth', '~> 0.5'
-  gem.add_runtime_dependency 'grpc', '~> 1.2.5'
+  gem.add_runtime_dependency 'google-api-client', '~> 0.17'
+  gem.add_runtime_dependency 'google-cloud-logging', '~> 1.3', '>= 1.3.2'
+  gem.add_runtime_dependency 'googleauth', '~> 0.6'
+  gem.add_runtime_dependency 'grpc', '~> 1.0'
   gem.add_runtime_dependency 'json', '~> 1.8'
 
   gem.add_development_dependency 'mocha', '~> 1.1'

data/lib/fluent/plugin/out_google_cloud.rb

@@ -120,6 +120,7 @@ module Fluent
       DEFAULT_SOURCE_LOCATION_KEY =
         'logging.googleapis.com/sourceLocation'.freeze
       DEFAULT_TRACE_KEY = 'logging.googleapis.com/trace'.freeze
+      DEFAULT_SPAN_ID_KEY = 'logging.googleapis.com/spanId'.freeze
 
       DEFAULT_METADATA_AGENT_URL =
         'http://local-metadata-agent.stackdriver.com:8000'.freeze
@@ -127,6 +128,8 @@ module Fluent
 
     # Internal constants.
     module InternalConstants
+      DEFAULT_LOGGING_API_URL = 'https://logging.googleapis.com'.freeze
+
       # The label name of local_resource_id in the json payload. When a record
       # has this field in the payload, we will use the value to retrieve
       # monitored resource from Stackdriver Metadata agent.
@@ -195,7 +198,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
-    PLUGIN_VERSION = '0.6.13'.freeze
+    PLUGIN_VERSION = '0.6.14'.freeze
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
@@ -219,6 +222,9 @@ module Fluent
     # Whether to attempt to obtain metadata from the local metadata service.
     # It is safe to specify 'true' even on platforms with no metadata service.
     config_param :use_metadata_service, :bool, :default => true
+    # A compatibility option to enable the legacy behavior of setting the AWS
+    # location to the availability zone rather than the region.
+    config_param :use_aws_availability_zone, :bool, :default => true
     # These parameters override any values obtained from the metadata service.
     config_param :project_id, :string, :default => nil
     config_param :zone, :string, :default => nil
@@ -232,6 +238,7 @@ module Fluent
     config_param :source_location_key, :string, :default =>
       DEFAULT_SOURCE_LOCATION_KEY
     config_param :trace_key, :string, :default => DEFAULT_TRACE_KEY
+    config_param :span_id_key, :string, :default => DEFAULT_SPAN_ID_KEY
 
     # Whether to try to detect if the record is a text log entry with JSON
     # content that needs to be parsed.
@@ -323,6 +330,12 @@ module Fluent
                  :default => nil,
                  :secret => true
 
+    # The URL of Stackdriver Logging API. Right now this only works with the
+    # gRPC path (use_grpc = true). An unsecured channel is used if the URL
+    # scheme is 'http' instead of 'https'. One common use case of this config is
+    # to provide a mocked / stubbed Logging API, e.g., http://localhost:52000.
+    config_param :logging_api_url, :string, :default => DEFAULT_LOGGING_API_URL
+
     # Whether to collect metrics about the plugin usage. The mechanism for
     # collecting and exposing metrics is controlled by the monitoring_type
     # parameter.
@@ -368,11 +381,11 @@ module Fluent
 
       # TODO(qingling128): Remove this warning after the support is added. Also
       # remove the comment in the description of this configuration.
-      if @partial_success && @use_grpc
-        @log.warn 'Detected partial_success enabled while use_grpc is also' \
-                  ' enabled. The support for partial success in the gRPC path' \
-                  ' is to be added in the near future. For now the ' \
-                  ' partial_success flag will be ignored.'
+      unless @logging_api_url == DEFAULT_LOGGING_API_URL || @use_grpc
+        @log.warn 'Detected customized logging_api_url while use_grpc is not' \
+                  ' enabled. Customized logging_api_url for the non-gRPC path' \
+                  ' is not supported. The logging_api_url option will be' \
+                  ' ignored.'
       end
 
       # If monitoring is enabled, register metrics in the default registry
@@ -397,7 +410,7 @@ module Fluent
           'The number of log entries that failed to be ingested by the'\
             ' Stackdriver output plugin due to a transient error and were'\
             ' retried')
-        @ok_code = @use_grpc ? 0 : 200
+        @ok_code = @use_grpc ? GRPC::Core::StatusCodes::OK : 200
       end
 
       # Alert on old authentication configuration.
@@ -540,6 +553,9 @@ module Fluent
           fq_trace_id = record.delete(@trace_key)
           entry.trace = fq_trace_id if fq_trace_id
 
+          span_id = record.delete(@span_id_key)
+          entry.span_id = span_id if span_id
+
           set_log_entry_fields(record, entry)
           set_payload(entry_level_resource.type, record, entry, is_json)
 
@@ -634,7 +650,6 @@ module Fluent
       client = api_client
       entries_count = entries.length
       client.write_log_entries(
-        # Ignore partial_success for gRPC path.
         entries,
         log_name: log_name,
         # Leave resource nil if it's nil.
@@ -646,7 +661,8 @@ module Fluent
                   end,
         labels: labels.map do |k, v|
           [k.encode('utf-8'), convert_to_utf8(v)]
-        end.to_h
+        end.to_h,
+        partial_success: @partial_success
       )
       increment_successful_requests_count
       increment_ingested_entries_count(entries_count)
@@ -683,14 +699,10 @@ module Fluent
       # Most client errors indicate a problem with the request itself and
       # should not be retried.
       when \
-          # HTTP status 400 (Bad Request).
-          GRPC::InvalidArgument,
           # HTTP status 401 (Unauthorized).
-          # These are usually solved via a `gcloud auth` call, or by
-          # modifying the permissions on the Google Cloud project.
+          # These are usually solved via a `gcloud auth` call, or by modifying
+          # the permissions on the Google Cloud project.
           GRPC::Unauthenticated,
-          # HTTP status 403 (Forbidden).
-          GRPC::PermissionDenied,
           # HTTP status 404 (Not Found).
           GRPC::NotFound,
           # HTTP status 409 (Conflict).
@@ -711,6 +723,33 @@ module Fluent
         @log.warn "Dropping #{entries_count} log message(s)",
                   error: error.to_s, error_code: error.code.to_s
 
+      # If partial_success is enabled, valid entries should have be
+      # written even if some other entries fail due to InvalidArgument or
+      # PermissionDenied errors. Only invalid entries will be dropped.
+      when \
+          # HTTP status 400 (Bad Request).
+          GRPC::InvalidArgument,
+          # HTTP status 403 (Forbidden).
+          GRPC::PermissionDenied
+        error_details_map = construct_error_details_map_grpc(gax_error)
+        if error_details_map.empty?
+          increment_dropped_entries_count(entries_count, error.code)
+          @log.warn "Dropping #{entries_count} log message(s)",
+                    error: error.to_s, error_code: error.code.to_s
+        else
+          error_details_map.each do |(error_code, error_message), indexes|
+            partial_errors_count = indexes.length
+            increment_dropped_entries_count(partial_errors_count,
+                                            error_code)
+            entries_count -= partial_errors_count
+            @log.warn "Dropping #{partial_errors_count} log message(s)",
+                      error: error_message, error_code: error_code.to_s
+          end
+          # Consider partially successful requests successful.
+          increment_successful_requests_count
+          increment_ingested_entries_count(entries_count)
+        end
+
       else
         # Assume it's a problem with the request itself and don't retry.
         increment_failed_requests_count(error.code)
@@ -771,7 +810,7 @@ module Fluent
       increment_failed_requests_count(error.status_code)
       increment_dropped_entries_count(entries_count, error.status_code)
       @log.warn "Dropping #{entries_count} log message(s)",
-                error_class: error.class.to_s, error: error.to_s
+                error: error.to_s, error_code: error.status_code.to_s
 
     rescue Google::Apis::ClientError => error
       # 4xx client errors. Most client errors indicate a problem with the
@@ -781,15 +820,15 @@ module Fluent
         increment_failed_requests_count(error.status_code)
         increment_dropped_entries_count(entries_count, error.status_code)
         @log.warn "Dropping #{entries_count} log message(s)",
-                  error_class: error.class.to_s, error: error.to_s
+                  error: error.to_s, error_code: error.status_code.to_s
       else
         error_details_map.each do |(error_code, error_message), indexes|
-          partial_error_count = indexes.length
-          increment_dropped_entries_count(partial_error_count, error_code)
-          entries_count -= partial_error_count
-          @log.warn "Dropping #{partial_error_count} log message(s)",
-                    error_code: "google.rpc.Code[#{error_code}]",
-                    error: error_message
+          partial_errors_count = indexes.length
+          increment_dropped_entries_count(partial_errors_count, error_code)
+          entries_count -= partial_errors_count
+          @log.warn "Dropping #{partial_errors_count} log message(s)",
+                    error: error_message,
+                    error_code: "google.rpc.Code[#{error_code}]"
         end
         # Consider partially successful requests successful.
         increment_successful_requests_count
@@ -945,8 +984,13 @@ module Fluent
       # Response format: "projects/<number>/zones/<zone>"
       @zone ||= fetch_gce_metadata('instance/zone').rpartition('/')[2] if
         @platform == Platform::GCE
-      @zone ||= 'aws:' + ec2_metadata['availabilityZone'] if
-        @platform == Platform::EC2 && ec2_metadata.key?('availabilityZone')
+      aws_location_key = if @use_aws_availability_zone
+                           'availabilityZone'
+                         else
+                           'region'
+                         end
+      @zone ||= 'aws:' + ec2_metadata[aws_location_key] if
+        @platform == Platform::EC2 && ec2_metadata.key?(aws_location_key)
     rescue StandardError => e
       @log.error 'Failed to obtain location: ', error: e
     end
@@ -1828,13 +1872,24 @@ module Fluent
 
     def init_api_client
       if @use_grpc
-        ssl_creds = GRPC::Core::ChannelCredentials.new
-        authentication = Google::Auth.get_application_default
-        creds = GRPC::Core::CallCredentials.new(authentication.updater_proc)
-        creds = ssl_creds.compose(creds)
+        uri = URI.parse(@logging_api_url)
+        host = uri.host
+        unless host
+          raise Fluent::ConfigError,
+                'The logging_api_url option specifies an invalid URL:' \
+                " #{@logging_api_url}."
+        end
+        if uri.scheme == 'https'
+          ssl_creds = GRPC::Core::ChannelCredentials.new
+          authentication = Google::Auth.get_application_default
+          creds = GRPC::Core::CallCredentials.new(authentication.updater_proc)
+          creds = ssl_creds.compose(creds)
+        else
+          creds = :this_channel_is_insecure
+        end
+        port = ":#{uri.port}" if uri.port
         @client = Google::Cloud::Logging::V2::LoggingServiceV2Client.new(
-          channel: GRPC::Core::Channel.new(
-            'logging.googleapis.com', nil, creds))
+          credentials: GRPC::Core::Channel.new("#{host}#{port}", nil, creds))
       else
         # TODO: Use a non-default ClientOptions object.
         Google::Apis::ClientOptions.default.application_name = PLUGIN_NAME
@@ -1885,8 +1940,8 @@ module Fluent
       end
     end
 
-    # Extract a map of error details from an potentially partially successful
-    # request. Return an empty map if @partial_success is not enabled.
+    # Extract a map of error details from a potentially partially successful
+    # REST request. Return an empty map if @partial_success is not enabled.
     #
     # The keys in this map are [error_code, error_message] pairs, and the values
     # are a list of stringified indexes of log entries that failed due to this
@@ -1989,6 +2044,65 @@ module Fluent
       {}
     end
 
+    # Extract a map of error details from a potentially partially successful
+    # gRPC request. Return an empty map if @partial_success is not enabled.
+    #
+    # The keys in this map are [error_code, error_message] pairs, and the values
+    # are a list of indexes of log entries that failed due to this error.
+    #
+    # A sample error looks like:
+    # <Google::Gax::RetryError:
+    #   message: 'GaxError Exception occurred in retry method that was not class
+    #             ified as transient, caused by 7:User not authorized.',
+    #   details: [
+    #     <Google::Logging::V2::WriteLogEntriesPartialErrors:
+    #       log_entry_errors: {
+    #         0 => <Google::Rpc::Status:
+    #                code: 7,
+    #                message: "User not authorized.",
+    #                details: []>,
+    #         1 => <Google::Rpc::Status:
+    #                code: 3,
+    #                message: "Log name contains illegal character :",
+    #                details: []>,
+    #         3 => <Google::Rpc::Status:
+    #                code: 3,
+    #                message: "Log name contains illegal character :",
+    #                details: []>
+    #       }
+    #     >,
+    #     <Google::Rpc::DebugInfo:
+    #       stack_entries: [],
+    #       detail: "..."
+    #     >
+    #   ]
+    #   cause: <GRPC::PermissionDenied: 7:User not authorized.>
+    # }
+    #
+    # The ultimate map that is constructed is:
+    # {
+    #   [7, 'User not authorized.']: [0],
+    #   [3, 'Log name contains illegal character :']: [1, 3]
+    # }
+    def construct_error_details_map_grpc(gax_error)
+      return {} unless @partial_success
+      error_details_map = Hash.new { |h, k| h[k] = [] }
+
+      error_details = ensure_array(gax_error.status_details)
+      raise JSON::ParserError, 'The error details are empty.' if
+        error_details.empty?
+      log_entry_errors = ensure_hash(error_details[0].log_entry_errors)
+      log_entry_errors.each do |index, log_entry_error|
+        error_key = [log_entry_error[:code], log_entry_error[:message]].freeze
+        error_details_map[error_key] << index
+      end
+      error_details_map
+    rescue JSON::ParserError => e
+      @log.warn 'Failed to extract log entry errors from the error details:' \
+                " #{gax_error.details.inspect}.", error: e
+      {}
+    end
+
     def ensure_array(value)
       Array.try_convert(value) || (raise JSON::ParserError, value.class.to_s)
     end

data/test/plugin/base_test.rb

@@ -62,6 +62,17 @@ module BaseTest
     assert_equal 1, exception_count
   end
 
+  def test_configure_logging_api_url
+    setup_gce_metadata_stubs
+    {
+      APPLICATION_DEFAULT_CONFIG => DEFAULT_LOGGING_API_URL,
+      CUSTOM_LOGGING_API_URL_CONFIG => CUSTOM_LOGGING_API_URL
+    }.each do |(config, url)|
+      d = create_driver(config)
+      assert_equal url, d.instance.instance_variable_get(:@logging_api_url)
+    end
+  end
+
   def test_configure_custom_metadata
     setup_no_metadata_service_stubs
     d = create_driver(CUSTOM_METADATA_CONFIG)
@@ -163,7 +174,9 @@ module BaseTest
       CONFIG_EC2_PROJECT_ID =>
         ['ec2', EC2_PROJECT_ID, EC2_PREFIXED_ZONE, EC2_VM_ID],
       CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID =>
-        ['ec2', EC2_PROJECT_ID, EC2_PREFIXED_ZONE, CUSTOM_VM_ID]
+        ['ec2', EC2_PROJECT_ID, EC2_PREFIXED_ZONE, CUSTOM_VM_ID],
+      CONFIG_EC2_PROJECT_ID_USE_REGION =>
+        ['ec2', EC2_PROJECT_ID, EC2_PREFIXED_REGION, EC2_VM_ID]
     }.each_with_index do |(config, parts), index|
       send("setup_#{parts[0]}_metadata_stubs")
       d = create_driver(config)
@@ -257,7 +270,18 @@ module BaseTest
       d.emit('message' => log_entry(0))
       d.run
     end
-    verify_log_entries(1, EC2_PARAMS)
+    verify_log_entries(1, EC2_ZONE_PARAMS)
+  end
+
+  def test_one_log_ec2_region
+    ENV['GOOGLE_APPLICATION_CREDENTIALS'] = IAM_CREDENTIALS[:path]
+    setup_ec2_metadata_stubs
+    setup_logging_stubs do
+      d = create_driver(CONFIG_EC2_PROJECT_ID_USE_REGION)
+      d.emit('message' => log_entry(0))
+      d.run
+    end
+    verify_log_entries(1, EC2_REGION_PARAMS)
   end
 
   def test_structured_payload_log
@@ -1154,55 +1178,14 @@ module BaseTest
   end
 
   def test_log_entry_trace_field
-    setup_gce_metadata_stubs
-    message = log_entry(0)
-    trace = 'projects/project-1/traces/1234567890abcdef1234567890abcdef'
-    [
-      {
-        # It leaves trace entry field nil if no trace value sent.
-        driver_config: APPLICATION_DEFAULT_CONFIG,
-        emitted_log: { 'msg' => message },
-        expected_fields: { 'msg' => message },
-        expected_trace_value: nil
-      },
-      {
-        # By default, it sets trace via Google-specific key.
-        driver_config: APPLICATION_DEFAULT_CONFIG,
-        emitted_log: { 'msg' => message, DEFAULT_TRACE_KEY => trace },
-        expected_fields: { 'msg' => message },
-        expected_trace_value: trace
-      },
-      {
-        # It allows setting the trace via a custom configured key.
-        driver_config: CONFIG_CUSTOM_TRACE_KEY_SPECIFIED,
-        emitted_log: { 'msg' => message, 'custom_trace_key' => trace },
-        expected_fields: { 'msg' => message },
-        expected_trace_value: trace
-      },
-      {
-        # It no longer sets trace by the default key if custom key specified.
-        driver_config: CONFIG_CUSTOM_TRACE_KEY_SPECIFIED,
-        emitted_log: { 'msg' => message, DEFAULT_TRACE_KEY => trace },
-        expected_fields: { 'msg' => message, DEFAULT_TRACE_KEY => trace },
-        expected_trace_value: nil
-      }
-    ].each do |input|
-      setup_logging_stubs do
-        @logs_sent = []
-        d = create_driver(input[:driver_config])
-        d.emit(input[:emitted_log])
-        d.run
-      end
-      verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
-        assert_equal input[:expected_trace_value], entry['trace'], input
+    verify_field_key('trace', DEFAULT_TRACE_KEY, 'custom_trace_key',
+                     CONFIG_CUSTOM_TRACE_KEY_SPECIFIED,
+                     'projects/proj1/traces/1234567890abcdef1234567890abcdef')
+  end
 
-        fields = get_fields(entry['jsonPayload'])
-        assert_equal input[:expected_fields].size, fields.size, input
-        fields.each do |key, value|
-          assert_equal input[:expected_fields][key], get_string(value), input
-        end
-      end
-    end
+  def test_log_entry_span_id_field
+    verify_field_key('spanId', DEFAULT_SPAN_ID_KEY, 'custom_span_id_key',
+                     CONFIG_CUSTOM_SPAN_ID_KEY_SPECIFIED, '000000000000004a')
   end
 
   # Metadata Agent related tests.
@@ -1765,6 +1748,57 @@ module BaseTest
     end
   end
 
+  def verify_field_key(log_entry_field, default_key, custom_key,
+                       custom_key_config, sample_value)
+    setup_gce_metadata_stubs
+    message = log_entry(0)
+    [
+      {
+        # It leaves log entry field nil if no keyed value sent.
+        driver_config: APPLICATION_DEFAULT_CONFIG,
+        emitted_log: { 'msg' => message },
+        expected_payload: { 'msg' => message },
+        expected_field_value: nil
+      },
+      {
+        # By default, it sets log entry field via a default key.
+        driver_config: APPLICATION_DEFAULT_CONFIG,
+        emitted_log: { 'msg' => message, default_key => sample_value },
+        expected_payload: { 'msg' => message },
+        expected_field_value: sample_value
+      },
+      {
+        # It allows setting the log entry field via a custom configured key.
+        driver_config: custom_key_config,
+        emitted_log: { 'msg' => message, custom_key => sample_value },
+        expected_payload: { 'msg' => message },
+        expected_field_value: sample_value
+      },
+      {
+        # It doesn't set log entry field by default key if custom key specified.
+        driver_config: custom_key_config,
+        emitted_log: { 'msg' => message, default_key => sample_value },
+        expected_payload: { 'msg' => message, default_key => sample_value },
+        expected_field_value: nil
+      }
+    ].each do |input|
+      setup_logging_stubs do
+        @logs_sent = []
+        d = create_driver(input[:driver_config])
+        d.emit(input[:emitted_log])
+        d.run
+      end
+      verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+        assert_equal input[:expected_field_value], entry[log_entry_field], input
+        payload_fields = get_fields(entry['jsonPayload'])
+        assert_equal input[:expected_payload].size, payload_fields.size, input
+        payload_fields.each do |key, value|
+          assert_equal input[:expected_payload][key], get_string(value), input
+        end
+      end
+    end
+  end
+
   def http_request_message
     HTTP_REQUEST_MESSAGE
   end

data/test/plugin/constants.rb

@@ -20,6 +20,7 @@ module Constants
 
   # Generic attributes.
   HOSTNAME = Socket.gethostname
+  CUSTOM_LOGGING_API_URL = 'http://localhost:52000'.freeze
 
   # TODO(qingling128) Separate constants into different submodules.
   # Attributes used for the GCE metadata service.
@@ -43,6 +44,8 @@ module Constants
   EC2_PROJECT_ID = 'test-ec2-project-id'.freeze
   EC2_ZONE = 'us-west-2b'.freeze
   EC2_PREFIXED_ZONE = "aws:#{EC2_ZONE}".freeze
+  EC2_REGION = 'us-west-2'.freeze
+  EC2_PREFIXED_REGION = "aws:#{EC2_REGION}".freeze
   EC2_VM_ID = 'i-81c16767'.freeze
   EC2_ACCOUNT_ID = '123456789012'.freeze
 
@@ -50,6 +53,7 @@ module Constants
   EC2_IDENTITY_DOCUMENT = %({
     "accountId" : "#{EC2_ACCOUNT_ID}",
     "availabilityZone" : "#{EC2_ZONE}",
+    "region" : "#{EC2_REGION}",
     "instanceId" : "#{EC2_VM_ID}"
   }).freeze
 
@@ -138,6 +142,10 @@ module Constants
   APPLICATION_DEFAULT_CONFIG = %(
   ).freeze
 
+  CUSTOM_LOGGING_API_URL_CONFIG = %(
+    logging_api_url #{CUSTOM_LOGGING_API_URL}
+  ).freeze
+
   DETECT_JSON_CONFIG = %(
     detect_json true
   ).freeze
@@ -220,6 +228,11 @@ module Constants
     vm_id #{CUSTOM_VM_ID}
   ).freeze
 
+  CONFIG_EC2_PROJECT_ID_USE_REGION = %(
+    project_id #{EC2_PROJECT_ID}
+    use_aws_availability_zone false
+  ).freeze
+
   CONFIG_DATAFLOW = %(
     subservice_name "#{DATAFLOW_CONSTANTS[:service]}"
     labels {
@@ -244,6 +257,10 @@ module Constants
     trace_key custom_trace_key
   ).freeze
 
+  CONFIG_CUSTOM_SPAN_ID_KEY_SPECIFIED = %(
+    span_id_key custom_span_id_key
+  ).freeze
+
   # Service configurations for various services.
 
   # GCE.
@@ -506,12 +523,12 @@ module Constants
     }
   }.freeze
 
-  EC2_PARAMS = {
+  EC2_REGION_PARAMS = {
     resource: {
       type: EC2_CONSTANTS[:resource_type],
       labels: {
         'instance_id' => EC2_VM_ID,
-        'region' => EC2_PREFIXED_ZONE,
+        'region' => EC2_PREFIXED_REGION,
         'aws_account' => EC2_ACCOUNT_ID
       }
     },
@@ -522,6 +539,14 @@ module Constants
     }
   }.freeze
 
+  EC2_ZONE_PARAMS = EC2_REGION_PARAMS.merge(
+    resource: EC2_REGION_PARAMS[:resource].merge(
+      labels: EC2_REGION_PARAMS[:resource][:labels].merge(
+        'region' => EC2_PREFIXED_ZONE
+      )
+    )
+  ).freeze
+
   HTTP_REQUEST_MESSAGE = {
     'requestMethod' => 'POST',
     'requestUrl' => 'http://example/',
@@ -651,4 +676,46 @@ module Constants
       ]
     }
   }.freeze
+
+  # rubocop:disable Style/StringLiterals
+  PARTIAL_SUCCESS_GRPC_METADATA = {
+    'google.logging.v2.writelogentriespartialerrors-bin' =>
+      Google::Logging::V2::WriteLogEntriesPartialErrors.encode(
+        Google::Logging::V2::WriteLogEntriesPartialErrors.new(
+          log_entry_errors: {
+            0 => Google::Rpc::Status.new(
+              code: GRPC::Core::StatusCodes::PERMISSION_DENIED,
+              message: "User not authorized.",
+              details: []),
+            1 => Google::Rpc::Status.new(
+              code: GRPC::Core::StatusCodes::INVALID_ARGUMENT,
+              message: "Log name contains illegal character :",
+              details: []),
+            3 => Google::Rpc::Status.new(
+              code: GRPC::Core::StatusCodes::INVALID_ARGUMENT,
+              message: "Log name contains illegal character :",
+              details: []) })),
+    'google.rpc.debuginfo-bin' =>
+      "\x12\xA7\x03[ORIGINAL ERROR] generic::permission_denied: User not auth" \
+      "orized. [google.rpc.error_details_ext] { message: \"User not authorize" \
+      "d.\" details { type_url: \"type.googleapis.com/google.logging.v2.Write" \
+      "LogEntriesPartialErrors\" value: \"\\n\\034\\010\\000\\022\\030\\010\\" \
+      "007\\022\\024User not authorized.\\n-\\010\\001\\022)\\010\\003\\022%L" \
+      "og name contains illegal character :\\n-\\010\\002\\022)\\010\\003\\02" \
+      "2%Log name contains illegal character :\" } }",
+    'grpc-status-details-bin' =>
+      "\b\a\x12\x14User not authorized.\x1A\xC2\x01\nBtype.googleapis.com/goo" \
+      "gle.logging.v2.WriteLogEntriesPartialErrors\x12|\n\x1C\b\x00\x12\x18\b" \
+      "\a\x12\x14User not authorized.\n-\b\x01\x12)\b\x03\x12%Log name contai" \
+      "ns illegal character :\n-\b\x02\x12)\b\x03\x12%Log name contains illeg" \
+      "al character :\x1A\xD7\x03\n(type.googleapis.com/google.rpc.DebugInfo" \
+      "\x12\xAA\x03\x12\xA7\x03[ORIGINAL ERROR] generic::permission_denied: U" \
+      "ser not authorized. [google.rpc.error_details_ext] { message: \"User n" \
+      "ot authorized.\" details { type_url: \"type.googleapis.com/google.logg" \
+      "ing.v2.WriteLogEntriesPartialErrors\" value: \"\\n\\034\\010\\000\\022" \
+      "\\030\\010\\007\\022\\024User not authorized.\\n-\\010\\001\\022)\\010" \
+      "\\003\\022%Log name contains illegal character :\\n-\\010\\002\\022)" \
+      "\\010\\003\\022%Log name contains illegal character :\" } }"
+  }.freeze
+  # rubocop:enable Style/StringLiterals
 end

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -51,6 +51,39 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
   end
 
+  def test_partial_success
+    setup_gce_metadata_stubs
+    setup_prometheus
+    setup_logging_stubs(
+      true, GRPC::Core::StatusCodes::PERMISSION_DENIED,
+      'User not authorized.', PARTIAL_SUCCESS_GRPC_METADATA) do
+      # The API Client should not retry this and the plugin should consume
+      # the exception.
+      d = create_driver(PROMETHEUS_ENABLE_CONFIG + PARTIAL_SUCCESS_CONFIG)
+      4.times do |i|
+        d.emit('message' => log_entry(i.to_s))
+      end
+      d.run
+      assert_prometheus_metric_value(
+        :stackdriver_successful_requests_count, 1,
+        grpc: true, code: GRPC::Core::StatusCodes::OK)
+      assert_prometheus_metric_value(
+        :stackdriver_failed_requests_count, 0,
+        grpc: true, code: GRPC::Core::StatusCodes::PERMISSION_DENIED)
+      assert_prometheus_metric_value(
+        :stackdriver_ingested_entries_count, 1,
+        grpc: true, code: GRPC::Core::StatusCodes::OK)
+      assert_prometheus_metric_value(
+        :stackdriver_dropped_entries_count, 2,
+        grpc: true, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT)
+      assert_prometheus_metric_value(
+        :stackdriver_dropped_entries_count, 1,
+        grpc: true, code: GRPC::Core::StatusCodes::PERMISSION_DENIED)
+      assert_prometheus_metric_value(
+        :stackdriver_retried_entries_count, 0, grpc: true)
+    end
+  end
+
   def test_server_error
     setup_gce_metadata_stubs
     {
@@ -246,11 +279,6 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
 
     def api_client
-      ssl_creds = GRPC::Core::ChannelCredentials.new
-      authentication = Google::Auth.get_application_default
-      creds = GRPC::Core::CallCredentials.new(authentication.updater_proc)
-      ssl_creds.compose(creds)
-
       @grpc_stub
     end
   end
@@ -263,12 +291,17 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
       @requests_received = requests_received
     end
 
-    def write_log_entries(entries, log_name: nil, resource: nil, labels: nil)
+    def write_log_entries(entries,
+                          log_name: nil,
+                          resource: nil,
+                          labels: nil,
+                          partial_success: nil)
       request = Google::Apis::LoggingV2::WriteLogEntriesRequest.new(
         log_name: log_name,
         resource: resource,
         labels: labels,
-        entries: entries
+        entries: entries,
+        partial_success: partial_success
       )
       @requests_received << request
       WriteLogEntriesResponse.new
@@ -278,18 +311,23 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
   # GRPC logging mock that fails and returns server side or client side errors.
   class GRPCLoggingMockFailingService <
       Google::Cloud::Logging::V2::LoggingServiceV2Client
-    def initialize(code, message, failed_attempts)
+    def initialize(code, message, metadata, failed_attempts)
       @code = code
       @message = message
+      @metadata = metadata
       @failed_attempts = failed_attempts
       super()
     end
 
     # rubocop:disable Lint/UnusedMethodArgument
-    def write_log_entries(entries, log_name: nil, resource: nil, labels: nil)
+    def write_log_entries(entries,
+                          log_name: nil,
+                          resource: nil,
+                          labels: nil,
+                          partial_success: nil)
       @failed_attempts << 1
       begin
-        raise GRPC::BadStatus.new_status_exception(@code, @message)
+        raise GRPC::BadStatus.new_status_exception(@code, @message, @metadata)
       rescue
         # Google::Gax::GaxError will wrap the latest thrown exception as @cause.
         raise Google::Gax::GaxError, @message
@@ -299,11 +337,14 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
   end
 
   # Set up grpc stubs to mock the external calls.
-  def setup_logging_stubs(should_fail = false, code = nil, message = nil)
+  def setup_logging_stubs(should_fail = false,
+                          code = nil,
+                          message = nil,
+                          metadata = {})
     if should_fail
       @failed_attempts = []
       @grpc_stub = GRPCLoggingMockFailingService.new(
-        code, message, @failed_attempts)
+        code, message, metadata, @failed_attempts)
     else
       @requests_sent = []
       @grpc_stub = GRPCLoggingMockService.new(@requests_sent)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.13
+  version: 0.6.14
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-18 00:00:00.000000000 Z
+date: 2018-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -45,56 +45,62 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.14'
+        version: '0.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.14'
+        version: '0.17'
 - !ruby/object:Gem::Dependency
   name: google-cloud-logging
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.3
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.3
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.2
 - !ruby/object:Gem::Dependency
   name: googleauth
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: grpc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.5
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.5
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement