fluent-plugin-google-cloud 0.6.20 → 0.6.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 38cbf8464d2ca792e5d134d1edd66a2a4f6654ca
-  data.tar.gz: 2d15be19df0ca83a865c0cea38a4199f6ef42b7e
+  metadata.gz: 33b71adc0c744f682e9f3548911ec4f550988864
+  data.tar.gz: 3c45d732c056e34e32f05f2708012f4b69205cca
 SHA512:
-  metadata.gz: 04eaab1746f9d66c14cd402fb6f7960affbc2e36ac79ccecc84eb00c8af19e5793f7c591d94cfa1765f7a3f0d8e1807ecd24238a4209f99d29635a61a4793e68
-  data.tar.gz: 6a692ee7e882ce8b36e119432229b4fcc3f7aa855c59f1e806943885c465165a10d664b0a42baac1577277039d94ddb0d8f05a7fa035196a55b894d8ad606d1c
+  metadata.gz: 9dab6cb5bcecc828bc025458f3adb04ba0690fcac1052c936a7e3beaebca7ea35de72295f694e73342c92a5cf1129be7bcc2e5040d405280bb23822c6f7521fb
+  data.tar.gz: 4ea689e785e16c15c4b30286203aaf54caa5e23aa1987a30252f75ec02278a41a3fa064170e4b5233de9f2104cdf1ae86f7012dd85c19f773c44e25034cfcf10

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.20'
+  gem.version       = '0.6.21'
   gem.authors       = ['Ling Huang', 'Igor Peshansky']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')

data/lib/fluent/plugin/out_google_cloud.rb

@@ -227,7 +227,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
-    PLUGIN_VERSION = '0.6.20'.freeze
+    PLUGIN_VERSION = '0.6.21'.freeze
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
@@ -396,6 +396,9 @@ module Fluent
     # requests when talking to Stackdriver Logging API.
     config_param :split_logs_by_tag, :bool, :default => false
 
+    # Whether to attempt adjusting invalid log entry timestamps.
+    config_param :adjust_invalid_timestamps, :bool, :default => true
+
     # rubocop:enable Style/HashSyntax
 
     # TODO: Add a log_name config option rather than just using the tag?
@@ -792,6 +795,7 @@ module Fluent
           GRPC::PermissionDenied
         error_details_map = construct_error_details_map_grpc(gax_error)
         if error_details_map.empty?
+          increment_failed_requests_count(error.code)
           increment_dropped_entries_count(entries_count, error.code)
           @log.warn "Dropping #{entries_count} log message(s)",
                     error: error.to_s, error_code: error.code.to_s
@@ -1249,27 +1253,10 @@ module Fluent
       # Examples:
       # "container.<container_id>" // Docker container.
       # "k8s_pod.<namespace_name>.<pod_name>" // GKE pod.
-      if @enable_metadata_agent && local_resource_id
-        @log.debug 'Calling metadata agent with local_resource_id: ' \
-                  "#{local_resource_id}."
-        retrieved_resource = query_metadata_agent_for_monitored_resource(
+      if local_resource_id
+        converted_resource = monitored_resource_from_local_resource_id(
           local_resource_id)
-        @log.debug 'Retrieved monitored resource from metadata agent: ' \
-                  "#{retrieved_resource.inspect}."
-        if retrieved_resource
-          resource = retrieved_resource
-          # TODO(qingling128): Fix this temporary renaming from 'gke_container'
-          # to 'container'.
-          resource.type = 'container' if resource.type == 'gke_container'
-        else
-          # TODO(qingling128): This entire else clause is temporary before we
-          # implement buffering and caching.
-          @log.warn('Failed to retrieve monitored resource from Metadata' \
-                     " Agent with local_resource_id #{local_resource_id}.")
-          constructed_k8s_resource = construct_k8s_resource_locally(
-            local_resource_id)
-          resource = constructed_k8s_resource if constructed_k8s_resource
-        end
+        resource = converted_resource if converted_resource
       end
 
       # Once the resource type is settled down, determine the labels.
@@ -1349,6 +1336,32 @@ module Fluent
       [resource, common_labels]
     end
 
+    # Take a locally unique resource id and convert it to the globally unique
+    # monitored resource.
+    def monitored_resource_from_local_resource_id(local_resource_id)
+      return unless local_resource_id
+      if @enable_metadata_agent
+        @log.debug 'Calling metadata agent with local_resource_id: ' \
+                  "#{local_resource_id}."
+        resource = query_metadata_agent_for_monitored_resource(
+          local_resource_id)
+        @log.debug 'Retrieved monitored resource from metadata agent: ' \
+                  "#{resource.inspect}."
+        if resource
+          # TODO(qingling128): Fix this temporary renaming from 'gke_container'
+          # to 'container'.
+          resource.type = 'container' if resource.type == 'gke_container'
+          return resource
+        end
+      end
+      # Fall back to constructing monitored resource locally.
+      # TODO(qingling128): This entire else clause is temporary until we
+      # implement buffering and caching.
+      @log.debug('Failed to retrieve monitored resource from Metadata' \
+                 " Agent with local_resource_id #{local_resource_id}.")
+      construct_k8s_resource_locally(local_resource_id)
+    end
+
     # Extract entry level monitored resource and common labels that should be
     # applied to individual entries.
     def determine_entry_level_monitored_resource_and_labels(
@@ -1578,22 +1591,22 @@ module Fluent
                    ts_nanos
                  end
 
-      # Adjust timestamps from the future.
-      # There are two cases:
-      # 1. The parsed timestamp is later in the current year:
-      # This can happen when system log lines from previous years are missing
-      # the year, so the date parser assumes the current year.
-      # We treat these lines as coming from last year.  This could label
-      # 2-year-old logs incorrectly, but this probably isn't super important.
-      #
-      # 2. The parsed timestamp is past the end of the current year:
-      # Since the year is different from the current year, this isn't the
-      # missing year in system logs.  It is unlikely that users explicitly
-      # write logs at a future date.  This could result from an unsynchronized
-      # clock on a VM, or some random value being parsed as the timestamp.
-      # We reset the timestamp on those lines to the default value and let the
-      # downstream API handle it.
-      if timestamp
+      if @adjust_invalid_timestamps && timestamp
+        # Adjust timestamps from the future.
+        # There are two cases:
+        # 1. The parsed timestamp is later in the current year:
+        # This can happen when system log lines from previous years are missing
+        # the year, so the date parser assumes the current year.
+        # We treat these lines as coming from last year. This could label
+        # 2-year-old logs incorrectly, but this probably isn't super important.
+        #
+        # 2. The parsed timestamp is past the end of the current year:
+        # Since the year is different from the current year, this isn't the
+        # missing year in system logs. It is unlikely that users explicitly
+        # write logs at a future date. This could result from an unsynchronized
+        # clock on a VM, or some random value being parsed as the timestamp.
+        # We reset the timestamp on those lines to the default value and let the
+        # downstream API handle it.
         next_year = Time.mktime(current_time.year + 1)
         one_day_later = current_time.to_datetime.next_day.to_time
         if timestamp >= next_year # Case 2.
@@ -1605,6 +1618,7 @@ module Fluent
           # The value of ts_nanos should not change when subtracting a year.
         end
       end
+
       [ts_secs, ts_nanos]
     end
 
@@ -2172,6 +2186,9 @@ module Fluent
       error_details = ensure_array(gax_error.status_details)
       raise JSON::ParserError, 'The error details are empty.' if
         error_details.empty?
+      raise JSON::ParserError, 'No partial error info in error details.' unless
+        error_details[0].is_a?(
+          Google::Logging::V2::WriteLogEntriesPartialErrors)
       log_entry_errors = ensure_hash(error_details[0].log_entry_errors)
       log_entry_errors.each do |index, log_entry_error|
         error_key = [log_entry_error[:code], log_entry_error[:message]].freeze

data/test/plugin/base_test.rb

@@ -724,58 +724,65 @@ module BaseTest
     one_second_into_next_year = next_year + 1
     one_day_into_next_year = next_year.to_date.next_day.to_time
     {
-      Time.at(123_456.789) => Time.at(123_456.789),
-      Time.at(0) => Time.at(0),
-      current_time => current_time,
-      one_second_before_next_year => adjusted_to_last_year,
-      next_year => Time.at(0),
-      one_second_into_next_year => Time.at(0),
-      one_day_into_next_year => Time.at(0)
-    }.each do |ts, adjusted_ts|
-      expected_ts = []
-      emit_index = 0
-      setup_logging_stubs do
-        @logs_sent = []
-        d = create_driver
-        # Test the "native" fluentd timestamp as well as our nanosecond tags.
-        d.emit({ 'message' => log_entry(emit_index) }, ts.to_f)
-        expected_ts.push(adjusted_ts)
-        emit_index += 1
-        d.emit('message' => log_entry(emit_index),
-               'timeNanos' => ts.tv_sec * 1_000_000_000 + ts.tv_nsec)
-        expected_ts.push(adjusted_ts)
-        emit_index += 1
-        d.emit('message' => log_entry(emit_index),
-               'timestamp' => { 'seconds' => ts.tv_sec, 'nanos' => ts.tv_nsec })
-        expected_ts.push(adjusted_ts)
-        emit_index += 1
-        d.emit('message' => log_entry(emit_index),
-               'timestampSeconds' => ts.tv_sec, 'timestampNanos' => ts.tv_nsec)
-        expected_ts.push(adjusted_ts)
-        emit_index += 1
-        d.emit('message' => log_entry(emit_index),
-               'timestampSeconds' => ts.tv_sec.to_s,
-               'timestampNanos' => ts.tv_nsec.to_s)
-        expected_ts.push(adjusted_ts)
-        emit_index += 1
-        d.run
-        verify_index = 0
-        verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry, i|
-          verify_default_log_entry_text(entry['textPayload'], i, entry)
-          assert_equal_with_default entry['timestamp']['seconds'],
-                                    expected_ts[verify_index].tv_sec, 0, entry
-          assert_equal_with_default \
-            entry['timestamp']['nanos'],
-            expected_ts[verify_index].tv_nsec, 0, entry do
-            # Fluentd v0.14 onwards supports nanosecond timestamp values.
-            # Added in 600 ns delta to avoid flaky tests introduced
-            # due to rounding error in double-precision floating-point numbers
-            # (to account for the missing 9 bits of precision ~ 512 ns).
-            # See http://wikipedia.org/wiki/Double-precision_floating-point_format.
-            assert_in_delta expected_ts[verify_index].tv_nsec,
-                            entry['timestamp']['nanos'], 600, entry
+      APPLICATION_DEFAULT_CONFIG => {
+        Time.at(123_456.789) => Time.at(123_456.789),
+        Time.at(0) => Time.at(0),
+        current_time => current_time,
+        one_second_before_next_year => adjusted_to_last_year,
+        next_year => Time.at(0),
+        one_second_into_next_year => Time.at(0),
+        one_day_into_next_year => Time.at(0)
+      },
+      NO_ADJUST_TIMESTAMPS_CONFIG => {
+        Time.at(123_456.789) => Time.at(123_456.789),
+        Time.at(0) => Time.at(0),
+        current_time => current_time,
+        one_second_before_next_year => one_second_before_next_year,
+        next_year => next_year,
+        one_second_into_next_year => one_second_into_next_year,
+        one_day_into_next_year => one_day_into_next_year
+      }
+    }.each do |config, timestamps|
+      timestamps.each do |ts, expected_ts|
+        emit_index = 0
+        setup_logging_stubs do
+          @logs_sent = []
+          d = create_driver(config)
+          # Test the "native" fluentd timestamp as well as our nanosecond tags.
+          d.emit({ 'message' => log_entry(emit_index) }, ts.to_f)
+          emit_index += 1
+          d.emit('message' => log_entry(emit_index),
+                 'timeNanos' => ts.tv_sec * 1_000_000_000 + ts.tv_nsec)
+          emit_index += 1
+          d.emit('message' => log_entry(emit_index),
+                 'timestamp' => { 'seconds' => ts.tv_sec,
+                                  'nanos' => ts.tv_nsec })
+          emit_index += 1
+          d.emit('message' => log_entry(emit_index),
+                 'timestampSeconds' => ts.tv_sec,
+                 'timestampNanos' => ts.tv_nsec)
+          emit_index += 1
+          d.emit('message' => log_entry(emit_index),
+                 'timestampSeconds' => ts.tv_sec.to_s,
+                 'timestampNanos' => ts.tv_nsec.to_s)
+          emit_index += 1
+          d.run
+          verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry, i|
+            verify_default_log_entry_text(entry['textPayload'], i, entry)
+            assert_equal_with_default entry['timestamp']['seconds'],
+                                      expected_ts.tv_sec, 0, entry
+            assert_equal_with_default \
+              entry['timestamp']['nanos'],
+              expected_ts.tv_nsec, 0, entry do
+              # Fluentd v0.14 onwards supports nanosecond timestamp values.
+              # Added in 600 ns delta to avoid flaky tests introduced
+              # due to rounding error in double-precision floating-point numbers
+              # (to account for the missing 9 bits of precision ~ 512 ns).
+              # See http://wikipedia.org/wiki/Double-precision_floating-point_format.
+              assert_in_delta expected_ts.tv_nsec,
+                              entry['timestamp']['nanos'], 600, entry
+            end
           end
-          verify_index += 1
         end
       end
     end
@@ -1338,14 +1345,14 @@ module BaseTest
         setup_metadata_agent_stub: true,
         setup_k8s_stub: true,
         log_entry: k8s_container_log_entry(log_entry(0)),
-        expected_params: COMPUTE_PARAMS
+        expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL
       },
       {
         config: APPLICATION_DEFAULT_CONFIG,
         setup_metadata_agent_stub: false,
         setup_k8s_stub: true,
         log_entry: k8s_container_log_entry(log_entry(0)),
-        expected_params: COMPUTE_PARAMS
+        expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL
       },
       # When enable_metadata_agent is true.
       {
@@ -1409,7 +1416,7 @@ module BaseTest
         setup_metadata_agent_stub: true,
         setup_k8s_stub: true,
         log_entry: k8s_node_log_entry(log_entry(0)),
-        expected_params: COMPUTE_PARAMS
+        expected_params: K8S_NODE_PARAMS_FROM_LOCAL
       },
       {
         config: ENABLE_METADATA_AGENT_CONFIG,

data/test/plugin/constants.rb

@@ -12,6 +12,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Add some helper methods to standard classes.
+module Google
+  module Protobuf
+    Any.class_eval do
+      # TODO(igorpeshansky): Remove this once
+      # https://github.com/google/protobuf/pull/4719 gets released.
+      def self.pack(msg, type_url_prefix = 'type.googleapis.com/')
+        any = Google::Protobuf::Any.new
+        any.pack(msg, type_url_prefix)
+        any
+      end
+    end
+  end
+end
+String.class_eval do
+  def inspect_octal
+    specials = {
+      'a' => '\\007',
+      'b' => '\\010',
+      'v' => '\\013',
+      'f' => '\\014',
+      'r' => '\\015'
+    }.freeze
+    inspect.gsub(/\\([abvfr])/) { specials[Regexp.last_match(1)] } \
+           .gsub(/\\x([0-9A-F][0-9A-F])/) do
+             format('\\%03o', Regexp.last_match(1).to_i(16))
+           end
+  end
+end
+
 # Constants used by unit tests for Google Cloud Logging plugin.
 module Constants
   include Fluent::GoogleCloudOutput::ServiceConstants
@@ -219,6 +249,10 @@ module Constants
     split_logs_by_tag true
   ).freeze
 
+  NO_ADJUST_TIMESTAMPS_CONFIG = %(
+    adjust_invalid_timestamps false
+  ).freeze
+
   ENABLE_PROMETHEUS_CONFIG = %(
     enable_monitoring true
     monitoring_type prometheus
@@ -809,45 +843,69 @@ module Constants
     }
   }.freeze
 
-  # rubocop:disable Style/StringLiterals
-  PARTIAL_SUCCESS_GRPC_METADATA = {
-    'google.logging.v2.writelogentriespartialerrors-bin' =>
-      Google::Logging::V2::WriteLogEntriesPartialErrors.encode(
-        Google::Logging::V2::WriteLogEntriesPartialErrors.new(
-          log_entry_errors: {
-            0 => Google::Rpc::Status.new(
-              code: GRPC::Core::StatusCodes::PERMISSION_DENIED,
-              message: "User not authorized.",
-              details: []),
-            1 => Google::Rpc::Status.new(
-              code: GRPC::Core::StatusCodes::INVALID_ARGUMENT,
-              message: "Log name contains illegal character :",
-              details: []),
-            3 => Google::Rpc::Status.new(
-              code: GRPC::Core::StatusCodes::INVALID_ARGUMENT,
-              message: "Log name contains illegal character :",
-              details: []) })),
-    'google.rpc.debuginfo-bin' =>
-      "\x12\xA7\x03[ORIGINAL ERROR] generic::permission_denied: User not auth" \
-      "orized. [google.rpc.error_details_ext] { message: \"User not authorize" \
-      "d.\" details { type_url: \"type.googleapis.com/google.logging.v2.Write" \
-      "LogEntriesPartialErrors\" value: \"\\n\\034\\010\\000\\022\\030\\010\\" \
-      "007\\022\\024User not authorized.\\n-\\010\\001\\022)\\010\\003\\022%L" \
-      "og name contains illegal character :\\n-\\010\\002\\022)\\010\\003\\02" \
-      "2%Log name contains illegal character :\" } }",
-    'grpc-status-details-bin' =>
-      "\b\a\x12\x14User not authorized.\x1A\xC2\x01\nBtype.googleapis.com/goo" \
-      "gle.logging.v2.WriteLogEntriesPartialErrors\x12|\n\x1C\b\x00\x12\x18\b" \
-      "\a\x12\x14User not authorized.\n-\b\x01\x12)\b\x03\x12%Log name contai" \
-      "ns illegal character :\n-\b\x02\x12)\b\x03\x12%Log name contains illeg" \
-      "al character :\x1A\xD7\x03\n(type.googleapis.com/google.rpc.DebugInfo" \
-      "\x12\xAA\x03\x12\xA7\x03[ORIGINAL ERROR] generic::permission_denied: U" \
-      "ser not authorized. [google.rpc.error_details_ext] { message: \"User n" \
-      "ot authorized.\" details { type_url: \"type.googleapis.com/google.logg" \
-      "ing.v2.WriteLogEntriesPartialErrors\" value: \"\\n\\034\\010\\000\\022" \
-      "\\030\\010\\007\\022\\024User not authorized.\\n-\\010\\001\\022)\\010" \
-      "\\003\\022%Log name contains illegal character :\\n-\\010\\002\\022)" \
-      "\\010\\003\\022%Log name contains illegal character :\" } }"
+  PARTIAL_SUCCESS_GRPC_METADATA = begin
+    partial_errors = Google::Logging::V2::WriteLogEntriesPartialErrors.new(
+      log_entry_errors: {
+        0 => Google::Rpc::Status.new(
+          code: GRPC::Core::StatusCodes::PERMISSION_DENIED,
+          message: 'User not authorized.',
+          details: []),
+        1 => Google::Rpc::Status.new(
+          code: GRPC::Core::StatusCodes::INVALID_ARGUMENT,
+          message: 'Log name contains illegal character :',
+          details: []),
+        3 => Google::Rpc::Status.new(
+          code: GRPC::Core::StatusCodes::INVALID_ARGUMENT,
+          message: 'Log name contains illegal character :',
+          details: [])
+      })
+    status = Google::Rpc::Status.new(
+      message: 'User not authorized.',
+      details: [Google::Protobuf::Any.pack(partial_errors)])
+    debug_info = Google::Rpc::DebugInfo.new(
+      detail: '[ORIGINAL ERROR] generic::permission_denied: User not' \
+        ' authorized. [google.rpc.error_details_ext] { message:' \
+        " #{status.message.inspect} details { type_url:" \
+        " #{status.details[0].type_url.inspect} value:" \
+        " #{status.details[0].value.inspect_octal} } }")
+    status_details = Google::Rpc::Status.new(
+      code: 7, message: 'User not authorized.',
+      details: [Google::Protobuf::Any.pack(partial_errors),
+                Google::Protobuf::Any.pack(debug_info)])
+    {
+      'google.logging.v2.writelogentriespartialerrors-bin' =>
+        partial_errors.to_proto,
+      'google.rpc.debuginfo-bin' => debug_info.to_proto,
+      'grpc-status-details-bin' => status_details.to_proto
+    }.freeze
+  end
+
+  PARSE_ERROR_RESPONSE_BODY = {
+    'error' => {
+      'code' => 400,
+      'message' => 'Request contains an invalid argument.',
+      'status' => 'INVALID_ARGUMENT',
+      'details' => [
+        {
+          '@type' => 'type.googleapis.com/google.rpc.DebugInfo',
+          'detail' =>
+            '[ORIGINAL ERROR] RPC::CLIENT_ERROR: server could not parse' \
+              " request sent by client; initialization error is: ''"
+        }
+      ]
+    }
   }.freeze
-  # rubocop:enable Style/StringLiterals
+
+  PARSE_ERROR_GRPC_METADATA = begin
+    debug_info = Google::Rpc::DebugInfo.new(
+      detail: '[ORIGINAL ERROR] RPC::CLIENT_ERROR: server could not parse' \
+        " request sent by client; initialization error is: ''")
+    status_details = Google::Rpc::Status.new(
+      code: 3, message: 'internal client error',
+      details: [Google::Protobuf::Any.pack(debug_info)])
+    {
+      'google.rpc.debuginfo-bin' => debug_info.to_proto,
+      'grpc-status-details-bin' => status_details.to_proto
+    }.freeze
+  end
 end

data/test/plugin/test_out_google_cloud.rb

@@ -81,6 +81,31 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
   end
 
+  def test_non_api_error
+    setup_gce_metadata_stubs
+    setup_prometheus
+    # The API Client should not retry this and the plugin should consume
+    # the exception.
+    root_error_code = PARSE_ERROR_RESPONSE_BODY['error']['code']
+    stub_request(:post, WRITE_LOG_ENTRIES_URI)
+      .to_return(status: root_error_code,
+                 body: PARSE_ERROR_RESPONSE_BODY.to_json)
+    d = create_driver(ENABLE_PROMETHEUS_CONFIG)
+    d.emit('message' => log_entry(0))
+    d.run
+    assert_prometheus_metric_value(
+      :stackdriver_successful_requests_count, 0, grpc: false, code: 200)
+    assert_prometheus_metric_value(
+      :stackdriver_failed_requests_count, 1, grpc: false, code: 400)
+    assert_prometheus_metric_value(
+      :stackdriver_ingested_entries_count, 0, grpc: false, code: 200)
+    assert_prometheus_metric_value(
+      :stackdriver_dropped_entries_count, 1, grpc: false, code: 400)
+    assert_prometheus_metric_value(
+      :stackdriver_retried_entries_count, 0, grpc: false)
+    assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
+  end
+
   def test_server_error
     setup_gce_metadata_stubs
     # The API client should retry this once, then throw an exception which

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -97,6 +97,34 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
   end
 
+  def test_non_api_error
+    setup_gce_metadata_stubs
+    setup_prometheus
+    setup_logging_stubs(
+      GRPC::InvalidArgument.new('internal client error',
+                                PARSE_ERROR_GRPC_METADATA)) do
+      # The API Client should not retry this and the plugin should consume
+      # the exception.
+      d = create_driver(ENABLE_PROMETHEUS_CONFIG)
+      d.emit('message' => log_entry(0))
+      d.run
+      assert_prometheus_metric_value(
+        :stackdriver_successful_requests_count, 0,
+        grpc: true, code: GRPC::Core::StatusCodes::OK)
+      assert_prometheus_metric_value(
+        :stackdriver_failed_requests_count, 1,
+        grpc: true, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT)
+      assert_prometheus_metric_value(
+        :stackdriver_ingested_entries_count, 0,
+        grpc: true, code: GRPC::Core::StatusCodes::OK)
+      assert_prometheus_metric_value(
+        :stackdriver_dropped_entries_count, 1,
+        grpc: true, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT)
+      assert_prometheus_metric_value(
+        :stackdriver_retried_entries_count, 0, grpc: true)
+    end
+  end
+
   def test_server_error
     setup_gce_metadata_stubs
     {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.20
+  version: 0.6.21
 platform: ruby
 authors:
 - Ling Huang
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-20 00:00:00.000000000 Z
+date: 2018-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd