fluent-plugin-google-cloud 0.7.21 → 0.7.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c02beaaa9eb340a7d8cc14d8306adbf706734056c1cf8bd5737e5a4dcf36db8
-  data.tar.gz: b3747ea4d2a10db6b9eb5b2a4b27ad6231e35d339d36a2571f02b803229b7aaf
+  metadata.gz: c0d3db1dc87f350081fd9b15fb276f5bb362216d7e3baee3fbfc65811185e178
+  data.tar.gz: f4dfa81c6a56eb92799ce59f391dd3a252ff0efd1f157f7c80df097b92731a58
 SHA512:
-  metadata.gz: fc9e660a5b707978a7b910a7df0f815509c92a5e5477f7cd0d9697fdb52d785e4d8a75601c8defcd3f2d97b6a32acc61ee0e1b1f93741f8b8301cf9468bafe6e
-  data.tar.gz: 8dceed3247b00d18d60dd2a222a320d444000ebef37a563de037b41c36ee4a5ef912865ac098c7e1f47c80dc8d21885438bab4867ae87c4785f53375a769c8e3
+  metadata.gz: 86f7fe07dfdfdfbb4fa787e99671a5b1e6f550cd8fc83da05aaf5bcd098bcff5e1bda4bb4cbd19397c2cbe2f97fb19e5a6eaf91da87839b819ce440c4ba4f6d6
+  data.tar.gz: 8c15a973178e824aadaced186a7635f73b5f24cd13b281b7fb2c83cf2432c495d8ee15a8363bff66518432100b918a88ca0a8a736fb0eeccf22c4693790cf6b7

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.7.21)
+    fluent-plugin-google-cloud (0.7.22)
       fluentd (= 1.6.3)
       google-api-client (= 0.30.8)
       google-cloud-logging (= 1.6.6)
@@ -45,7 +45,7 @@ GEM
       representable (~> 3.0)
       retriable (>= 2.0, < 4.0)
       signet (~> 0.10)
-    google-cloud-core (1.3.1)
+    google-cloud-core (1.3.2)
       google-cloud-env (~> 1.0)
     google-cloud-env (1.2.1)
       faraday (~> 0.11)
@@ -55,6 +55,14 @@ GEM
       google-gax (~> 1.7)
       googleapis-common-protos-types (>= 1.0.2)
       stackdriver-core (~> 1.3)
+    google-cloud-monitoring (0.29.5)
+      google-gax (~> 1.3)
+      googleapis-common-protos-types (>= 1.0.2)
+    google-cloud-trace (0.35.0)
+      concurrent-ruby (~> 1.1)
+      google-cloud-core (~> 1.2)
+      google-gax (~> 1.7)
+      stackdriver-core (~> 1.3)
     google-gax (1.7.1)
       google-protobuf (~> 3.2)
       googleapis-common-protos (>= 1.3.5, < 2.0)
@@ -89,8 +97,14 @@ GEM
     mocha (1.9.0)
       metaclass (~> 0.0.1)
     msgpack (1.3.1)
-    multi_json (1.13.1)
+    multi_json (1.14.1)
     multipart-post (2.1.1)
+    opencensus (0.5.0)
+    opencensus-stackdriver (0.3.0)
+      concurrent-ruby (~> 1.0)
+      google-cloud-monitoring (~> 0.29.2)
+      google-cloud-trace (~> 0.33)
+      opencensus (~> 0.5)
     os (1.0.1)
     parser (2.6.5.0)
       ast (~> 2.4.0)
@@ -149,6 +163,8 @@ PLATFORMS
 DEPENDENCIES
   fluent-plugin-google-cloud!
   mocha (= 1.9.0)
+  opencensus (= 0.5.0)
+  opencensus-stackdriver (= 0.3.0)
   prometheus-client (< 0.10)
   rake (= 10.5.0)
   rubocop (= 0.39.0)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.7.21'
+  gem.version       = '0.7.22'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
@@ -29,6 +29,8 @@ eos
   gem.add_runtime_dependency 'json', '2.2.0'
 
   gem.add_development_dependency 'mocha', '1.9.0'
+  gem.add_development_dependency 'opencensus', '0.5.0'
+  gem.add_development_dependency 'opencensus-stackdriver', '0.3.0'
   # Keep this the same as in
   # https://github.com/fluent/fluent-plugin-prometheus/blob/master/fluent-plugin-prometheus.gemspec
   gem.add_development_dependency 'prometheus-client', '< 0.10'

data/lib/fluent/plugin/monitoring.rb

@@ -31,11 +31,33 @@ module Monitoring
     end
   end
 
+  # OpenCensus implementation of counters.
+  class OpenCensusCounter < BaseCounter
+    def initialize(recorder, measure)
+      raise ArgumentError, 'measure must not be nil' if measure.nil?
+      @recorder = recorder
+      @measure = measure
+    end
+
+    def increment(by: 1, labels: {})
+      tag_map = OpenCensus::Tags::TagMap.new(
+        labels.map { |k, v| [k.to_s, v.to_s] }.to_h)
+      @recorder.record(@measure.create_measurement(value: by, tags: tag_map))
+    end
+  end
+
   # Base class for the monitoring registry.
   class BaseMonitoringRegistry
+    def initialize(_project_id, _monitored_resource)
+    end
+
     def counter(_name, _labels, _docstring)
       nil
     end
+
+    def export
+      nil
+    end
   end
 
   # Prometheus implementation of the monitoring registry, that uses the default
@@ -45,7 +67,8 @@ module Monitoring
       'prometheus'
     end
 
-    def initialize
+    def initialize(_project_id, _monitored_resource)
+      super
       require 'prometheus/client'
       @registry = Prometheus::Client.registry
     end
@@ -58,20 +81,91 @@ module Monitoring
     end
   end
 
+  # OpenCensus implementation of the monitoring registry.
+  class OpenCensusMonitoringRegistry < BaseMonitoringRegistry
+    def self.name
+      'opencensus'
+    end
+
+    def initialize(project_id, monitored_resource)
+      super
+      require 'opencensus'
+      require 'opencensus-stackdriver'
+      @log = $log # rubocop:disable Style/GlobalVars
+      @recorder = OpenCensus::Stats.ensure_recorder
+      @exporter = OpenCensus::Stats::Exporters::Stackdriver.new(
+        project_id: project_id,
+        metric_prefix: 'agent.googleapis.com/agent',
+        resource_type: monitored_resource.type,
+        resource_labels: monitored_resource.labels)
+      OpenCensus.configure do |c|
+        c.stats.exporter = @exporter
+      end
+      @log.debug "OpenCensus config=#{OpenCensus.config}"
+    end
+
+    def counter(name, labels, docstring)
+      name = OpenCensusMonitoringRegistry.translate_metric_name(name)
+      measure = OpenCensus::Stats::MeasureRegistry.get(name)
+      if measure.nil?
+        measure = OpenCensus::Stats.create_measure_int(
+          name: name,
+          unit: OpenCensus::Stats::Measure::UNIT_NONE,
+          description: docstring
+        )
+      end
+      OpenCensus::Stats.create_and_register_view(
+        name: name,
+        measure: measure,
+        aggregation: OpenCensus::Stats.create_sum_aggregation,
+        description: docstring,
+        columns: labels.map(&:to_s)
+      )
+      OpenCensusCounter.new(@recorder, measure)
+    end
+
+    def export
+      @exporter.export @recorder.views_data
+    end
+
+    class << self
+      # Translate the internal metrics to the curated metrics in Stackdriver.
+      # The Prometheus metrics are collected by Google Kubernetes Engine's
+      # monitoring, so we can't redefine them.
+      def translate_metric_name(name)
+        case name
+        when :stackdriver_successful_requests_count,
+             :stackdriver_failed_requests_count
+          :request_count
+        when :stackdriver_ingested_entries_count,
+             :stackdriver_dropped_entries_count
+          :log_entry_count
+        when :stackdriver_retried_entries_count
+          :log_entry_retry_count
+        else
+          name
+        end
+      end
+    end
+  end
+
   # Factory that is used to create a monitoring registry based on
   # the monitoring solution name.
   class MonitoringRegistryFactory
     @known_registry_types = {
       PrometheusMonitoringRegistry.name =>
-        PrometheusMonitoringRegistry
+        PrometheusMonitoringRegistry,
+      OpenCensusMonitoringRegistry.name =>
+        OpenCensusMonitoringRegistry
     }
 
     def self.supports_monitoring_type(name)
       @known_registry_types.key?(name)
     end
 
-    def self.create(name)
-      (@known_registry_types[name] || BaseMonitoringRegistry).new
+    def self.create(name, project_id, monitored_resource)
+      registry = @known_registry_types[name] || BaseMonitoringRegistry
+      registry.new(project_id, monitored_resource)
     end
   end
 end

data/lib/fluent/plugin/out_google_cloud.rb

@@ -64,6 +64,18 @@ module Google
   end
 end
 
+# FluentLogger exposes the Fluent logger to the gRPC library.
+module FluentLogger
+  def logger
+    $log # rubocop:disable Style/GlobalVars
+  end
+end
+
+# Define a gRPC module-level logger method before grpc/logconfig.rb loads.
+module GRPC
+  extend FluentLogger
+end
+
 module Fluent
   # fluentd output plugin for the Stackdriver Logging API
   class GoogleCloudOutput < BufferedOutput
@@ -238,7 +250,7 @@ module Fluent
 
     Fluent::Plugin.register_output('google_cloud', self)
 
-    helpers :server
+    helpers :server, :timer
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
 
@@ -497,41 +509,6 @@ module Fluent
           end
       end
 
-      # If monitoring is enabled, register metrics in the default registry
-      # and store metric objects for future use.
-      if @enable_monitoring
-        unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
-          @monitoring_type)
-          @log.warn "monitoring_type '#{@monitoring_type}' is unknown; "\
-                    'there will be no metrics'
-        end
-        registry = Monitoring::MonitoringRegistryFactory.create @monitoring_type
-        @successful_requests_count = registry.counter(
-          :stackdriver_successful_requests_count,
-          [:grpc, :code],
-          'A number of successful requests to the Stackdriver Logging API')
-        @failed_requests_count = registry.counter(
-          :stackdriver_failed_requests_count,
-          [:grpc, :code],
-          'A number of failed requests to the Stackdriver Logging '\
-          'API, broken down by the error code')
-        @ingested_entries_count = registry.counter(
-          :stackdriver_ingested_entries_count,
-          [:grpc, :code],
-          'A number of log entries ingested by Stackdriver Logging')
-        @dropped_entries_count = registry.counter(
-          :stackdriver_dropped_entries_count,
-          [:grpc, :code],
-          'A number of log entries dropped by the Stackdriver output plugin')
-        @retried_entries_count = registry.counter(
-          :stackdriver_retried_entries_count,
-          [:grpc, :code],
-          'The number of log entries that failed to be ingested by '\
-          'the Stackdriver output plugin due to a transient error '\
-          'and were retried')
-        @ok_code = @use_grpc ? GRPC::Core::StatusCodes::OK : 200
-      end
-
       # Alert on old authentication configuration.
       unless @auth_method.nil? && @private_key_email.nil? &&
              @private_key_path.nil? && @private_key_passphrase.nil?
@@ -564,6 +541,44 @@ module Fluent
       # to get it from Metadata Agent.
       @resource ||= determine_agent_level_monitored_resource_via_legacy
 
+      # If monitoring is enabled, register metrics in the default registry
+      # and store metric objects for future use.
+      if @enable_monitoring
+        unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
+          @monitoring_type)
+          @log.warn "monitoring_type '#{@monitoring_type}' is unknown; "\
+                    'there will be no metrics'
+        end
+        @registry = Monitoring::MonitoringRegistryFactory
+                    .create(@monitoring_type, @project_id, @resource)
+        # Export metrics every 60 seconds.
+        timer_execute(:export_metrics, 60) { @registry.export }
+        @successful_requests_count = @registry.counter(
+          :stackdriver_successful_requests_count,
+          [:grpc, :code],
+          'A number of successful requests to the Stackdriver Logging API')
+        @failed_requests_count = @registry.counter(
+          :stackdriver_failed_requests_count,
+          [:grpc, :code],
+          'A number of failed requests to the Stackdriver Logging '\
+          'API, broken down by the error code')
+        @ingested_entries_count = @registry.counter(
+          :stackdriver_ingested_entries_count,
+          [:grpc, :code],
+          'A number of log entries ingested by Stackdriver Logging')
+        @dropped_entries_count = @registry.counter(
+          :stackdriver_dropped_entries_count,
+          [:grpc, :code],
+          'A number of log entries dropped by the Stackdriver output plugin')
+        @retried_entries_count = @registry.counter(
+          :stackdriver_retried_entries_count,
+          [:grpc, :code],
+          'The number of log entries that failed to be ingested by '\
+          'the Stackdriver output plugin due to a transient error '\
+          'and were retried')
+        @ok_code = @use_grpc ? GRPC::Core::StatusCodes::OK : 200
+      end
+
       # Set regexp that we should match tags against later on. Using a list
       # instead of a map to ensure order.
       @tag_regexp_list = []
@@ -621,6 +636,9 @@ module Fluent
 
     def shutdown
       super
+      # Export metrics on shutdown. This is a best-effort attempt, and it might
+      # fail, for instance if there was a recent write to the same time series.
+      @registry.export unless @registry.nil?
     end
 
     def write(chunk)
@@ -2047,6 +2065,8 @@ module Fluent
     end
 
     def init_api_client
+      # Set up the logger for the auto-generated Google Cloud APIs.
+      Google::Apis.logger = @log
       if @use_grpc
         uri = URI.parse(@logging_api_url)
         host = uri.host

data/test/plugin/base_test.rb

@@ -20,6 +20,15 @@ require 'prometheus/client'
 
 require_relative 'constants'
 
+module Monitoring
+  # Prevent OpenCensus from writing to the network.
+  class OpenCensusMonitoringRegistry
+    def export
+      nil
+    end
+  end
+end
+
 # Unit tests for Google Cloud Logging plugin
 module BaseTest
   include Constants
@@ -903,7 +912,7 @@ module BaseTest
       # [] returns nil for any index.
       [ENABLE_SPLIT_LOGS_BY_TAG_CONFIG, log_entry_count, dynamic_log_names, []]
     ].each do |(config, request_count, request_log_names, entry_log_names)|
-      setup_prometheus
+      clear_metrics
       setup_logging_stubs do
         @logs_sent = []
         d = create_driver(config + ENABLE_PROMETHEUS_CONFIG, 'test', true)
@@ -2144,8 +2153,9 @@ module BaseTest
                           DATAPROC_REGION)
   end
 
-  def setup_prometheus
+  def clear_metrics
     Prometheus::Client.registry.instance_variable_set('@metrics', {})
+    OpenCensus::Stats.ensure_recorder.clear_stats
   end
 
   # Metadata Agent.
@@ -2723,6 +2733,25 @@ module BaseTest
     assert_equal(expected_value, metric_value)
   end
 
+  def assert_opencensus_metric_value(metric_name, expected_value, labels = {})
+    metric_name = Monitoring::OpenCensusMonitoringRegistry
+                  .translate_metric_name(metric_name)
+    labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
+    stats_recorder = OpenCensus::Stats.ensure_recorder
+    view_data = stats_recorder.view_data metric_name
+    assert_not_nil(view_data)
+    # For now assume all metrics are counters.
+    assert_kind_of(OpenCensus::Stats::Aggregation::Sum,
+                   view_data.view.aggregation)
+    assert_true(view_data.view.measure.int64?)
+    tag_values = view_data.view.columns.map { |column| labels[column] }
+    metric_value = 0
+    if view_data.data.key? tag_values
+      metric_value = view_data.data[tag_values].value
+    end
+    assert_equal(expected_value, metric_value)
+  end
+
   # Defined in specific gRPC or REST files.
   def expected_operation_message2
     _undefined

data/test/plugin/constants.rb

@@ -283,6 +283,11 @@ module Constants
     monitoring_type prometheus
   ).freeze
 
+  ENABLE_OPENCENSUS_CONFIG = %(
+    enable_monitoring true
+    monitoring_type opencensus
+  ).freeze
+
   ENABLE_METADATA_AGENT_CONFIG = %(
     enable_metadata_agent true
   ).freeze

data/test/plugin/test_out_google_cloud.rb

@@ -73,7 +73,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
 
   def test_partial_success
     setup_gce_metadata_stubs
-    setup_prometheus
+    clear_metrics
     # The API Client should not retry this and the plugin should consume
     # the exception.
     root_error_code = PARTIAL_SUCCESS_RESPONSE_BODY['error']['code']
@@ -98,7 +98,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
 
   def test_non_api_error
     setup_gce_metadata_stubs
-    setup_prometheus
+    clear_metrics
     # The API Client should not retry this and the plugin should consume
     # the exception.
     root_error_code = PARSE_ERROR_RESPONSE_BODY['error']['code']
@@ -141,56 +141,65 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   # TODO: The code in the non-gRPC and gRPC tests is nearly identical.
   # Refactor and remove duplication.
   # TODO: Use status codes instead of int literals.
-  def test_prometheus_metrics
+  def test_metrics
     setup_gce_metadata_stubs
     [
-      # Single successful request.
-      [200, 1, 1, [1, 0, 1, 0, 0]],
-      # Several successful requests.
-      [200, 2, 1, [2, 0, 2, 0, 0]],
-      # Single successful request with several entries.
-      [200, 1, 2, [1, 0, 2, 0, 0]],
-      # Single failed request that causes logs to be dropped.
-      [401, 1, 1, [0, 1, 0, 1, 0]],
-      # Single failed request that escalates without logs being dropped with
-      # several entries.
-      [500, 1, 2, [0, 0, 0, 0, 2]]
-    ].each do |code, request_count, entry_count, metric_values|
-      setup_prometheus
-      # TODO: Do this as part of setup_logging_stubs.
-      stub_request(:post, WRITE_LOG_ENTRIES_URI)
-        .to_return(status: code, body: 'Some Message')
-      (1..request_count).each do
-        d = create_driver(ENABLE_PROMETHEUS_CONFIG)
-        (1..entry_count).each do |i|
-          d.emit('message' => log_entry(i.to_s))
+      [ENABLE_PROMETHEUS_CONFIG, method(:assert_prometheus_metric_value)],
+      [ENABLE_OPENCENSUS_CONFIG, method(:assert_opencensus_metric_value)]
+    ].each do |config, assert_metric_value|
+      [
+        # Single successful request.
+        [200, 1, 1, [1, 0, 1, 0, 0]],
+        # Several successful requests.
+        [200, 2, 1, [2, 0, 2, 0, 0]],
+        # Single successful request with several entries.
+        [200, 1, 2, [1, 0, 2, 0, 0]],
+        # Single failed request that causes logs to be dropped.
+        [401, 1, 1, [0, 1, 0, 1, 0]],
+        # Single failed request that escalates without logs being dropped with
+        # several entries.
+        [500, 1, 2, [0, 0, 0, 0, 2]]
+      ].each do |code, request_count, entry_count, metric_values|
+        clear_metrics
+        # TODO: Do this as part of setup_logging_stubs.
+        stub_request(:post, WRITE_LOG_ENTRIES_URI)
+          .to_return(status: code, body: 'Some Message')
+        (1..request_count).each do
+          d = create_driver(config)
+          (1..entry_count).each do |i|
+            d.emit('message' => log_entry(i.to_s))
+          end
+          # rubocop:disable Lint/HandleExceptions
+          begin
+            d.run
+          rescue Google::Apis::AuthorizationError
+          rescue Google::Apis::ServerError
+          end
+          # rubocop:enable Lint/HandleExceptions
         end
-        # rubocop:disable Lint/HandleExceptions
-        begin
-          d.run
-        rescue Google::Apis::AuthorizationError
-        rescue Google::Apis::ServerError
-        end
-        # rubocop:enable Lint/HandleExceptions
-      end
-      successful_requests_count, failed_requests_count,
+        successful_requests_count, failed_requests_count,
         ingested_entries_count, dropped_entries_count,
         retried_entries_count = metric_values
-      assert_prometheus_metric_value(:stackdriver_successful_requests_count,
-                                     successful_requests_count,
-                                     grpc: false, code: 200)
-      assert_prometheus_metric_value(:stackdriver_failed_requests_count,
-                                     failed_requests_count,
-                                     grpc: false, code: code)
-      assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
-                                     ingested_entries_count,
-                                     grpc: false, code: 200)
-      assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
-                                     dropped_entries_count,
-                                     grpc: false, code: code)
-      assert_prometheus_metric_value(:stackdriver_retried_entries_count,
-                                     retried_entries_count,
-                                     grpc: false, code: code)
+        assert_metric_value.call(:stackdriver_successful_requests_count,
+                                 successful_requests_count,
+                                 grpc: false, code: 200)
+        assert_metric_value.call(:stackdriver_ingested_entries_count,
+                                 ingested_entries_count,
+                                 grpc: false, code: 200)
+        assert_metric_value.call(:stackdriver_retried_entries_count,
+                                 retried_entries_count,
+                                 grpc: false, code: code)
+        # Skip failure assertions when code indicates success, because the
+        # assertion will fail in the case when a single metric contains time
+        # series with success and failure events.
+        next if code == 200
+        assert_metric_value.call(:stackdriver_failed_requests_count,
+                                 failed_requests_count,
+                                 grpc: false, code: code)
+        assert_metric_value.call(:stackdriver_dropped_entries_count,
+                                 dropped_entries_count,
+                                 grpc: false, code: code)
+      end
     end
   end
 

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -86,7 +86,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
 
   def test_partial_success
     setup_gce_metadata_stubs
-    setup_prometheus
+    clear_metrics
     setup_logging_stubs(
       GRPC::PermissionDenied.new('User not authorized.',
                                  PARTIAL_SUCCESS_GRPC_METADATA)) do
@@ -117,7 +117,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
 
   def test_non_api_error
     setup_gce_metadata_stubs
-    setup_prometheus
+    clear_metrics
     setup_logging_stubs(
       GRPC::InvalidArgument.new('internal client error',
                                 PARSE_ERROR_GRPC_METADATA)) do
@@ -171,57 +171,66 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
   # TODO: The code in the non-gRPC and gRPC tests is nearly identical.
   # Refactor and remove duplication.
   # TODO: Use status codes instead of int literals.
-  def test_prometheus_metrics
+  def test_metrics
     setup_gce_metadata_stubs
     [
-      # Single successful request.
-      [false, 0, 1, 1, [1, 0, 1, 0, 0]],
-      # Several successful requests.
-      [false, 0, 2, 1, [2, 0, 2, 0, 0]],
-      # Single successful request with several entries.
-      [false, 0, 1, 2, [1, 0, 2, 0, 0]],
-      # Single failed request that causes logs to be dropped.
-      [true, 16, 1, 1, [0, 1, 0, 1, 0]],
-      # Single failed request that escalates without logs being dropped with
-      # several entries.
-      [true, 13, 1, 2, [0, 0, 0, 0, 2]]
-    ].each do |should_fail, code, request_count, entry_count, metric_values|
-      setup_prometheus
-      (1..request_count).each do
-        setup_logging_stubs(
-          if should_fail
-            GRPC::BadStatus.new_status_exception(code, 'SomeMessage')
-          end) do
-          d = create_driver(USE_GRPC_CONFIG + ENABLE_PROMETHEUS_CONFIG, 'test')
-          (1..entry_count).each do |i|
-            d.emit('message' => log_entry(i.to_s))
+      [ENABLE_PROMETHEUS_CONFIG, method(:assert_prometheus_metric_value)],
+      [ENABLE_OPENCENSUS_CONFIG, method(:assert_opencensus_metric_value)]
+    ].each do |config, assert_metric_value|
+      [
+        # Single successful request.
+        [false, 0, 1, 1, [1, 0, 1, 0, 0]],
+        # Several successful requests.
+        [false, 0, 2, 1, [2, 0, 2, 0, 0]],
+        # Single successful request with several entries.
+        [false, 0, 1, 2, [1, 0, 2, 0, 0]],
+        # Single failed request that causes logs to be dropped.
+        [true, 16, 1, 1, [0, 1, 0, 1, 0]],
+        # Single failed request that escalates without logs being dropped with
+        # several entries.
+        [true, 13, 1, 2, [0, 0, 0, 0, 2]]
+      ].each do |should_fail, code, request_count, entry_count, metric_values|
+        clear_metrics
+        (1..request_count).each do
+          setup_logging_stubs(
+            if should_fail
+              GRPC::BadStatus.new_status_exception(code, 'SomeMessage')
+            end) do
+            d = create_driver(USE_GRPC_CONFIG + config, 'test')
+            (1..entry_count).each do |i|
+              d.emit('message' => log_entry(i.to_s))
+            end
+            # rubocop:disable Lint/HandleExceptions
+            begin
+              d.run
+            rescue GRPC::BadStatus
+            end
+            # rubocop:enable Lint/HandleExceptions
           end
-          # rubocop:disable Lint/HandleExceptions
-          begin
-            d.run
-          rescue GRPC::BadStatus
-          end
-          # rubocop:enable Lint/HandleExceptions
         end
-      end
-      successful_requests_count, failed_requests_count,
+        successful_requests_count, failed_requests_count,
         ingested_entries_count, dropped_entries_count,
         retried_entries_count = metric_values
-      assert_prometheus_metric_value(:stackdriver_successful_requests_count,
-                                     successful_requests_count,
-                                     grpc: true, code: 0)
-      assert_prometheus_metric_value(:stackdriver_failed_requests_count,
-                                     failed_requests_count,
-                                     grpc: true, code: code)
-      assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
-                                     ingested_entries_count,
-                                     grpc: true, code: 0)
-      assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
-                                     dropped_entries_count,
-                                     grpc: true, code: code)
-      assert_prometheus_metric_value(:stackdriver_retried_entries_count,
-                                     retried_entries_count,
-                                     grpc: true, code: code)
+        assert_metric_value.call(:stackdriver_successful_requests_count,
+                                 successful_requests_count,
+                                 grpc: true, code: 0)
+        assert_metric_value.call(:stackdriver_ingested_entries_count,
+                                 ingested_entries_count,
+                                 grpc: true, code: 0)
+        assert_metric_value.call(:stackdriver_retried_entries_count,
+                                 retried_entries_count,
+                                 grpc: true, code: code)
+        # Skip failure assertions when code indicates success, because the
+        # assertion will fail in the case when a single metric contains time
+        # series with success and failure events.
+        next if code == 0
+        assert_metric_value.call(:stackdriver_failed_requests_count,
+                                 failed_requests_count,
+                                 grpc: true, code: code)
+        assert_metric_value.call(:stackdriver_dropped_entries_count,
+                                 dropped_entries_count,
+                                 grpc: true, code: code)
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.7.21
+  version: 0.7.22
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-13 00:00:00.000000000 Z
+date: 2019-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -136,6 +136,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.9.0
+- !ruby/object:Gem::Dependency
+  name: opencensus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+- !ruby/object:Gem::Dependency
+  name: opencensus-stackdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.0
 - !ruby/object:Gem::Dependency
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement