fluent-plugin-google-cloud 0.10.4 → 0.10.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1e4077eb78db81677fa78de507e0aa13acd11a5a07a94004a4df57b3b3ee09a
-  data.tar.gz: 03602f3a4196b0527a164954c1015e595b7364728513843304c098f7ce440d20
+  metadata.gz: fe8bae5aaf1c56046d8456696abbe28e42deda67ea58033790983bfa1f41c6b8
+  data.tar.gz: 5af589942475e72506274487ec0e5e932431d1fef023043d80cf97adc3054bf1
 SHA512:
-  metadata.gz: 93388ca7c2229d29dd42e6f9d35972841a4024d9e16fc5403993289c6e48f8911d7a12ff8944b73812b15b0fc2311dbf1919e11581343c90e0dfde599ad6c8a3
-  data.tar.gz: c04221d3ac69bfaa0a5831830a8138352a3956a3ad9adb9b99df027b1788252b8e1085e58b7ff180d574511f91cf2b408b0405c5310e3fd11982ad9f36f6d1c1
+  metadata.gz: '004228fe99d339dcdf71503d9bf503bbbdb1c03ac53067bd8423120fbdbe6f25af4679fcb61e04e1d9d4886e00a1761d90db7af3dea53f7d7e3fdfadde0015a8'
+  data.tar.gz: c09da939db031ae5baec55f7e0930f84b7e39ad3d0e36d467d3e7496e9a4c8b5db0080d196d99589971930b2fee6c37818a2f6d959c4e7f4e9f54928a30ed56b

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.10.4)
+    fluent-plugin-google-cloud (0.10.5)
       fluentd (= 1.11.2)
       google-api-client (= 0.30.8)
       google-cloud-logging (= 1.6.6)
@@ -27,10 +27,11 @@ GEM
       term-ansicolor (~> 1.3)
       thor (>= 0.19.4, < 2.0)
       tins (~> 1.6)
-    crack (0.4.4)
+    crack (0.4.5)
+      rexml
     declarative (0.0.20)
     declarative-option (0.1.0)
-    docile (1.3.2)
+    docile (1.3.4)
     faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
     fluentd (1.11.2)
@@ -131,7 +132,7 @@ GEM
     parser (2.7.2.0)
       ast (~> 2.4.1)
     power_assert (1.2.0)
-    powerpack (0.1.2)
+    powerpack (0.1.3)
     prometheus-client (0.9.0)
       quantile (~> 0.2.1)
     public_suffix (4.0.6)
@@ -144,6 +145,7 @@ GEM
       declarative-option (< 0.2.0)
       uber (< 0.2.0)
     retriable (3.1.2)
+    rexml (3.2.4)
     rly (0.2.3)
     rubocop (0.39.0)
       parser (>= 2.3.0.7, < 3.0)
@@ -151,8 +153,8 @@ GEM
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.10.1)
-    serverengine (2.2.1)
+    ruby-progressbar (1.11.0)
+    serverengine (2.2.2)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     signet (0.14.0)
@@ -176,9 +178,9 @@ GEM
     thor (1.0.1)
     tins (1.26.0)
       sync
-    tzinfo (2.0.2)
+    tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2020.4)
+    tzinfo-data (1.2020.6)
       tzinfo (>= 1.0.0)
     uber (0.1.0)
     unicode-display_width (1.7.0)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.10.4'
+  gem.version       = '0.10.5'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')

data/lib/fluent/plugin/filter_analyze_config.rb

@@ -15,6 +15,7 @@
 require 'fileutils'
 require 'fluent/config'
 require 'fluent/config/v1_parser'
+require 'fluent/plugin_helper'
 require 'googleauth'
 require 'google/apis/logging_v2'
 require 'open-uri'
@@ -32,6 +33,10 @@ module Fluent
     include Fluent::Config
     Fluent::Plugin.register_filter('analyze_config', self)
 
+    # Required for the timer_execute method below
+    include PluginHelper::Mixin
+    helpers :timer
+
     module Constants
       # Built-in plugins that are ok to reference in metrics.
       KNOWN_PLUGINS = {
@@ -212,7 +217,7 @@ module Fluent
       if File.file?(@google_fluentd_config_path) &&
          File.file?(@google_fluentd_baseline_config_path)
         @log.info(
-          'google-fluentd configuration file found at' \
+          'analyze_config plugin: google-fluentd configuration file found at' \
           " #{@google_fluentd_config_path}. " \
           'google-fluentd baseline configuration file found at' \
           " #{@google_fluentd_baseline_config_path}. " \
@@ -240,27 +245,49 @@ module Fluent
             "analyze_config plugin: monitoring_type #{@monitoring_type} is " \
             'unknown; there will be no metrics.')
         end
-        registry = Monitoring::MonitoringRegistryFactory.create(
+
+        @registry = Monitoring::MonitoringRegistryFactory.create(
           @monitoring_type, project_id, resource, @gcm_service_address)
+        # Export metrics every 60 seconds.
+        timer_execute(:export_config_analysis_metrics, 60) { @registry.export }
 
-        plugin_usage = registry.counter(
+        @log.info('analyze_config plugin: Registering counters.')
+        enabled_plugins_counter = @registry.counter(
           :enabled_plugins,
-          [:plugin_name, :is_default_plugin, :has_default_config],
+          [:plugin_name, :is_default_plugin,
+           :has_default_config, :has_ruby_snippet],
           'Enabled plugins',
-          'agent.googleapis.com/agent/internal/logging/config')
-        config_usage = registry.counter(
+          'agent.googleapis.com/agent/internal/logging/config',
+          'GAUGE')
+        @log.info(
+          'analyze_config plugin: registered enable_plugins counter. ' \
+          "#{enabled_plugins_counter}")
+        plugin_config_counter = @registry.counter(
           :plugin_config,
           [:plugin_name, :param, :is_present, :has_default_config],
           'Configuration parameter usage for plugins relevant to Google Cloud.',
-          'agent.googleapis.com/agent/internal/logging/config')
-        config_bool_values = registry.counter(
+          'agent.googleapis.com/agent/internal/logging/config',
+          'GAUGE')
+        @log.info('analyze_config plugin: registered plugin_config counter. ' \
+          "#{plugin_config_counter}")
+        config_bool_values_counter = @registry.counter(
           :config_bool_values,
           [:plugin_name, :param, :value],
           'Values for bool parameters in Google Cloud plugins',
-          'agent.googleapis.com/agent/internal/logging/config')
+          'agent.googleapis.com/agent/internal/logging/config',
+          'GAUGE')
+        @log.info('analyze_config plugin: registered config_bool_values ' \
+          "counter. #{config_bool_values_counter}")
 
         config = parse_config(@google_fluentd_config_path)
+        @log.debug(
+          'analyze_config plugin: successfully parsed google-fluentd' \
+          " configuration file at #{@google_fluentd_config_path}. #{config}")
         baseline_config = parse_config(@google_fluentd_baseline_config_path)
+        @log.debug(
+          'analyze_config plugin: successfully parsed google-fluentd' \
+          ' baseline configuration file at' \
+          " #{@google_fluentd_baseline_config_path}: #{baseline_config}")
 
         # Create hash of all baseline elements by their plugin names.
         baseline_elements = Hash[baseline_config.elements.collect do |e|
@@ -290,7 +317,7 @@ module Fluent
             is_default_plugin = false
             has_default_config = false
           end
-          plugin_usage.increment(
+          enabled_plugins_counter.increment(
             labels: {
               plugin_name: plugin_name,
               is_default_plugin: is_default_plugin,
@@ -303,7 +330,7 @@ module Fluent
           # detect_exceptions).
           next unless GOOGLE_PLUGIN_PARAMS.key?(e['@type'])
           GOOGLE_PLUGIN_PARAMS[e['@type']].each do |p|
-            config_usage.increment(
+            plugin_config_counter.increment(
               labels: {
                 plugin_name: e['@type'],
                 param: p,
@@ -314,7 +341,7 @@ module Fluent
               },
               by: 1)
             next unless e.key?(p) && %w(true false).include?(e[p])
-            config_bool_values.increment(
+            config_bool_values_counter.increment(
               labels: {
                 plugin_name: e['@type'],
                 param: p,
@@ -323,6 +350,8 @@ module Fluent
               by: 1)
           end
         end
+        @log.info(
+          'analyze_config plugin: Successfully finished analyzing config.')
       else
         @log.info(
           'analyze_config plugin: google-fluentd configuration file does not ' \
@@ -341,6 +370,9 @@ module Fluent
 
     def shutdown
       super
+      # Export metrics on shutdown. This is a best-effort attempt, and it might
+      # fail, for instance if there was a recent write to the same time series.
+      @registry.export unless @registry.nil?
     end
 
     # rubocop:disable Lint/UnusedMethodArgument

data/lib/fluent/plugin/monitoring.rb

@@ -52,7 +52,7 @@ module Monitoring
     def initialize(_project_id, _monitored_resource, _gcm_service_address)
     end
 
-    def counter(_name, _labels, _docstring, _prefix)
+    def counter(_name, _labels, _docstring, _prefix, _aggregation)
       BaseCounter.new
     end
 
@@ -75,7 +75,7 @@ module Monitoring
     end
 
     # Exception-driven behavior to avoid synchronization errors.
-    def counter(name, _labels, docstring, _prefix)
+    def counter(name, _labels, docstring, _prefix, _aggregation)
       # When we upgrade to Prometheus client 0.10.0 or higher, pass the
       # labels in the metric constructor. The 'labels' field in
       # Prometheus client 0.9.0 has a different function and will not
@@ -102,12 +102,18 @@ module Monitoring
       @gcm_service_address = gcm_service_address
       @recorders = {}
       @exporters = {}
+      @log.info(
+        'monitoring module: Successfully initialized Open Census monitoring ' \
+        'registry.')
     end
 
-    def counter(name, labels, docstring, prefix)
+    def counter(name, labels, docstring, prefix, aggregation)
       translator = MetricTranslator.new(name, labels)
       measure = OpenCensus::Stats::MeasureRegistry.get(translator.name)
       if measure.nil?
+        @log.info(
+          'monitoring module: Registering a new measure registry for ' \
+          "#{translator.name}")
         measure = OpenCensus::Stats.create_measure_int(
           name: translator.name,
           unit: OpenCensus::Stats::Measure::UNIT_NONE,
@@ -115,7 +121,10 @@ module Monitoring
         )
       end
       unless @exporters.keys.include?(prefix)
-        @recorders[prefix] = OpenCensus::Stats.ensure_recorder
+        @log.info(
+          'monitoring module: Registering a new exporter for ' \
+          "#{prefix}")
+        @recorders[prefix] = OpenCensus::Stats::Recorder.new
         @exporters[prefix] = \
           OpenCensus::Stats::Exporters::Stackdriver.new(
             project_id: @project_id,
@@ -124,25 +133,47 @@ module Monitoring
             resource_labels: @metrics_monitored_resource.labels,
             gcm_service_address: @gcm_service_address
           )
+        @log.info(
+          'monitoring module: Registered recorders and exporters for ' \
+          "#{prefix}.\n#{@exporters[prefix]}")
       end
-      OpenCensus::Stats.create_and_register_view(
-        name: translator.name,
-        measure: measure,
-        aggregation: OpenCensus::Stats.create_sum_aggregation,
-        description: docstring,
-        columns: translator.view_labels.map(&:to_s)
+      if aggregation == 'GAUGE'
+        stats_aggregation = OpenCensus::Stats.create_last_value_aggregation
+      else
+        stats_aggregation = OpenCensus::Stats.create_sum_aggregation
+      end
+      @recorders[prefix].register_view(
+        OpenCensus::Stats::View.new(
+          name: translator.name,
+          measure: measure,
+          aggregation: stats_aggregation,
+          description: docstring,
+          columns: translator.view_labels.map(&:to_s)
+        )
       )
-      OpenCensusCounter.new(@recorders[prefix], measure, translator)
+      counter = OpenCensusCounter.new(@recorders[prefix], measure, translator)
+      @log.info(
+        'monitoring module: Successfully initialized Open Census counter for ' \
+        "#{prefix}/#{name}.")
+      counter
     rescue StandardError => e
       @log.warn "Failed to count metrics for #{name}.", error: e
       raise e
     end
 
     def export
+      @log.debug(
+        "monitoring module: Exporting metrics for #{@exporters.keys}.")
       @exporters.keys.each do |prefix|
+        @log.debug(
+          "monitoring module: Exporting metrics for #{prefix}. " \
+          "#{@recorders[prefix].views_data}")
         @exporters[prefix].export @recorders[prefix].views_data
       end
     rescue StandardError => e
+      # TODO(lingshi): Fix the error handling here. Seems like the export is
+      # done asynchronously. So any failure happens silently. More details at
+      # https://github.com/census-ecosystem/opencensus-ruby-exporter-stackdriver/blob/f8de506204972548ca535eff6010d15f328df6c3/lib/opencensus/stats/exporters/stackdriver.rb#L156
       @log.warn 'Failed to export some metrics.', error: e
       raise e
     end

data/lib/fluent/plugin/out_google_cloud.rb

@@ -564,37 +564,37 @@ module Fluent
         # we can't change it.
         @uptime_metric = @registry.counter(
           :uptime, [:version], 'Uptime of Logging agent',
-          'agent.googleapis.com/agent')
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         update_uptime
         timer_execute(:update_uptime, 1) { update_uptime }
         @successful_requests_count = @registry.counter(
           :stackdriver_successful_requests_count,
           [:grpc, :code],
           'A number of successful requests to the Stackdriver Logging API',
-          'agent.googleapis.com/agent')
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @failed_requests_count = @registry.counter(
           :stackdriver_failed_requests_count,
           [:grpc, :code],
           'A number of failed requests to the Stackdriver Logging '\
           'API, broken down by the error code',
-          'agent.googleapis.com/agent')
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @ingested_entries_count = @registry.counter(
           :stackdriver_ingested_entries_count,
           [:grpc, :code],
           'A number of log entries ingested by Stackdriver Logging',
-          'agent.googleapis.com/agent')
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @dropped_entries_count = @registry.counter(
           :stackdriver_dropped_entries_count,
           [:grpc, :code],
           'A number of log entries dropped by the Stackdriver output plugin',
-          'agent.googleapis.com/agent')
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @retried_entries_count = @registry.counter(
           :stackdriver_retried_entries_count,
           [:grpc, :code],
           'The number of log entries that failed to be ingested by '\
           'the Stackdriver output plugin due to a transient error '\
           'and were retried',
-          'agent.googleapis.com/agent')
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @ok_code = @use_grpc ? GRPC::Core::StatusCodes::OK : 200
       end
 

data/test/plugin/asserts.rb

@@ -37,7 +37,10 @@ module Asserts
     assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
   end
 
-  def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
+  # rubocop:disable Metrics/ParameterLists
+  def assert_prometheus_metric_value(metric_name, expected_value, _prefix,
+                                     _aggregation, _test_driver, labels = {})
+    # rubocop:enable Metrics/ParameterLists
     metric = Prometheus::Client.registry.get(metric_name)
     assert_not_nil(metric)
     metric_value = if labels == :aggregate
@@ -49,7 +52,10 @@ module Asserts
     assert_equal(expected_value, metric_value)
   end
 
-  def assert_opencensus_metric_value(metric_name, expected_value, labels = {})
+  # rubocop:disable Metrics/ParameterLists
+  def assert_opencensus_metric_value(metric_name, expected_value, prefix,
+                                     aggregation, test_driver, labels = {})
+    # rubocop:enable Metrics/ParameterLists
     translator = Monitoring::MetricTranslator.new(metric_name, labels)
     metric_name = translator.name
     labels = translator.translate_labels(labels)
@@ -60,18 +66,22 @@ module Asserts
     # label that never changes during runtime.
     labels.select! { |k, _| translator.view_labels.include? k }
     labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
-    stats_recorder = OpenCensus::Stats.ensure_recorder
-    view_data = stats_recorder.view_data metric_name
-    assert_not_nil(view_data)
+
+    registry = test_driver.instance.instance_variable_get(:@registry)
+    recorder = registry.instance_variable_get(:@recorders)[prefix]
+    view_data = recorder.measure_views_data[metric_name][0].data
+    view = recorder.instance_variable_get(:@views)[metric_name]
+
+    # Assert values in the view.
+    assert_kind_of(aggregation, view.aggregation)
+    assert_equal(labels.keys, view.columns)
+    assert_equal(metric_name, view.measure.name)
+    assert_equal('INT64', view.measure.type)
+
     # For now assume all metrics are counters.
-    assert_kind_of(OpenCensus::Stats::Aggregation::Sum,
-                   view_data.view.aggregation)
-    assert_true(view_data.view.measure.int64?)
-    tag_values = view_data.view.columns.map { |column| labels[column] }
+    tag_values = view.columns.map { |column| labels[column] }
     metric_value = 0
-    if view_data.data.key? tag_values
-      metric_value = view_data.data[tag_values].value
-    end
+    metric_value = view_data[tag_values].value if view_data.key? tag_values
     assert_equal(expected_value, metric_value)
   end
 end

data/test/plugin/base_test.rb

@@ -943,22 +943,30 @@ module BaseTest
           d.emit("tag#{i}", 'message' => log_entry(i))
         end
         d.run
+        @logs_sent.zip(request_log_names).each do |request, log_name|
+          assert_equal log_name, request['logName']
+        end
+        verify_log_entries(log_entry_count, COMPUTE_PARAMS_NO_LOG_NAME,
+                           'textPayload') do |entry, entry_index|
+          verify_default_log_entry_text(entry['textPayload'], entry_index,
+                                        entry)
+          assert_equal entry_log_names[entry_index], entry['logName']
+        end
+        # Verify the number of requests is different based on whether the
+        # 'split_logs_by_tag' flag is enabled.
+        assert_prometheus_metric_value(
+          :stackdriver_successful_requests_count,
+          request_count,
+          'agent.googleapis.com/agent',
+          OpenCensus::Stats::Aggregation::Sum, d,
+          :aggregate)
+        assert_prometheus_metric_value(
+          :stackdriver_ingested_entries_count,
+          log_entry_count,
+          'agent.googleapis.com/agent',
+          OpenCensus::Stats::Aggregation::Sum, d,
+          :aggregate)
       end
-      @logs_sent.zip(request_log_names).each do |request, log_name|
-        assert_equal log_name, request['logName']
-      end
-      verify_log_entries(log_entry_count, COMPUTE_PARAMS_NO_LOG_NAME,
-                         'textPayload') do |entry, entry_index|
-        verify_default_log_entry_text(entry['textPayload'], entry_index,
-                                      entry)
-        assert_equal entry_log_names[entry_index], entry['logName']
-      end
-      # Verify the number of requests is different based on whether the
-      # 'split_logs_by_tag' flag is enabled.
-      assert_prometheus_metric_value(:stackdriver_successful_requests_count,
-                                     request_count, :aggregate)
-      assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
-                                     log_entry_count, :aggregate)
     end
   end
 
@@ -1968,7 +1976,9 @@ module BaseTest
         expected = Time.now.to_i - start_time
         d.instance.update_uptime
         assert_metric_value.call(
-          :uptime, expected, version: Fluent::GoogleCloudOutput.version_string)
+          :uptime, expected, 'agent.googleapis.com/agent',
+          OpenCensus::Stats::Aggregation::Sum, d,
+          version: Fluent::GoogleCloudOutput.version_string)
       rescue Test::Unit::AssertionFailedError
         retry if (retries += 1) < 3
       end
@@ -1976,6 +1986,102 @@ module BaseTest
     end
   end
 
+  def test_metrics
+    setup_gce_metadata_stubs
+    [
+      [ENABLE_PROMETHEUS_CONFIG, method(:assert_prometheus_metric_value)],
+      [ENABLE_OPENCENSUS_CONFIG, method(:assert_opencensus_metric_value)]
+    ].each do |config, assert_metric_value|
+      [
+        # Single successful request.
+        [ok_status_code, 1, 1, [0, 0, 0]],
+        # Several successful requests.
+        [ok_status_code, 2, 1, [0, 0, 0]],
+        # Single successful request with several entries.
+        [ok_status_code, 1, 2, [0, 0, 0]],
+        # Single failed request that causes logs to be dropped.
+        [client_error_status_code, 1, 1, [1, 1, 0]],
+        # Single failed request that escalates without logs being dropped with
+        # several entries.
+        [server_error_status_code, 1, 2, [0, 0, 2]]
+      ].each do |code, request_count, entry_count, metric_values|
+        clear_metrics
+        setup_logging_stubs(nil, code, 'SomeMessage') do
+          (1..request_count).each do |request_index|
+            d = create_driver(config)
+            (1..entry_count).each do |entry_index|
+              d.emit('message' => log_entry(entry_index.to_s))
+            end
+            # rubocop:disable Lint/HandleExceptions
+            begin
+              d.run
+            rescue mock_error_type
+            end
+            # rubocop:enable Lint/HandleExceptions
+            failed_requests_count, dropped_entries_count,
+            retried_entries_count = metric_values
+
+            successful_requests_count = \
+              if code != ok_status_code
+                0
+              elsif config == ENABLE_OPENCENSUS_CONFIG
+                # TODO(b/173215689) Improve the Open Census side of testing.
+                # The test driver instance variables can not survive between
+                # test driver runs. So the open cencensus side counter gets
+                # reset as expected.
+                1
+              else
+                request_index
+              end
+
+            ingested_entries_count = \
+              if code != ok_status_code
+                0
+              elsif config == ENABLE_OPENCENSUS_CONFIG
+                # TODO(b/173215689) Improve the Open Census side of testing.
+                # The test driver instance variables can not survive between
+                # test driver runs. So the open cencensus side counter gets
+                # reset as expected.
+                entry_count
+              else
+                request_index * entry_count
+              end
+
+            assert_metric_value.call(:stackdriver_successful_requests_count,
+                                     successful_requests_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: ok_status_code)
+            assert_metric_value.call(:stackdriver_ingested_entries_count,
+                                     ingested_entries_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: ok_status_code)
+            assert_metric_value.call(:stackdriver_retried_entries_count,
+                                     retried_entries_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: code)
+            # Skip failure assertions when code indicates success, because the
+            # assertion will fail in the case when a single metric contains time
+            # series with success and failure events.
+            next if code == ok_status_code
+            assert_metric_value.call(:stackdriver_failed_requests_count,
+                                     failed_requests_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: code)
+            assert_metric_value.call(:stackdriver_dropped_entries_count,
+                                     dropped_entries_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: code)
+          end
+        end
+      end
+    end
+  end
+
   private
 
   # Provide a stub context that initializes @logs_sent, executes the block and
@@ -2456,7 +2562,32 @@ module BaseTest
   end
 
   # Set up http or grpc stubs to mock the external calls.
-  def setup_logging_stubs
+  def setup_logging_stubs(_error = nil, _code = nil, _message = nil)
+    _undefined
+  end
+
+  # Whether this is the grpc path
+  def use_grpc
+    _undefined
+  end
+
+  # The OK status code.
+  def ok_status_code
+    _undefined
+  end
+
+  # A client side error status code.
+  def client_error_status_code
+    _undefined
+  end
+
+  # A server side error status code.
+  def server_error_status_code
+    _undefined
+  end
+
+  # The parent error type to expect in the mock
+  def mock_error_type
     _undefined
   end
 

data/test/plugin/test_filter_analyze_config.rb

@@ -51,12 +51,15 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
        method(:assert_opencensus_metric_value)]
     ].each do |config, assert_metric_value|
       clear_metrics
-      create_driver(config)
+      d = create_driver(config)
 
       # Default plugins, with default config.
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'source/syslog/tcp',
         is_default_plugin: true,
         has_default_config: true,
@@ -64,6 +67,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'source/tail/apache-access',
         is_default_plugin: true,
         has_default_config: true,
@@ -71,6 +77,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'filter/add_insert_ids',
         is_default_plugin: true,
         has_default_config: true,
@@ -80,6 +89,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'match/google_cloud',
         is_default_plugin: true,
         has_default_config: false,
@@ -89,6 +101,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'filter',
         is_default_plugin: false,
         has_default_config: false,
@@ -96,6 +111,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'filter/record_transformer',
         is_default_plugin: false,
         has_default_config: false,
@@ -103,6 +121,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'match/stdout',
         is_default_plugin: false,
         has_default_config: false,
@@ -112,6 +133,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :plugin_config,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'adjust_invalid_timestamps',
         is_present: true,
@@ -119,6 +143,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :plugin_config,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'autoformat_stackdriver_trace',
         is_present: true,
@@ -126,6 +153,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :plugin_config,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'coerce_to_utf8',
         is_present: true,
@@ -165,6 +195,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
         assert_metric_value.call(
           :plugin_config,
           1,
+          'agent.googleapis.com/agent/internal/logging/config',
+          OpenCensus::Stats::Aggregation::LastValue,
+          d,
           plugin_name: 'google_cloud',
           param: p,
           is_present: false,
@@ -175,18 +208,27 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :config_bool_values,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'adjust_invalid_timestamps',
         value: true)
       assert_metric_value.call(
         :config_bool_values,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'autoformat_stackdriver_trace',
         value: false)
       assert_metric_value.call(
         :config_bool_values,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'coerce_to_utf8',
         value: true)

data/test/plugin/test_out_google_cloud.rb

@@ -86,13 +86,21 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     end
     d.run
     assert_prometheus_metric_value(
-      :stackdriver_successful_requests_count, 1, grpc: false, code: 200)
+      :stackdriver_successful_requests_count, 1,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 200)
     assert_prometheus_metric_value(
-      :stackdriver_ingested_entries_count, 1, grpc: false, code: 200)
+      :stackdriver_ingested_entries_count, 1,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 200)
     assert_prometheus_metric_value(
-      :stackdriver_dropped_entries_count, 2, grpc: false, code: 3)
+      :stackdriver_dropped_entries_count, 2,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 3)
     assert_prometheus_metric_value(
-      :stackdriver_dropped_entries_count, 1, grpc: false, code: 7)
+      :stackdriver_dropped_entries_count, 1,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 7)
     assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
   end
 
@@ -109,13 +117,21 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     d.emit('message' => log_entry(0))
     d.run
     assert_prometheus_metric_value(
-      :stackdriver_successful_requests_count, 0, grpc: false, code: 200)
+      :stackdriver_successful_requests_count, 0,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 200)
     assert_prometheus_metric_value(
-      :stackdriver_failed_requests_count, 1, grpc: false, code: 400)
+      :stackdriver_failed_requests_count, 1,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 400)
     assert_prometheus_metric_value(
-      :stackdriver_ingested_entries_count, 0, grpc: false, code: 200)
+      :stackdriver_ingested_entries_count, 0,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 200)
     assert_prometheus_metric_value(
-      :stackdriver_dropped_entries_count, 1, grpc: false, code: 400)
+      :stackdriver_dropped_entries_count, 1,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 400)
     assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
   end
 
@@ -138,71 +154,6 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_equal 1, exception_count
   end
 
-  # TODO: The code in the non-gRPC and gRPC tests is nearly identical.
-  # Refactor and remove duplication.
-  # TODO: Use status codes instead of int literals.
-  def test_metrics
-    setup_gce_metadata_stubs
-    [
-      [ENABLE_PROMETHEUS_CONFIG, method(:assert_prometheus_metric_value)],
-      [ENABLE_OPENCENSUS_CONFIG, method(:assert_opencensus_metric_value)]
-    ].each do |config, assert_metric_value|
-      [
-        # Single successful request.
-        [200, 1, 1, [1, 0, 1, 0, 0]],
-        # Several successful requests.
-        [200, 2, 1, [2, 0, 2, 0, 0]],
-        # Single successful request with several entries.
-        [200, 1, 2, [1, 0, 2, 0, 0]],
-        # Single failed request that causes logs to be dropped.
-        [401, 1, 1, [0, 1, 0, 1, 0]],
-        # Single failed request that escalates without logs being dropped with
-        # several entries.
-        [500, 1, 2, [0, 0, 0, 0, 2]]
-      ].each do |code, request_count, entry_count, metric_values|
-        clear_metrics
-        # TODO: Do this as part of setup_logging_stubs.
-        stub_request(:post, WRITE_LOG_ENTRIES_URI)
-          .to_return(status: code, body: 'Some Message')
-        (1..request_count).each do
-          d = create_driver(config)
-          (1..entry_count).each do |i|
-            d.emit('message' => log_entry(i.to_s))
-          end
-          # rubocop:disable Lint/HandleExceptions
-          begin
-            d.run
-          rescue Google::Apis::AuthorizationError
-          rescue Google::Apis::ServerError
-          end
-          # rubocop:enable Lint/HandleExceptions
-        end
-        successful_requests_count, failed_requests_count,
-        ingested_entries_count, dropped_entries_count,
-        retried_entries_count = metric_values
-        assert_metric_value.call(:stackdriver_successful_requests_count,
-                                 successful_requests_count,
-                                 grpc: false, code: 200)
-        assert_metric_value.call(:stackdriver_ingested_entries_count,
-                                 ingested_entries_count,
-                                 grpc: false, code: 200)
-        assert_metric_value.call(:stackdriver_retried_entries_count,
-                                 retried_entries_count,
-                                 grpc: false, code: code)
-        # Skip failure assertions when code indicates success, because the
-        # assertion will fail in the case when a single metric contains time
-        # series with success and failure events.
-        next if code == 200
-        assert_metric_value.call(:stackdriver_failed_requests_count,
-                                 failed_requests_count,
-                                 grpc: false, code: code)
-        assert_metric_value.call(:stackdriver_dropped_entries_count,
-                                 dropped_entries_count,
-                                 grpc: false, code: code)
-      end
-    end
-  end
-
   # This test looks similar between the grpc and non-grpc paths except that when
   # parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
   # responds with "100".
@@ -400,14 +351,39 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   # Set up http stubs to mock the external calls.
-  def setup_logging_stubs
+  def setup_logging_stubs(_error = nil, code = nil, message = 'some message')
     stub_request(:post, WRITE_LOG_ENTRIES_URI).to_return do |request|
       @logs_sent << JSON.parse(request.body)
-      { body: '' }
+      { status: code, body: message }
     end
     yield
   end
 
+  # Whether this is the grpc path
+  def use_grpc
+    false
+  end
+
+  # The OK status code for the grpc path.
+  def ok_status_code
+    200
+  end
+
+  # A client side error status code for the grpc path.
+  def client_error_status_code
+    401
+  end
+
+  # A server side error status code for the grpc path.
+  def server_error_status_code
+    500
+  end
+
+  # The parent error type to expect in the mock
+  def mock_error_type
+    Google::Apis::Error
+  end
+
   # The conversions from user input to output.
   def latency_conversion
     {

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -60,8 +60,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
       GRPC::Core::StatusCodes::ABORTED => 'Aborted',
       GRPC::Core::StatusCodes::UNAUTHENTICATED => 'Unauthenticated'
     }.each_with_index do |(code, message), index|
-      setup_logging_stubs(
-        GRPC::BadStatus.new_status_exception(code, message)) do
+      setup_logging_stubs(nil, code, message) do
         d = create_driver(USE_GRPC_CONFIG, 'test')
         # The API Client should not retry this and the plugin should consume the
         # exception.
@@ -99,19 +98,24 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
       d.run
       assert_prometheus_metric_value(
         :stackdriver_successful_requests_count, 1,
-        grpc: true, code: GRPC::Core::StatusCodes::OK)
+        'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+        grpc: use_grpc, code: GRPC::Core::StatusCodes::OK)
       assert_prometheus_metric_value(
         :stackdriver_failed_requests_count, 0,
-        grpc: true, code: GRPC::Core::StatusCodes::PERMISSION_DENIED)
+        'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+        grpc: use_grpc, code: GRPC::Core::StatusCodes::PERMISSION_DENIED)
       assert_prometheus_metric_value(
         :stackdriver_ingested_entries_count, 1,
-        grpc: true, code: GRPC::Core::StatusCodes::OK)
+        'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+        grpc: use_grpc, code: GRPC::Core::StatusCodes::OK)
       assert_prometheus_metric_value(
         :stackdriver_dropped_entries_count, 2,
-        grpc: true, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT)
+        'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+        grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT)
       assert_prometheus_metric_value(
         :stackdriver_dropped_entries_count, 1,
-        grpc: true, code: GRPC::Core::StatusCodes::PERMISSION_DENIED)
+        'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+        grpc: use_grpc, code: GRPC::Core::StatusCodes::PERMISSION_DENIED)
     end
   end
 
@@ -128,16 +132,20 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
       d.run
       assert_prometheus_metric_value(
         :stackdriver_successful_requests_count, 0,
-        grpc: true, code: GRPC::Core::StatusCodes::OK)
+        'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+        grpc: use_grpc, code: GRPC::Core::StatusCodes::OK)
       assert_prometheus_metric_value(
         :stackdriver_failed_requests_count, 1,
-        grpc: true, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT)
+        'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+        grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT)
       assert_prometheus_metric_value(
         :stackdriver_ingested_entries_count, 0,
-        grpc: true, code: GRPC::Core::StatusCodes::OK)
+        'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+        grpc: use_grpc, code: GRPC::Core::StatusCodes::OK)
       assert_prometheus_metric_value(
         :stackdriver_dropped_entries_count, 1,
-        grpc: true, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT)
+        'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+        grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT)
     end
   end
 
@@ -150,8 +158,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
       GRPC::Core::StatusCodes::UNAVAILABLE => 'Unavailable'
     }.each_with_index do |(code, message), index|
       exception_count = 0
-      setup_logging_stubs(
-        GRPC::BadStatus.new_status_exception(code, message)) do
+      setup_logging_stubs(nil, code, message) do
         d = create_driver(USE_GRPC_CONFIG, 'test')
         # The API client should retry this once, then throw an exception which
         # gets propagated through the plugin
@@ -168,72 +175,6 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
   end
 
-  # TODO: The code in the non-gRPC and gRPC tests is nearly identical.
-  # Refactor and remove duplication.
-  # TODO: Use status codes instead of int literals.
-  def test_metrics
-    setup_gce_metadata_stubs
-    [
-      [ENABLE_PROMETHEUS_CONFIG, method(:assert_prometheus_metric_value)],
-      [ENABLE_OPENCENSUS_CONFIG, method(:assert_opencensus_metric_value)]
-    ].each do |config, assert_metric_value|
-      [
-        # Single successful request.
-        [false, 0, 1, 1, [1, 0, 1, 0, 0]],
-        # Several successful requests.
-        [false, 0, 2, 1, [2, 0, 2, 0, 0]],
-        # Single successful request with several entries.
-        [false, 0, 1, 2, [1, 0, 2, 0, 0]],
-        # Single failed request that causes logs to be dropped.
-        [true, 16, 1, 1, [0, 1, 0, 1, 0]],
-        # Single failed request that escalates without logs being dropped with
-        # several entries.
-        [true, 13, 1, 2, [0, 0, 0, 0, 2]]
-      ].each do |should_fail, code, request_count, entry_count, metric_values|
-        clear_metrics
-        (1..request_count).each do
-          setup_logging_stubs(
-            if should_fail
-              GRPC::BadStatus.new_status_exception(code, 'SomeMessage')
-            end) do
-            d = create_driver(USE_GRPC_CONFIG + config, 'test')
-            (1..entry_count).each do |i|
-              d.emit('message' => log_entry(i.to_s))
-            end
-            # rubocop:disable Lint/HandleExceptions
-            begin
-              d.run
-            rescue GRPC::BadStatus
-            end
-            # rubocop:enable Lint/HandleExceptions
-          end
-        end
-        successful_requests_count, failed_requests_count,
-        ingested_entries_count, dropped_entries_count,
-        retried_entries_count = metric_values
-        assert_metric_value.call(:stackdriver_successful_requests_count,
-                                 successful_requests_count,
-                                 grpc: true, code: 0)
-        assert_metric_value.call(:stackdriver_ingested_entries_count,
-                                 ingested_entries_count,
-                                 grpc: true, code: 0)
-        assert_metric_value.call(:stackdriver_retried_entries_count,
-                                 retried_entries_count,
-                                 grpc: true, code: code)
-        # Skip failure assertions when code indicates success, because the
-        # assertion will fail in the case when a single metric contains time
-        # series with success and failure events.
-        next if code == 0
-        assert_metric_value.call(:stackdriver_failed_requests_count,
-                                 failed_requests_count,
-                                 grpc: true, code: code)
-        assert_metric_value.call(:stackdriver_dropped_entries_count,
-                                 dropped_entries_count,
-                                 grpc: true, code: code)
-      end
-    end
-  end
-
   # This test looks similar between the grpc and non-grpc paths except that when
   # parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
   # responds with "100".
@@ -428,17 +369,45 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
   end
 
   # Set up grpc stubs to mock the external calls.
-  def setup_logging_stubs(error = nil)
-    if error.nil?
+  def setup_logging_stubs(error = nil, code = nil, message = 'some message')
+    if error.nil? && (code.nil? || code == 0)
       @requests_sent = []
       @grpc_stub = GRPCLoggingMockService.new(@requests_sent)
     else
       @failed_attempts = []
+      # Only fall back to constructing an error with code and message if no
+      # error is passed in.
+      error ||= GRPC::BadStatus.new_status_exception(code, message)
       @grpc_stub = GRPCLoggingMockFailingService.new(error, @failed_attempts)
     end
     yield
   end
 
+  # Whether this is the grpc path
+  def use_grpc
+    true
+  end
+
+  # The OK status code for the grpc path.
+  def ok_status_code
+    0
+  end
+
+  # A client side error status code for the grpc path.
+  def client_error_status_code
+    16
+  end
+
+  # A server side error status code for the grpc path.
+  def server_error_status_code
+    13
+  end
+
+  # The parent error type to expect in the mock
+  def mock_error_type
+    GRPC::BadStatus
+  end
+
   # Verify the number and the content of the log entries match the expectation.
   # The caller can optionally provide a block which is called for each entry.
   def verify_log_entries(n, params, payload_type = 'textPayload',

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.10.4
+  version: 0.10.5
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-29 00:00:00.000000000 Z
+date: 2021-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd