fluent-plugin-google-cloud 0.10.2 → 0.10.7

data/lib/fluent/plugin/out_google_cloud.rb

@@ -563,32 +563,38 @@ module Fluent
         # Uptime should be a gauge, but the metric definition is a counter and
         # we can't change it.
         @uptime_metric = @registry.counter(
-          :uptime, [:version], 'Uptime of Logging agent')
+          :uptime, [:version], 'Uptime of Logging agent',
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         update_uptime
         timer_execute(:update_uptime, 1) { update_uptime }
         @successful_requests_count = @registry.counter(
           :stackdriver_successful_requests_count,
           [:grpc, :code],
-          'A number of successful requests to the Stackdriver Logging API')
+          'A number of successful requests to the Stackdriver Logging API',
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @failed_requests_count = @registry.counter(
           :stackdriver_failed_requests_count,
           [:grpc, :code],
           'A number of failed requests to the Stackdriver Logging '\
-          'API, broken down by the error code')
+          'API, broken down by the error code',
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @ingested_entries_count = @registry.counter(
           :stackdriver_ingested_entries_count,
           [:grpc, :code],
-          'A number of log entries ingested by Stackdriver Logging')
+          'A number of log entries ingested by Stackdriver Logging',
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @dropped_entries_count = @registry.counter(
           :stackdriver_dropped_entries_count,
           [:grpc, :code],
-          'A number of log entries dropped by the Stackdriver output plugin')
+          'A number of log entries dropped by the Stackdriver output plugin',
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @retried_entries_count = @registry.counter(
           :stackdriver_retried_entries_count,
           [:grpc, :code],
           'The number of log entries that failed to be ingested by '\
           'the Stackdriver output plugin due to a transient error '\
-          'and were retried')
+          'and were retried',
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @ok_code = @use_grpc ? GRPC::Core::StatusCodes::OK : 200
       end
 

data/test/plugin/asserts.rb

@@ -37,7 +37,10 @@ module Asserts
     assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
   end
 
-  def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
+  # rubocop:disable Metrics/ParameterLists
+  def assert_prometheus_metric_value(metric_name, expected_value, _prefix,
+                                     _aggregation, _test_driver, labels = {})
+    # rubocop:enable Metrics/ParameterLists
     metric = Prometheus::Client.registry.get(metric_name)
     assert_not_nil(metric)
     metric_value = if labels == :aggregate
@@ -49,7 +52,10 @@ module Asserts
     assert_equal(expected_value, metric_value)
   end
 
-  def assert_opencensus_metric_value(metric_name, expected_value, labels = {})
+  # rubocop:disable Metrics/ParameterLists
+  def assert_opencensus_metric_value(metric_name, expected_value, prefix,
+                                     aggregation, test_driver, labels = {})
+    # rubocop:enable Metrics/ParameterLists
     translator = Monitoring::MetricTranslator.new(metric_name, labels)
     metric_name = translator.name
     labels = translator.translate_labels(labels)
@@ -60,18 +66,22 @@ module Asserts
     # label that never changes during runtime.
     labels.select! { |k, _| translator.view_labels.include? k }
     labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
-    stats_recorder = OpenCensus::Stats.ensure_recorder
-    view_data = stats_recorder.view_data metric_name
-    assert_not_nil(view_data)
+
+    registry = test_driver.instance.instance_variable_get(:@registry)
+    recorder = registry.instance_variable_get(:@recorders)[prefix]
+    view_data = recorder.measure_views_data[metric_name][0].data
+    view = recorder.instance_variable_get(:@views)[metric_name]
+
+    # Assert values in the view.
+    assert_kind_of(aggregation, view.aggregation)
+    assert_equal(labels.keys, view.columns)
+    assert_equal(metric_name, view.measure.name)
+    assert_equal('INT64', view.measure.type)
+
     # For now assume all metrics are counters.
-    assert_kind_of(OpenCensus::Stats::Aggregation::Sum,
-                   view_data.view.aggregation)
-    assert_true(view_data.view.measure.int64?)
-    tag_values = view_data.view.columns.map { |column| labels[column] }
+    tag_values = view.columns.map { |column| labels[column] }
     metric_value = 0
-    if view_data.data.key? tag_values
-      metric_value = view_data.data[tag_values].value
-    end
+    metric_value = view_data[tag_values].value if view_data.key? tag_values
     assert_equal(expected_value, metric_value)
   end
 end

data/test/plugin/base_test.rb

@@ -169,10 +169,11 @@ module BaseTest
       assert_true d.instance.instance_variable_get(:@enable_monitoring)
       registry = d.instance.instance_variable_get(:@registry)
       assert_not_nil registry
-      exporter = registry.instance_variable_get(:@exporter)
-      assert_equal 'custom_resource', exporter.resource_type, "Index #{index}"
+      monitored_resource = registry.instance_variable_get(
+        :@metrics_monitored_resource)
+      assert_equal('custom_resource', monitored_resource.type, "Index #{index}")
       assert_equal({ 'label1' => '123', 'label2' => 'abc' },
-                   exporter.resource_labels, "Index #{index}")
+                   monitored_resource.labels, "Index #{index}")
     end
   end
 
@@ -942,22 +943,30 @@ module BaseTest
           d.emit("tag#{i}", 'message' => log_entry(i))
         end
         d.run
+        @logs_sent.zip(request_log_names).each do |request, log_name|
+          assert_equal log_name, request['logName']
+        end
+        verify_log_entries(log_entry_count, COMPUTE_PARAMS_NO_LOG_NAME,
+                           'textPayload') do |entry, entry_index|
+          verify_default_log_entry_text(entry['textPayload'], entry_index,
+                                        entry)
+          assert_equal entry_log_names[entry_index], entry['logName']
+        end
+        # Verify the number of requests is different based on whether the
+        # 'split_logs_by_tag' flag is enabled.
+        assert_prometheus_metric_value(
+          :stackdriver_successful_requests_count,
+          request_count,
+          'agent.googleapis.com/agent',
+          OpenCensus::Stats::Aggregation::Sum, d,
+          :aggregate)
+        assert_prometheus_metric_value(
+          :stackdriver_ingested_entries_count,
+          log_entry_count,
+          'agent.googleapis.com/agent',
+          OpenCensus::Stats::Aggregation::Sum, d,
+          :aggregate)
       end
-      @logs_sent.zip(request_log_names).each do |request, log_name|
-        assert_equal log_name, request['logName']
-      end
-      verify_log_entries(log_entry_count, COMPUTE_PARAMS_NO_LOG_NAME,
-                         'textPayload') do |entry, entry_index|
-        verify_default_log_entry_text(entry['textPayload'], entry_index,
-                                      entry)
-        assert_equal entry_log_names[entry_index], entry['logName']
-      end
-      # Verify the number of requests is different based on whether the
-      # 'split_logs_by_tag' flag is enabled.
-      assert_prometheus_metric_value(:stackdriver_successful_requests_count,
-                                     request_count, :aggregate)
-      assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
-                                     log_entry_count, :aggregate)
     end
   end
 
@@ -1967,7 +1976,9 @@ module BaseTest
         expected = Time.now.to_i - start_time
         d.instance.update_uptime
         assert_metric_value.call(
-          :uptime, expected, version: Fluent::GoogleCloudOutput.version_string)
+          :uptime, expected, 'agent.googleapis.com/agent',
+          OpenCensus::Stats::Aggregation::Sum, d,
+          version: Fluent::GoogleCloudOutput.version_string)
       rescue Test::Unit::AssertionFailedError
         retry if (retries += 1) < 3
       end
@@ -1975,6 +1986,102 @@ module BaseTest
     end
   end
 
+  def test_metrics
+    setup_gce_metadata_stubs
+    [
+      [ENABLE_PROMETHEUS_CONFIG, method(:assert_prometheus_metric_value)],
+      [ENABLE_OPENCENSUS_CONFIG, method(:assert_opencensus_metric_value)]
+    ].each do |config, assert_metric_value|
+      [
+        # Single successful request.
+        [ok_status_code, 1, 1, [0, 0, 0]],
+        # Several successful requests.
+        [ok_status_code, 2, 1, [0, 0, 0]],
+        # Single successful request with several entries.
+        [ok_status_code, 1, 2, [0, 0, 0]],
+        # Single failed request that causes logs to be dropped.
+        [client_error_status_code, 1, 1, [1, 1, 0]],
+        # Single failed request that escalates without logs being dropped with
+        # several entries.
+        [server_error_status_code, 1, 2, [0, 0, 2]]
+      ].each do |code, request_count, entry_count, metric_values|
+        clear_metrics
+        setup_logging_stubs(nil, code, 'SomeMessage') do
+          (1..request_count).each do |request_index|
+            d = create_driver(config)
+            (1..entry_count).each do |entry_index|
+              d.emit('message' => log_entry(entry_index.to_s))
+            end
+            # rubocop:disable Lint/HandleExceptions
+            begin
+              d.run
+            rescue mock_error_type
+            end
+            # rubocop:enable Lint/HandleExceptions
+            failed_requests_count, dropped_entries_count,
+            retried_entries_count = metric_values
+
+            successful_requests_count = \
+              if code != ok_status_code
+                0
+              elsif config == ENABLE_OPENCENSUS_CONFIG
+                # TODO(b/173215689) Improve the Open Census side of testing.
+                # The test driver instance variables can not survive between
+                # test driver runs. So the open cencensus side counter gets
+                # reset as expected.
+                1
+              else
+                request_index
+              end
+
+            ingested_entries_count = \
+              if code != ok_status_code
+                0
+              elsif config == ENABLE_OPENCENSUS_CONFIG
+                # TODO(b/173215689) Improve the Open Census side of testing.
+                # The test driver instance variables can not survive between
+                # test driver runs. So the open cencensus side counter gets
+                # reset as expected.
+                entry_count
+              else
+                request_index * entry_count
+              end
+
+            assert_metric_value.call(:stackdriver_successful_requests_count,
+                                     successful_requests_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: ok_status_code)
+            assert_metric_value.call(:stackdriver_ingested_entries_count,
+                                     ingested_entries_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: ok_status_code)
+            assert_metric_value.call(:stackdriver_retried_entries_count,
+                                     retried_entries_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: code)
+            # Skip failure assertions when code indicates success, because the
+            # assertion will fail in the case when a single metric contains time
+            # series with success and failure events.
+            next if code == ok_status_code
+            assert_metric_value.call(:stackdriver_failed_requests_count,
+                                     failed_requests_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: code)
+            assert_metric_value.call(:stackdriver_dropped_entries_count,
+                                     dropped_entries_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: code)
+          end
+        end
+      end
+    end
+  end
+
   private
 
   # Provide a stub context that initializes @logs_sent, executes the block and
@@ -2455,7 +2562,32 @@ module BaseTest
   end
 
   # Set up http or grpc stubs to mock the external calls.
-  def setup_logging_stubs
+  def setup_logging_stubs(_error = nil, _code = nil, _message = nil)
+    _undefined
+  end
+
+  # Whether this is the grpc path
+  def use_grpc
+    _undefined
+  end
+
+  # The OK status code.
+  def ok_status_code
+    _undefined
+  end
+
+  # A client side error status code.
+  def client_error_status_code
+    _undefined
+  end
+
+  # A server side error status code.
+  def server_error_status_code
+    _undefined
+  end
+
+  # The parent error type to expect in the mock
+  def mock_error_type
     _undefined
   end
 

data/test/plugin/test_filter_analyze_config.rb

@@ -51,85 +51,115 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
        method(:assert_opencensus_metric_value)]
     ].each do |config, assert_metric_value|
       clear_metrics
-      create_driver(config)
+      d = create_driver(config)
 
       # Default plugins, with default config.
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'source/syslog/tcp',
         is_default_plugin: true,
-        has_default_value: true,
+        has_default_config: true,
         has_ruby_snippet: false)
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'source/tail/apache-access',
         is_default_plugin: true,
-        has_default_value: true,
+        has_default_config: true,
         has_ruby_snippet: false)
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'filter/add_insert_ids',
         is_default_plugin: true,
-        has_default_value: true,
+        has_default_config: true,
         has_ruby_snippet: false)
 
       # Default plugins, with custom config.
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'match/google_cloud',
         is_default_plugin: true,
-        has_default_value: false,
+        has_default_config: false,
         has_ruby_snippet: false)
 
       # Custom plugins, some with embedded Ruby.
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'filter',
         is_default_plugin: false,
-        has_default_value: false,
+        has_default_config: false,
         has_ruby_snippet: false)
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'filter/record_transformer',
         is_default_plugin: false,
-        has_default_value: false,
+        has_default_config: false,
         has_ruby_snippet: true)
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'match/stdout',
         is_default_plugin: false,
-        has_default_value: false,
+        has_default_config: false,
         has_ruby_snippet: true)
 
       # For out_google_cloud, 3 params are present.
       assert_metric_value.call(
-        :stackdriver_config_usage,
+        :plugin_config,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'adjust_invalid_timestamps',
         is_present: true,
-        has_default_value: true)
+        has_default_config: true)
       assert_metric_value.call(
-        :stackdriver_config_usage,
+        :plugin_config,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'autoformat_stackdriver_trace',
         is_present: true,
-        has_default_value: false)
+        has_default_config: false)
       assert_metric_value.call(
-        :stackdriver_config_usage,
+        :plugin_config,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'coerce_to_utf8',
         is_present: true,
-        has_default_value: false)
+        has_default_config: false)
       # The remaining "google_cloud" params are not present.
       # The are no params for "detect_exceptions".
       %w(
@@ -163,30 +193,42 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
         zone
       ).each do |p|
         assert_metric_value.call(
-          :stackdriver_config_usage,
+          :plugin_config,
           1,
+          'agent.googleapis.com/agent/internal/logging/config',
+          OpenCensus::Stats::Aggregation::LastValue,
+          d,
           plugin_name: 'google_cloud',
           param: p,
           is_present: false,
-          has_default_value: false)
+          has_default_config: false)
       end
 
       # We also export values for the bools.
       assert_metric_value.call(
-        :stackdriver_config_bool_values,
+        :config_bool_values,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'adjust_invalid_timestamps',
         value: true)
       assert_metric_value.call(
-        :stackdriver_config_bool_values,
+        :config_bool_values,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'autoformat_stackdriver_trace',
         value: false)
       assert_metric_value.call(
-        :stackdriver_config_bool_values,
+        :config_bool_values,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'coerce_to_utf8',
         value: true)