fluent-plugin-google-cloud 0.10.4 → 0.10.9

data/test/plugin/asserts.rb

@@ -37,7 +37,10 @@ module Asserts
     assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
   end
 
-  def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
+  # rubocop:disable Metrics/ParameterLists
+  def assert_prometheus_metric_value(metric_name, expected_value, _prefix,
+                                     _aggregation, _test_driver, labels = {})
+    # rubocop:enable Metrics/ParameterLists
     metric = Prometheus::Client.registry.get(metric_name)
     assert_not_nil(metric)
     metric_value = if labels == :aggregate
@@ -49,7 +52,10 @@ module Asserts
     assert_equal(expected_value, metric_value)
   end
 
-  def assert_opencensus_metric_value(metric_name, expected_value, labels = {})
+  # rubocop:disable Metrics/ParameterLists
+  def assert_opencensus_metric_value(metric_name, expected_value, prefix,
+                                     aggregation, test_driver, labels = {})
+    # rubocop:enable Metrics/ParameterLists
     translator = Monitoring::MetricTranslator.new(metric_name, labels)
     metric_name = translator.name
     labels = translator.translate_labels(labels)
@@ -60,18 +66,22 @@ module Asserts
     # label that never changes during runtime.
     labels.select! { |k, _| translator.view_labels.include? k }
     labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
-    stats_recorder = OpenCensus::Stats.ensure_recorder
-    view_data = stats_recorder.view_data metric_name
-    assert_not_nil(view_data)
+
+    registry = test_driver.instance.instance_variable_get(:@registry)
+    recorder = registry.instance_variable_get(:@recorders)[prefix]
+    view_data = recorder.measure_views_data[metric_name][0].data
+    view = recorder.instance_variable_get(:@views)[metric_name]
+
+    # Assert values in the view.
+    assert_kind_of(aggregation, view.aggregation)
+    assert_equal(labels.keys, view.columns)
+    assert_equal(metric_name, view.measure.name)
+    assert_equal('INT64', view.measure.type)
+
     # For now assume all metrics are counters.
-    assert_kind_of(OpenCensus::Stats::Aggregation::Sum,
-                   view_data.view.aggregation)
-    assert_true(view_data.view.measure.int64?)
-    tag_values = view_data.view.columns.map { |column| labels[column] }
+    tag_values = view.columns.map { |column| labels[column] }
     metric_value = 0
-    if view_data.data.key? tag_values
-      metric_value = view_data.data[tag_values].value
-    end
+    metric_value = view_data[tag_values].value if view_data.key? tag_values
     assert_equal(expected_value, metric_value)
   end
 end

data/test/plugin/base_test.rb

@@ -943,22 +943,30 @@ module BaseTest
           d.emit("tag#{i}", 'message' => log_entry(i))
         end
         d.run
+        @logs_sent.zip(request_log_names).each do |request, log_name|
+          assert_equal log_name, request['logName']
+        end
+        verify_log_entries(log_entry_count, COMPUTE_PARAMS_NO_LOG_NAME,
+                           'textPayload') do |entry, entry_index|
+          verify_default_log_entry_text(entry['textPayload'], entry_index,
+                                        entry)
+          assert_equal entry_log_names[entry_index], entry['logName']
+        end
+        # Verify the number of requests is different based on whether the
+        # 'split_logs_by_tag' flag is enabled.
+        assert_prometheus_metric_value(
+          :stackdriver_successful_requests_count,
+          request_count,
+          'agent.googleapis.com/agent',
+          OpenCensus::Stats::Aggregation::Sum, d,
+          :aggregate)
+        assert_prometheus_metric_value(
+          :stackdriver_ingested_entries_count,
+          log_entry_count,
+          'agent.googleapis.com/agent',
+          OpenCensus::Stats::Aggregation::Sum, d,
+          :aggregate)
       end
-      @logs_sent.zip(request_log_names).each do |request, log_name|
-        assert_equal log_name, request['logName']
-      end
-      verify_log_entries(log_entry_count, COMPUTE_PARAMS_NO_LOG_NAME,
-                         'textPayload') do |entry, entry_index|
-        verify_default_log_entry_text(entry['textPayload'], entry_index,
-                                      entry)
-        assert_equal entry_log_names[entry_index], entry['logName']
-      end
-      # Verify the number of requests is different based on whether the
-      # 'split_logs_by_tag' flag is enabled.
-      assert_prometheus_metric_value(:stackdriver_successful_requests_count,
-                                     request_count, :aggregate)
-      assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
-                                     log_entry_count, :aggregate)
     end
   end
 
@@ -1968,7 +1976,9 @@ module BaseTest
         expected = Time.now.to_i - start_time
         d.instance.update_uptime
         assert_metric_value.call(
-          :uptime, expected, version: Fluent::GoogleCloudOutput.version_string)
+          :uptime, expected, 'agent.googleapis.com/agent',
+          OpenCensus::Stats::Aggregation::Sum, d,
+          version: Fluent::GoogleCloudOutput.version_string)
       rescue Test::Unit::AssertionFailedError
         retry if (retries += 1) < 3
       end
@@ -1976,6 +1986,102 @@ module BaseTest
     end
   end
 
+  def test_metrics
+    setup_gce_metadata_stubs
+    [
+      [ENABLE_PROMETHEUS_CONFIG, method(:assert_prometheus_metric_value)],
+      [ENABLE_OPENCENSUS_CONFIG, method(:assert_opencensus_metric_value)]
+    ].each do |config, assert_metric_value|
+      [
+        # Single successful request.
+        [ok_status_code, 1, 1, [0, 0, 0]],
+        # Several successful requests.
+        [ok_status_code, 2, 1, [0, 0, 0]],
+        # Single successful request with several entries.
+        [ok_status_code, 1, 2, [0, 0, 0]],
+        # Single failed request that causes logs to be dropped.
+        [client_error_status_code, 1, 1, [1, 1, 0]],
+        # Single failed request that escalates without logs being dropped with
+        # several entries.
+        [server_error_status_code, 1, 2, [0, 0, 2]]
+      ].each do |code, request_count, entry_count, metric_values|
+        clear_metrics
+        setup_logging_stubs(nil, code, 'SomeMessage') do
+          (1..request_count).each do |request_index|
+            d = create_driver(config)
+            (1..entry_count).each do |entry_index|
+              d.emit('message' => log_entry(entry_index.to_s))
+            end
+            # rubocop:disable Lint/HandleExceptions
+            begin
+              d.run
+            rescue mock_error_type
+            end
+            # rubocop:enable Lint/HandleExceptions
+            failed_requests_count, dropped_entries_count,
+            retried_entries_count = metric_values
+
+            successful_requests_count = \
+              if code != ok_status_code
+                0
+              elsif config == ENABLE_OPENCENSUS_CONFIG
+                # TODO(b/173215689) Improve the Open Census side of testing.
+                # The test driver instance variables can not survive between
+                # test driver runs. So the open cencensus side counter gets
+                # reset as expected.
+                1
+              else
+                request_index
+              end
+
+            ingested_entries_count = \
+              if code != ok_status_code
+                0
+              elsif config == ENABLE_OPENCENSUS_CONFIG
+                # TODO(b/173215689) Improve the Open Census side of testing.
+                # The test driver instance variables can not survive between
+                # test driver runs. So the open cencensus side counter gets
+                # reset as expected.
+                entry_count
+              else
+                request_index * entry_count
+              end
+
+            assert_metric_value.call(:stackdriver_successful_requests_count,
+                                     successful_requests_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: ok_status_code)
+            assert_metric_value.call(:stackdriver_ingested_entries_count,
+                                     ingested_entries_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: ok_status_code)
+            assert_metric_value.call(:stackdriver_retried_entries_count,
+                                     retried_entries_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: code)
+            # Skip failure assertions when code indicates success, because the
+            # assertion will fail in the case when a single metric contains time
+            # series with success and failure events.
+            next if code == ok_status_code
+            assert_metric_value.call(:stackdriver_failed_requests_count,
+                                     failed_requests_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: code)
+            assert_metric_value.call(:stackdriver_dropped_entries_count,
+                                     dropped_entries_count,
+                                     'agent.googleapis.com/agent',
+                                     OpenCensus::Stats::Aggregation::Sum, d,
+                                     grpc: use_grpc, code: code)
+          end
+        end
+      end
+    end
+  end
+
   private
 
   # Provide a stub context that initializes @logs_sent, executes the block and
@@ -2456,7 +2562,32 @@ module BaseTest
   end
 
   # Set up http or grpc stubs to mock the external calls.
-  def setup_logging_stubs
+  def setup_logging_stubs(_error = nil, _code = nil, _message = nil)
+    _undefined
+  end
+
+  # Whether this is the grpc path
+  def use_grpc
+    _undefined
+  end
+
+  # The OK status code.
+  def ok_status_code
+    _undefined
+  end
+
+  # A client side error status code.
+  def client_error_status_code
+    _undefined
+  end
+
+  # A server side error status code.
+  def server_error_status_code
+    _undefined
+  end
+
+  # The parent error type to expect in the mock
+  def mock_error_type
     _undefined
   end
 

data/test/plugin/test_filter_analyze_config.rb

@@ -51,12 +51,15 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
        method(:assert_opencensus_metric_value)]
     ].each do |config, assert_metric_value|
       clear_metrics
-      create_driver(config)
+      d = create_driver(config)
 
       # Default plugins, with default config.
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'source/syslog/tcp',
         is_default_plugin: true,
         has_default_config: true,
@@ -64,6 +67,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'source/tail/apache-access',
         is_default_plugin: true,
         has_default_config: true,
@@ -71,6 +77,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'filter/add_insert_ids',
         is_default_plugin: true,
         has_default_config: true,
@@ -80,6 +89,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'match/google_cloud',
         is_default_plugin: true,
         has_default_config: false,
@@ -89,6 +101,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'filter',
         is_default_plugin: false,
         has_default_config: false,
@@ -96,6 +111,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'filter/record_transformer',
         is_default_plugin: false,
         has_default_config: false,
@@ -103,6 +121,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :enabled_plugins,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'match/stdout',
         is_default_plugin: false,
         has_default_config: false,
@@ -112,6 +133,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :plugin_config,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'adjust_invalid_timestamps',
         is_present: true,
@@ -119,6 +143,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :plugin_config,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'autoformat_stackdriver_trace',
         is_present: true,
@@ -126,6 +153,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :plugin_config,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'coerce_to_utf8',
         is_present: true,
@@ -165,6 +195,9 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
         assert_metric_value.call(
           :plugin_config,
           1,
+          'agent.googleapis.com/agent/internal/logging/config',
+          OpenCensus::Stats::Aggregation::LastValue,
+          d,
           plugin_name: 'google_cloud',
           param: p,
           is_present: false,
@@ -175,18 +208,27 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
       assert_metric_value.call(
         :config_bool_values,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'adjust_invalid_timestamps',
         value: true)
       assert_metric_value.call(
         :config_bool_values,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'autoformat_stackdriver_trace',
         value: false)
       assert_metric_value.call(
         :config_bool_values,
         1,
+        'agent.googleapis.com/agent/internal/logging/config',
+        OpenCensus::Stats::Aggregation::LastValue,
+        d,
         plugin_name: 'google_cloud',
         param: 'coerce_to_utf8',
         value: true)

data/test/plugin/test_out_google_cloud.rb

@@ -86,13 +86,21 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     end
     d.run
     assert_prometheus_metric_value(
-      :stackdriver_successful_requests_count, 1, grpc: false, code: 200)
+      :stackdriver_successful_requests_count, 1,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 200)
     assert_prometheus_metric_value(
-      :stackdriver_ingested_entries_count, 1, grpc: false, code: 200)
+      :stackdriver_ingested_entries_count, 1,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 200)
     assert_prometheus_metric_value(
-      :stackdriver_dropped_entries_count, 2, grpc: false, code: 3)
+      :stackdriver_dropped_entries_count, 2,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 3)
     assert_prometheus_metric_value(
-      :stackdriver_dropped_entries_count, 1, grpc: false, code: 7)
+      :stackdriver_dropped_entries_count, 1,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 7)
     assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
   end
 
@@ -109,13 +117,21 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     d.emit('message' => log_entry(0))
     d.run
     assert_prometheus_metric_value(
-      :stackdriver_successful_requests_count, 0, grpc: false, code: 200)
+      :stackdriver_successful_requests_count, 0,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 200)
     assert_prometheus_metric_value(
-      :stackdriver_failed_requests_count, 1, grpc: false, code: 400)
+      :stackdriver_failed_requests_count, 1,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 400)
     assert_prometheus_metric_value(
-      :stackdriver_ingested_entries_count, 0, grpc: false, code: 200)
+      :stackdriver_ingested_entries_count, 0,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 200)
     assert_prometheus_metric_value(
-      :stackdriver_dropped_entries_count, 1, grpc: false, code: 400)
+      :stackdriver_dropped_entries_count, 1,
+      'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
+      grpc: false, code: 400)
     assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
   end
 
@@ -138,71 +154,6 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_equal 1, exception_count
   end
 
-  # TODO: The code in the non-gRPC and gRPC tests is nearly identical.
-  # Refactor and remove duplication.
-  # TODO: Use status codes instead of int literals.
-  def test_metrics
-    setup_gce_metadata_stubs
-    [
-      [ENABLE_PROMETHEUS_CONFIG, method(:assert_prometheus_metric_value)],
-      [ENABLE_OPENCENSUS_CONFIG, method(:assert_opencensus_metric_value)]
-    ].each do |config, assert_metric_value|
-      [
-        # Single successful request.
-        [200, 1, 1, [1, 0, 1, 0, 0]],
-        # Several successful requests.
-        [200, 2, 1, [2, 0, 2, 0, 0]],
-        # Single successful request with several entries.
-        [200, 1, 2, [1, 0, 2, 0, 0]],
-        # Single failed request that causes logs to be dropped.
-        [401, 1, 1, [0, 1, 0, 1, 0]],
-        # Single failed request that escalates without logs being dropped with
-        # several entries.
-        [500, 1, 2, [0, 0, 0, 0, 2]]
-      ].each do |code, request_count, entry_count, metric_values|
-        clear_metrics
-        # TODO: Do this as part of setup_logging_stubs.
-        stub_request(:post, WRITE_LOG_ENTRIES_URI)
-          .to_return(status: code, body: 'Some Message')
-        (1..request_count).each do
-          d = create_driver(config)
-          (1..entry_count).each do |i|
-            d.emit('message' => log_entry(i.to_s))
-          end
-          # rubocop:disable Lint/HandleExceptions
-          begin
-            d.run
-          rescue Google::Apis::AuthorizationError
-          rescue Google::Apis::ServerError
-          end
-          # rubocop:enable Lint/HandleExceptions
-        end
-        successful_requests_count, failed_requests_count,
-        ingested_entries_count, dropped_entries_count,
-        retried_entries_count = metric_values
-        assert_metric_value.call(:stackdriver_successful_requests_count,
-                                 successful_requests_count,
-                                 grpc: false, code: 200)
-        assert_metric_value.call(:stackdriver_ingested_entries_count,
-                                 ingested_entries_count,
-                                 grpc: false, code: 200)
-        assert_metric_value.call(:stackdriver_retried_entries_count,
-                                 retried_entries_count,
-                                 grpc: false, code: code)
-        # Skip failure assertions when code indicates success, because the
-        # assertion will fail in the case when a single metric contains time
-        # series with success and failure events.
-        next if code == 200
-        assert_metric_value.call(:stackdriver_failed_requests_count,
-                                 failed_requests_count,
-                                 grpc: false, code: code)
-        assert_metric_value.call(:stackdriver_dropped_entries_count,
-                                 dropped_entries_count,
-                                 grpc: false, code: code)
-      end
-    end
-  end
-
   # This test looks similar between the grpc and non-grpc paths except that when
   # parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
   # responds with "100".
@@ -400,14 +351,39 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   # Set up http stubs to mock the external calls.
-  def setup_logging_stubs
+  def setup_logging_stubs(_error = nil, code = nil, message = 'some message')
     stub_request(:post, WRITE_LOG_ENTRIES_URI).to_return do |request|
       @logs_sent << JSON.parse(request.body)
-      { body: '' }
+      { status: code, body: message }
     end
     yield
   end
 
+  # Whether this is the grpc path
+  def use_grpc
+    false
+  end
+
+  # The OK status code for the grpc path.
+  def ok_status_code
+    200
+  end
+
+  # A client side error status code for the grpc path.
+  def client_error_status_code
+    401
+  end
+
+  # A server side error status code for the grpc path.
+  def server_error_status_code
+    500
+  end
+
+  # The parent error type to expect in the mock
+  def mock_error_type
+    Google::Apis::Error
+  end
+
   # The conversions from user input to output.
   def latency_conversion
     {