fluent-plugin-google-cloud 0.8.5 → 0.10.0

data/test/plugin/asserts.rb

@@ -0,0 +1,77 @@
+# Copyright 2020 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'prometheus/client'
+
+# Additional assert functions.
+module Asserts
+  # For an optional field with default values, Protobuf omits the field when it
+  # is deserialized to json. So we need to add an extra check for gRPC which
+  # uses Protobuf.
+  #
+  # An optional block can be passed in if we need to assert something other than
+  # a plain equal. e.g. assert_in_delta.
+  def assert_equal_with_default(_field, _expected_value, _default_value, _entry)
+    _undefined
+  end
+
+  # Compare the timestamp seconds and nanoseconds with the expected timestamp.
+  def assert_timestamp_matches(expected_ts, ts_secs, ts_nanos, entry)
+    assert_equal expected_ts.tv_sec, ts_secs, entry
+    # Fluentd v0.14 onwards supports nanosecond timestamp values.
+    # Added in 600 ns delta to avoid flaky tests introduced
+    # due to rounding error in double-precision floating-point numbers
+    # (to account for the missing 9 bits of precision ~ 512 ns).
+    # See http://wikipedia.org/wiki/Double-precision_floating-point_format.
+    assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
+  end
+
+  def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
+    metric = Prometheus::Client.registry.get(metric_name)
+    assert_not_nil(metric)
+    metric_value = if labels == :aggregate
+                     # Sum up all metric values regardless of the labels.
+                     metric.values.values.reduce(0.0, :+)
+                   else
+                     metric.get(labels)
+                   end
+    assert_equal(expected_value, metric_value)
+  end
+
+  def assert_opencensus_metric_value(metric_name, expected_value, labels = {})
+    translator = Monitoring::MetricTranslator.new(metric_name, labels)
+    metric_name = translator.name
+    labels = translator.translate_labels(labels)
+    # The next line collapses the labels to assert against the aggregated data,
+    # which can have some labels removed. Without this, it would test against
+    # the raw data. The view is more representative of the user experience, even
+    # though both tests should work because currently we only aggregate away one
+    # label that never changes during runtime.
+    labels.select! { |k, _| translator.view_labels.include? k }
+    labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
+    stats_recorder = OpenCensus::Stats.ensure_recorder
+    view_data = stats_recorder.view_data metric_name
+    assert_not_nil(view_data)
+    # For now assume all metrics are counters.
+    assert_kind_of(OpenCensus::Stats::Aggregation::Sum,
+                   view_data.view.aggregation)
+    assert_true(view_data.view.measure.int64?)
+    tag_values = view_data.view.columns.map { |column| labels[column] }
+    metric_value = 0
+    if view_data.data.key? tag_values
+      metric_value = view_data.data[tag_values].value
+    end
+    assert_equal(expected_value, metric_value)
+  end
+end

data/test/plugin/base_test.rb

@@ -19,10 +19,12 @@ Coveralls.wear!
 require 'google/apis'
 require 'helper'
 require 'mocha/test_unit'
-require 'webmock/test_unit'
 require 'prometheus/client'
+require 'webmock/test_unit'
 
+require_relative 'asserts'
 require_relative 'constants'
+require_relative 'utils'
 
 module Monitoring
   # Prevent OpenCensus from writing to the network.
@@ -35,22 +37,13 @@ end
 
 # Unit tests for Google Cloud Logging plugin
 module BaseTest
+  include Asserts
   include Constants
+  include Utils
 
   def setup
     Fluent::Test.setup
-    # delete environment variables that googleauth uses to find credentials.
-    ENV.delete(CREDENTIALS_PATH_ENV_VAR)
-    # service account env.
-    ENV.delete(PRIVATE_KEY_VAR)
-    ENV.delete(CLIENT_EMAIL_VAR)
-    ENV.delete(PROJECT_ID_VAR)
-    # authorized_user env.
-    ENV.delete(CLIENT_ID_VAR)
-    ENV.delete(CLIENT_SECRET_VAR)
-    ENV.delete(REFRESH_TOKEN_VAR)
-    # home var, which is used to find $HOME/.gcloud/...
-    ENV.delete('HOME')
+    delete_env_vars
 
     # Unregister Prometheus metrics.
     registry = Prometheus::Client.registry
@@ -60,7 +53,7 @@ module BaseTest
     registry.unregister(:stackdriver_dropped_entries_count)
     registry.unregister(:stackdriver_retried_entries_count)
 
-    setup_auth_stubs
+    setup_auth_stubs('https://www.googleapis.com/oauth2/v4/token')
     @logs_sent = []
   end
 
@@ -105,7 +98,7 @@ module BaseTest
 
   def test_configure_metadata_missing_parts_on_other_platforms
     setup_no_metadata_service_stubs
-    Fluent::GoogleCloudOutput::CredentialsInfo.stubs(:project_id).returns(nil)
+    Common::Utils::CredentialsInfo.stubs(:project_id).returns(nil)
     [[CONFIG_MISSING_METADATA_PROJECT_ID, ['project_id'], false],
      [CONFIG_MISSING_METADATA_ZONE, [], true],
      [CONFIG_MISSING_METADATA_VM_ID, [], true],
@@ -129,8 +122,8 @@ module BaseTest
 
   def test_configure_ignores_unknown_monitoring_type
     # Verify that driver creation succeeds when monitoring type is not
-    # "prometheus" (in which case, we simply don't record metrics),
-    # and that the counters are set to nil.
+    # "prometheus" or "opencensus" (in which case, we simply don't record
+    # metrics), and that the counters are set to nil.
     setup_gce_metadata_stubs
     create_driver(CONFIG_UNKNOWN_MONITORING_TYPE)
     assert_nil(Prometheus::Client.registry.get(
@@ -143,6 +136,67 @@ module BaseTest
                  :stackdriver_dropped_entries_count))
     assert_nil(Prometheus::Client.registry.get(
                  :stackdriver_retried_entries_count))
+    assert_nil(OpenCensus::Stats::MeasureRegistry.get(
+                 Monitoring::MetricTranslator.new(
+                   :stackdriver_successful_requests_count, {})))
+    assert_nil(OpenCensus::Stats::MeasureRegistry.get(
+                 Monitoring::MetricTranslator.new(
+                   :stackdriver_failed_requests_count, {})))
+    assert_nil(OpenCensus::Stats::MeasureRegistry.get(
+                 Monitoring::MetricTranslator.new(
+                   :stackdriver_ingested_entries_count, {})))
+    assert_nil(OpenCensus::Stats::MeasureRegistry.get(
+                 Monitoring::MetricTranslator.new(
+                   :stackdriver_dropped_entries_count, {})))
+    assert_nil(OpenCensus::Stats::MeasureRegistry.get(
+                 Monitoring::MetricTranslator.new(
+                   :stackdriver_retried_entries_count, {})))
+  end
+
+  def test_configure_uses_metrics_resource
+    setup_gce_metadata_stubs
+    [CONFIG_METRICS_RESOURCE_JSON,
+     CONFIG_METRICS_RESOURCE_HASH,
+     CONFIG_METRICS_RESOURCE_JSON_HASH
+    ].each_with_index do |config, index|
+      d = create_driver(config)
+      assert_equal 'custom_resource', d.instance.monitoring_resource.type, \
+                   "Index #{index}"
+      assert_equal '123', d.instance.monitoring_resource.labels['label1'], \
+                   "Index #{index}"
+      assert_equal 'abc', d.instance.monitoring_resource.labels['label2'], \
+                   "Index #{index}"
+      assert_true d.instance.instance_variable_get(:@enable_monitoring)
+      registry = d.instance.instance_variable_get(:@registry)
+      assert_not_nil registry
+      exporter = registry.instance_variable_get(:@exporter)
+      assert_equal 'custom_resource', exporter.resource_type, "Index #{index}"
+      assert_equal({ 'label1' => '123', 'label2' => 'abc' },
+                   exporter.resource_labels, "Index #{index}")
+    end
+  end
+
+  def test_configure_metrics_resource_validation
+    setup_gce_metadata_stubs
+    {
+      CONFIG_METRICS_RESOURCE_JSON_NO_TYPE => /type must be a string/,
+      CONFIG_METRICS_RESOURCE_JSON_BAD_LABELS => /labels must be a hash/,
+      CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS =>
+        /unrecognized keys: \[:random\]/,
+      CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_LABELS =>
+        /unrecognized keys: \[:"labels\.random"\]/,
+      CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_NO_LABELS =>
+        /unrecognized keys: \[:random\]/
+    }.each_with_index do |(config, pattern), index|
+      begin
+        create_driver(config)
+        assert false,
+               "Invalid config at index #{index} should have raised an error."
+      rescue Fluent::ConfigError => error
+        assert error.message.match?(pattern), \
+               "Index #{index} failed: got #{error.message}."
+      end
+    end
   end
 
   def test_metadata_loading
@@ -1923,116 +1977,6 @@ module BaseTest
 
   private
 
-  def stub_metadata_request(metadata_path, response_body)
-    stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
-                 metadata_path)
-      .to_return(body: response_body, status: 200,
-                 headers: { 'Content-Length' => response_body.length })
-  end
-
-  def setup_no_metadata_service_stubs
-    # Simulate a machine with no metadata service present
-    stub_request(:any, %r{http://169.254.169.254/.*})
-      .to_raise(Errno::EHOSTUNREACH)
-  end
-
-  def setup_gce_metadata_stubs
-    # Stub the root, used for platform detection by the plugin and 'googleauth'.
-    stub_request(:get, 'http://169.254.169.254')
-      .to_return(status: 200, headers: { 'Metadata-Flavor' => 'Google' })
-
-    # Create stubs for all the GCE metadata lookups the agent needs to make.
-    stub_metadata_request('project/project-id', PROJECT_ID)
-    stub_metadata_request('instance/zone', FULLY_QUALIFIED_ZONE)
-    stub_metadata_request('instance/id', VM_ID)
-    stub_metadata_request('instance/attributes/',
-                          "attribute1\nattribute2\nattribute3")
-
-    # Used by 'googleauth' to fetch the default service account credentials.
-    stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' \
-                 'instance/service-accounts/default/token')
-      .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
-                 status: 200,
-                 headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
-                            'Content-Type' => 'application/json' })
-  end
-
-  def setup_ec2_metadata_stubs
-    # Stub the root, used for platform detection.
-    stub_request(:get, 'http://169.254.169.254')
-      .to_return(status: 200, headers: { 'Server' => 'EC2ws' })
-
-    # Stub the identity document lookup made by the agent.
-    stub_request(:get, 'http://169.254.169.254/latest/dynamic/' \
-                 'instance-identity/document')
-      .to_return(body: EC2_IDENTITY_DOCUMENT, status: 200,
-                 headers: { 'Content-Length' => EC2_IDENTITY_DOCUMENT.length })
-  end
-
-  def setup_auth_stubs
-    # Used when loading credentials from a JSON file.
-    stub_request(:post, 'https://www.googleapis.com/oauth2/v4/token')
-      .with(body: hash_including(grant_type: AUTH_GRANT_TYPE))
-      .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
-                 status: 200,
-                 headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
-                            'Content-Type' => 'application/json' })
-
-    stub_request(:post, 'https://www.googleapis.com/oauth2/v4/token')
-      .with(body: hash_including(grant_type: 'refresh_token'))
-      .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
-                 status: 200,
-                 headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
-                            'Content-Type' => 'application/json' })
-  end
-
-  def setup_managed_vm_metadata_stubs
-    stub_metadata_request(
-      'instance/attributes/',
-      "attribute1\ngae_backend_name\ngae_backend_version\nlast_attribute")
-    stub_metadata_request('instance/attributes/gae_backend_name',
-                          MANAGED_VM_BACKEND_NAME)
-    stub_metadata_request('instance/attributes/gae_backend_version',
-                          MANAGED_VM_BACKEND_VERSION)
-  end
-
-  def setup_k8s_metadata_stubs(should_respond = true)
-    if should_respond
-      stub_metadata_request(
-        'instance/attributes/',
-        "attribute1\ncluster-location\ncluster-name\nlast_attribute")
-      stub_metadata_request('instance/attributes/cluster-location',
-                            K8S_LOCATION2)
-      stub_metadata_request('instance/attributes/cluster-name',
-                            K8S_CLUSTER_NAME)
-    else
-      ['cluster-location', 'cluster-name'].each do |metadata_name|
-        stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
-          .to_return(status: 404,
-                     body: 'The requested URL /computeMetadata/v1/instance/' \
-                           "attributes/#{metadata_name} was not found on this" \
-                           ' server.')
-      end
-    end
-  end
-
-  def setup_dataproc_metadata_stubs
-    stub_metadata_request(
-      'instance/attributes/',
-      "attribute1\ndataproc-cluster-uuid\ndataproc-cluster-name")
-    stub_metadata_request('instance/attributes/dataproc-cluster-name',
-                          DATAPROC_CLUSTER_NAME)
-    stub_metadata_request('instance/attributes/dataproc-cluster-uuid',
-                          DATAPROC_CLUSTER_UUID)
-    stub_metadata_request('instance/attributes/dataproc-region',
-                          DATAPROC_REGION)
-  end
-
-  def clear_metrics
-    Prometheus::Client.registry.instance_variable_set('@metrics', {})
-    OpenCensus::Stats.ensure_recorder.clear_stats
-  end
-
   # Provide a stub context that initializes @logs_sent, executes the block and
   # resets WebMock at the end.
   def new_stub_context
@@ -2074,18 +2018,6 @@ module BaseTest
     }
   end
 
-  def gke_container_log_entry(log)
-    {
-      log: log,
-      LOCAL_RESOURCE_ID_KEY =>
-        "#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
-        ".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}"
-    }
-  end
-
-  # TODO(qingling128): Temporary fallback for metadata agent restarts.
-  # k8s resources.
-
   def k8s_container_log_entry(log,
                               local_resource_id: K8S_LOCAL_RESOURCE_ID)
     {
@@ -2534,65 +2466,6 @@ module BaseTest
     _undefined
   end
 
-  # For an optional field with default values, Protobuf omits the field when it
-  # is deserialized to json. So we need to add an extra check for gRPC which
-  # uses Protobuf.
-  #
-  # An optional block can be passed in if we need to assert something other than
-  # a plain equal. e.g. assert_in_delta.
-  def assert_equal_with_default(_field, _expected_value, _default_value, _entry)
-    _undefined
-  end
-
-  # Compare the timestamp seconds and nanoseconds with the expected timestamp.
-  def assert_timestamp_matches(expected_ts, ts_secs, ts_nanos, entry)
-    assert_equal expected_ts.tv_sec, ts_secs, entry
-    # Fluentd v0.14 onwards supports nanosecond timestamp values.
-    # Added in 600 ns delta to avoid flaky tests introduced
-    # due to rounding error in double-precision floating-point numbers
-    # (to account for the missing 9 bits of precision ~ 512 ns).
-    # See http://wikipedia.org/wiki/Double-precision_floating-point_format.
-    assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
-  end
-
-  def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
-    metric = Prometheus::Client.registry.get(metric_name)
-    assert_not_nil(metric)
-    metric_value = if labels == :aggregate
-                     # Sum up all metric values regardless of the labels.
-                     metric.values.values.reduce(0.0, :+)
-                   else
-                     metric.get(labels)
-                   end
-    assert_equal(expected_value, metric_value)
-  end
-
-  def assert_opencensus_metric_value(metric_name, expected_value, labels = {})
-    translator = Monitoring::MetricTranslator.new(metric_name, labels)
-    metric_name = translator.name
-    labels = translator.translate_labels(labels)
-    # The next line collapses the labels to assert against the aggregated data,
-    # which can have some labels removed. Without this, it would test against
-    # the raw data. The view is more representative of the user experience, even
-    # though both tests should work because currently we only aggregate away one
-    # label that never changes during runtime.
-    labels.select! { |k, _| translator.view_labels.include? k }
-    labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
-    stats_recorder = OpenCensus::Stats.ensure_recorder
-    view_data = stats_recorder.view_data metric_name
-    assert_not_nil(view_data)
-    # For now assume all metrics are counters.
-    assert_kind_of(OpenCensus::Stats::Aggregation::Sum,
-                   view_data.view.aggregation)
-    assert_true(view_data.view.measure.int64?)
-    tag_values = view_data.view.columns.map { |column| labels[column] }
-    metric_value = 0
-    if view_data.data.key? tag_values
-      metric_value = view_data.data[tag_values].value
-    end
-    assert_equal(expected_value, metric_value)
-  end
-
   # Defined in specific gRPC or REST files.
   def expected_operation_message2
     _undefined

data/test/plugin/constants.rb

@@ -45,7 +45,7 @@ end
 
 # Constants used by unit tests for Google Cloud Logging plugin.
 module Constants
-  include Fluent::GoogleCloudOutput::ServiceConstants
+  include Common::ServiceConstants
   include Fluent::GoogleCloudOutput::ConfigConstants
   include Fluent::GoogleCloudOutput::InternalConstants
 
@@ -433,6 +433,56 @@ module Constants
     monitoring_type    not_prometheus
   ).freeze
 
+  # rubocop:disable Metrics/LineLength
+  CONFIG_METRICS_RESOURCE_JSON = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"type":"custom_resource","labels":{"label1":"123","label2":"abc"}}
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_HASH = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   type:custom_resource, labels.label1:123, labels.label2:abc
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_JSON_HASH = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"type":"custom_resource","labels.label1":"123","labels.label2":"abc"}
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_JSON_NO_TYPE = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"labels":{"label1":"123","label2":"abc"}}
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_JSON_BAD_LABELS = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"type":"custom_resource","labels":"123"}
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"type":"custom_resource","labels":{"label1":"123"},"random":"x"}
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_LABELS = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"type":"custom_resource","labels":{"label1":"123"},"labels.random":"x"}
+  ).freeze
+
+  CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_NO_LABELS = %(
+    enable_monitoring  true
+    monitoring_type    opencensus
+    metrics_resource   {"type":"custom_resource","labels.label1":"123","random":"x"}
+  ).freeze
+  # rubocop:enable Metrics/LineLength
+
   # For statusz.
   CONFIG_STATUSZ = %(
     statusz_port                  5678
@@ -471,6 +521,22 @@ module Constants
     zone                          asia-east2
   ).freeze
 
+  # For analyze_config.
+  CONFIG_ANALYZE_CONFIG_PROMETHEUS = %(
+    google_fluentd_config_path          \
+      test/plugin/data/google-fluentd-custom.conf
+    google_fluentd_baseline_config_path \
+      test/plugin/data/google-fluentd-baseline.conf
+    monitoring_type prometheus
+  ).freeze
+  CONFIG_ANALYZE_CONFIG_OPENCENSUS = %(
+    google_fluentd_config_path          \
+      test/plugin/data/google-fluentd-custom.conf
+    google_fluentd_baseline_config_path \
+      test/plugin/data/google-fluentd-baseline.conf
+    monitoring_type opencensus
+  ).freeze
+
   # Service configurations for various services.
 
   # GCE.