fluent-plugin-google-cloud 0.8.2 → 0.8.7

data/test/plugin/asserts.rb

@@ -0,0 +1,75 @@
+# Copyright 2020 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Additional assert functions.
+module Asserts
+  # For an optional field with default values, Protobuf omits the field when it
+  # is deserialized to json. So we need to add an extra check for gRPC which
+  # uses Protobuf.
+  #
+  # An optional block can be passed in if we need to assert something other than
+  # a plain equal. e.g. assert_in_delta.
+  def assert_equal_with_default(_field, _expected_value, _default_value, _entry)
+    _undefined
+  end
+
+  # Compare the timestamp seconds and nanoseconds with the expected timestamp.
+  def assert_timestamp_matches(expected_ts, ts_secs, ts_nanos, entry)
+    assert_equal expected_ts.tv_sec, ts_secs, entry
+    # Fluentd v0.14 onwards supports nanosecond timestamp values.
+    # Added in 600 ns delta to avoid flaky tests introduced
+    # due to rounding error in double-precision floating-point numbers
+    # (to account for the missing 9 bits of precision ~ 512 ns).
+    # See http://wikipedia.org/wiki/Double-precision_floating-point_format.
+    assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
+  end
+
+  def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
+    metric = Prometheus::Client.registry.get(metric_name)
+    assert_not_nil(metric)
+    metric_value = if labels == :aggregate
+                     # Sum up all metric values regardless of the labels.
+                     metric.values.values.reduce(0.0, :+)
+                   else
+                     metric.get(labels)
+                   end
+    assert_equal(expected_value, metric_value)
+  end
+
+  def assert_opencensus_metric_value(metric_name, expected_value, labels = {})
+    translator = Monitoring::MetricTranslator.new(metric_name, labels)
+    metric_name = translator.name
+    labels = translator.translate_labels(labels)
+    # The next line collapses the labels to assert against the aggregated data,
+    # which can have some labels removed. Without this, it would test against
+    # the raw data. The view is more representative of the user experience, even
+    # though both tests should work because currently we only aggregate away one
+    # label that never changes during runtime.
+    labels.select! { |k, _| translator.view_labels.include? k }
+    labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
+    stats_recorder = OpenCensus::Stats.ensure_recorder
+    view_data = stats_recorder.view_data metric_name
+    assert_not_nil(view_data)
+    # For now assume all metrics are counters.
+    assert_kind_of(OpenCensus::Stats::Aggregation::Sum,
+                   view_data.view.aggregation)
+    assert_true(view_data.view.measure.int64?)
+    tag_values = view_data.view.columns.map { |column| labels[column] }
+    metric_value = 0
+    if view_data.data.key? tag_values
+      metric_value = view_data.data[tag_values].value
+    end
+    assert_equal(expected_value, metric_value)
+  end
+end

data/test/plugin/base_test.rb

@@ -22,6 +22,7 @@ require 'mocha/test_unit'
 require 'webmock/test_unit'
 require 'prometheus/client'
 
+require_relative 'asserts'
 require_relative 'constants'
 
 module Monitoring
@@ -35,6 +36,7 @@ end
 
 # Unit tests for Google Cloud Logging plugin
 module BaseTest
+  include Asserts
   include Constants
 
   def setup
@@ -105,7 +107,7 @@ module BaseTest
 
   def test_configure_metadata_missing_parts_on_other_platforms
     setup_no_metadata_service_stubs
-    Fluent::GoogleCloudOutput::CredentialsInfo.stubs(:project_id).returns(nil)
+    Common::Utils::CredentialsInfo.stubs(:project_id).returns(nil)
     [[CONFIG_MISSING_METADATA_PROJECT_ID, ['project_id'], false],
      [CONFIG_MISSING_METADATA_ZONE, [], true],
      [CONFIG_MISSING_METADATA_VM_ID, [], true],
@@ -2534,65 +2536,6 @@ module BaseTest
     _undefined
   end
 
-  # For an optional field with default values, Protobuf omits the field when it
-  # is deserialized to json. So we need to add an extra check for gRPC which
-  # uses Protobuf.
-  #
-  # An optional block can be passed in if we need to assert something other than
-  # a plain equal. e.g. assert_in_delta.
-  def assert_equal_with_default(_field, _expected_value, _default_value, _entry)
-    _undefined
-  end
-
-  # Compare the timestamp seconds and nanoseconds with the expected timestamp.
-  def assert_timestamp_matches(expected_ts, ts_secs, ts_nanos, entry)
-    assert_equal expected_ts.tv_sec, ts_secs, entry
-    # Fluentd v0.14 onwards supports nanosecond timestamp values.
-    # Added in 600 ns delta to avoid flaky tests introduced
-    # due to rounding error in double-precision floating-point numbers
-    # (to account for the missing 9 bits of precision ~ 512 ns).
-    # See http://wikipedia.org/wiki/Double-precision_floating-point_format.
-    assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
-  end
-
-  def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
-    metric = Prometheus::Client.registry.get(metric_name)
-    assert_not_nil(metric)
-    metric_value = if labels == :aggregate
-                     # Sum up all metric values regardless of the labels.
-                     metric.values.values.reduce(0.0, :+)
-                   else
-                     metric.get(labels)
-                   end
-    assert_equal(expected_value, metric_value)
-  end
-
-  def assert_opencensus_metric_value(metric_name, expected_value, labels = {})
-    translator = Monitoring::MetricTranslator.new(metric_name, labels)
-    metric_name = translator.name
-    labels = translator.translate_labels(labels)
-    # The next line collapses the labels to assert against the aggregated data,
-    # which can have some labels removed. Without this, it would test against
-    # the raw data. The view is more representative of the user experience, even
-    # though both tests should work because currently we only aggregate away one
-    # label that never changes during runtime.
-    labels.select! { |k, _| translator.view_labels.include? k }
-    labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
-    stats_recorder = OpenCensus::Stats.ensure_recorder
-    view_data = stats_recorder.view_data metric_name
-    assert_not_nil(view_data)
-    # For now assume all metrics are counters.
-    assert_kind_of(OpenCensus::Stats::Aggregation::Sum,
-                   view_data.view.aggregation)
-    assert_true(view_data.view.measure.int64?)
-    tag_values = view_data.view.columns.map { |column| labels[column] }
-    metric_value = 0
-    if view_data.data.key? tag_values
-      metric_value = view_data.data[tag_values].value
-    end
-    assert_equal(expected_value, metric_value)
-  end
-
   # Defined in specific gRPC or REST files.
   def expected_operation_message2
     _undefined

data/test/plugin/constants.rb

@@ -45,7 +45,7 @@ end
 
 # Constants used by unit tests for Google Cloud Logging plugin.
 module Constants
-  include Fluent::GoogleCloudOutput::ServiceConstants
+  include Common::ServiceConstants
   include Fluent::GoogleCloudOutput::ConfigConstants
   include Fluent::GoogleCloudOutput::InternalConstants
 
@@ -471,6 +471,14 @@ module Constants
     zone                          asia-east2
   ).freeze
 
+  # For analyze_config.
+  CONFIG_ANALYZE_CONFIG = %(
+    google_fluentd_config_path          \
+      test/plugin/data/google-fluentd-custom.conf
+    google_fluentd_baseline_config_path \
+      test/plugin/data/google-fluentd-baseline.conf
+  ).freeze
+
   # Service configurations for various services.
 
   # GCE.

data/test/plugin/data/google-fluentd-baseline.conf

@@ -0,0 +1,24 @@
+<source>
+  @type syslog
+  port 514
+  protocol_type tcp
+  bind 127.0.0.1
+  tag syslog
+</source>
+
+<source>
+  @type tail
+  path /var/log/apache*/access.log,/var/log/apache*/access_log,/var/log/httpd/access.log,/var/log/httpd/access_log
+  pos_file /var/lib/google-fluentd/pos/apache-access.pos
+  tag apache-access
+</source>
+
+<filter **>
+  @type add_insert_ids
+</filter>
+
+<match **>
+  @type google_cloud
+  adjust_invalid_timestamps    true
+  autoformat_stackdriver_trace true
+</match>

data/test/plugin/data/google-fluentd-custom.conf

@@ -0,0 +1,40 @@
+<source>
+  @type syslog
+  port 514
+  protocol_type tcp
+  bind 127.0.0.1
+  tag syslog
+</source>
+
+<source>
+  @type tail
+  path /var/log/apache*/access.log,/var/log/apache*/access_log,/var/log/httpd/access.log,/var/log/httpd/access_log
+  pos_file /var/lib/google-fluentd/pos/apache-access.pos
+  tag apache-access
+</source>
+
+<filter **>
+  @type add_insert_ids
+</filter>
+
+<match **>
+  @type google_cloud
+  adjust_invalid_timestamps    true
+  autoformat_stackdriver_trace false
+  coerce_to_utf8               true
+</match>
+
+<filter **>
+  @type some_custom_filter
+</filter>
+
+<filter **>
+  @type record_transformer
+  <record>
+    host_param "#{Socket.gethostname}"
+  </record>
+</filter>
+
+<match "app.#{ENV['FLUENTD_TAG']}">
+  @type stdout
+</match>

data/test/plugin/test_filter_analyze_config.rb

@@ -0,0 +1,187 @@
+# Copyright 2020 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative '../helper'
+require_relative 'asserts'
+require_relative 'constants'
+
+require 'fluent/test/driver/filter'
+require 'fluent/plugin/filter_analyze_config'
+
+# Unit tests for filter_analyze_config plugin.
+class FilterAnalyzeConfigTest < Test::Unit::TestCase
+  include Asserts
+  include Constants
+  include Fluent::AnalyzeConfigFilter::Constants
+
+  APPLICATION_DEFAULT_CONFIG = ''.freeze
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def test_config_file_does_not_exist
+    # By default, the FilterTestDriver.new does not set up a config file at:
+    # /etc/google-fluentd/google-fluentd.conf. The plugin should still proceed.
+    create_driver
+    # No exceptions were thrown.
+  end
+
+  def test_analyze_config
+    create_driver(CONFIG_ANALYZE_CONFIG)
+
+    # Default plugins, with default config.
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'source/syslog/tcp',
+      is_default_plugin: true,
+      has_default_value: true,
+      has_ruby_snippet: false)
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'source/tail/apache-access',
+      is_default_plugin: true,
+      has_default_value: true,
+      has_ruby_snippet: false)
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'filter/add_insert_ids',
+      is_default_plugin: true,
+      has_default_value: true,
+      has_ruby_snippet: false)
+
+    # Default plugins, with custom config.
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'match/google_cloud',
+      is_default_plugin: true,
+      has_default_value: false,
+      has_ruby_snippet: false)
+
+    # Custom plugins, some with embedded Ruby.
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'filter',
+      is_default_plugin: false,
+      has_default_value: false,
+      has_ruby_snippet: false)
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'filter/record_transformer',
+      is_default_plugin: false,
+      has_default_value: false,
+      has_ruby_snippet: true)
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'match/stdout',
+      is_default_plugin: false,
+      has_default_value: false,
+      has_ruby_snippet: true)
+
+    # For out_google_cloud, 3 params are present.
+    assert_prometheus_metric_value(
+      :stackdriver_config_usage,
+      1,
+      plugin_name: 'google_cloud',
+      param: 'adjust_invalid_timestamps',
+      is_present: true,
+      has_default_value: true)
+    assert_prometheus_metric_value(
+      :stackdriver_config_usage,
+      1,
+      plugin_name: 'google_cloud',
+      param: 'autoformat_stackdriver_trace',
+      is_present: true,
+      has_default_value: false)
+    assert_prometheus_metric_value(
+      :stackdriver_config_usage,
+      1,
+      plugin_name: 'google_cloud',
+      param: 'coerce_to_utf8',
+      is_present: true,
+      has_default_value: false)
+    # The remaining "google_cloud" params are not present.
+    # The are no params for "detect_exceptions".
+    %w(
+      auth_method
+      detect_json
+      enable_monitoring
+      gcm_service_address
+      grpc_compression_algorithm
+      http_request_key
+      insert_id_key
+      label_map
+      labels
+      labels_key
+      logging_api_url
+      monitoring_type
+      non_utf8_replacement_string
+      operation_key
+      private_key_email
+      private_key_passphrase
+      private_key_path
+      project_id
+      source_location_key
+      span_id_key
+      statusz_port
+      trace_key
+      trace_sampled_key
+      use_grpc
+      use_metadata_service
+      vm_id
+      vm_name
+      zone
+    ).each do |p|
+      assert_prometheus_metric_value(
+        :stackdriver_config_usage,
+        1,
+        plugin_name: 'google_cloud',
+        param: p,
+        is_present: false,
+        has_default_value: false)
+    end
+
+    # We also export values for the bools.
+    assert_prometheus_metric_value(
+      :stackdriver_config_bool_values,
+      1,
+      plugin_name: 'google_cloud',
+      param: 'adjust_invalid_timestamps',
+      value: true)
+    assert_prometheus_metric_value(
+      :stackdriver_config_bool_values,
+      1,
+      plugin_name: 'google_cloud',
+      param: 'autoformat_stackdriver_trace',
+      value: false)
+    assert_prometheus_metric_value(
+      :stackdriver_config_bool_values,
+      1,
+      plugin_name: 'google_cloud',
+      param: 'coerce_to_utf8',
+      value: true)
+  end
+
+  def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
+    Fluent::Test::FilterTestDriver.new(
+      Fluent::AnalyzeConfigFilter).configure(conf, true)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.7
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-16 00:00:00.000000000 Z
+date: 2020-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -266,21 +266,27 @@ files:
 - README.rdoc
 - Rakefile
 - fluent-plugin-google-cloud.gemspec
+- lib/fluent/plugin/common.rb
 - lib/fluent/plugin/filter_add_insert_ids.rb
+- lib/fluent/plugin/filter_analyze_config.rb
 - lib/fluent/plugin/in_object_space_dump.rb
 - lib/fluent/plugin/monitoring.rb
 - lib/fluent/plugin/out_google_cloud.rb
 - lib/fluent/plugin/statusz.rb
 - test/helper.rb
+- test/plugin/asserts.rb
 - test/plugin/base_test.rb
 - test/plugin/constants.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
 - test/plugin/data/credentials.json
+- test/plugin/data/google-fluentd-baseline.conf
+- test/plugin/data/google-fluentd-custom.conf
 - test/plugin/data/iam-credentials.json
 - test/plugin/data/invalid_credentials.json
 - test/plugin/data/new-style-credentials.json
 - test/plugin/test_driver.rb
 - test/plugin/test_filter_add_insert_ids.rb
+- test/plugin/test_filter_analyze_config.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb
 homepage: https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
@@ -302,20 +308,24 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: fluentd plugins for the Stackdriver Logging API
 test_files:
 - test/helper.rb
+- test/plugin/asserts.rb
 - test/plugin/base_test.rb
 - test/plugin/constants.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
 - test/plugin/data/credentials.json
+- test/plugin/data/google-fluentd-baseline.conf
+- test/plugin/data/google-fluentd-custom.conf
 - test/plugin/data/iam-credentials.json
 - test/plugin/data/invalid_credentials.json
 - test/plugin/data/new-style-credentials.json
 - test/plugin/test_driver.rb
 - test/plugin/test_filter_add_insert_ids.rb
+- test/plugin/test_filter_analyze_config.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb