fluent-plugin-google-cloud 0.8.1 → 0.8.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a2701d1d315a64fec05aea3f289e2924c5c72aedec4467a76dfe4e29f429dcf
-  data.tar.gz: 7ab51bcbd2994ace82f0b0c61b318f31ca2ef57827b1ed80dd8e88980bebd420
+  metadata.gz: 44d38e36606e310f5ec6dae2931125f139b1e40c8c2b5eafd45c3497e6aeb1c9
+  data.tar.gz: b2aecc00518d8787a9d3a2fcd2fad880246cf63cfddc3b4967277eb07baa2e5b
 SHA512:
-  metadata.gz: 4d4ea1d83d1f3b23cc1299684802c3d1a15e5f178d6389aad3d49dc351a4efb7ce036027cb07756ce05b381733b46f2ba7924067b4f819e6b8e04556c3600585
-  data.tar.gz: 60599065092f8b1e19bfde366c2068e735e764b4aff64792f3dedd913f82c023a1895d9ae79edc88bca39d09dadacd26bb771fdf252de44dd9e9ce87d36620d9
+  metadata.gz: c4b3ead22e4a4ee3bcb20cbd5102f4847be31bf99606e1072b668a75176b452feb5cf84fd6d19297739ca76fba6c261fa33e562faabc07553587510554e8c867
+  data.tar.gz: c5659ca5819695d2799331bb22fe5f05fe39434b7f744e324ad8aa43eeb3b917f5e530868e9c11d2916b896ae2c1f2dee74880a11b0f278da1a0742db3fd6b82

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.8.1)
-      fluentd (= 1.6.3)
+    fluent-plugin-google-cloud (0.8.6)
+      fluentd (= 1.7.4)
       google-api-client (= 0.30.8)
       google-cloud-logging (= 1.6.6)
       google-protobuf (= 3.9.0)
@@ -35,15 +35,15 @@ GEM
     docile (1.3.2)
     faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
-    fluentd (1.6.3)
+    fluentd (1.7.4)
       cool.io (>= 1.4.5, < 2.0.0)
       dig_rb (~> 1.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 0.7.0, < 2.0.0)
+      msgpack (>= 1.2.0, < 2.0.0)
       serverengine (>= 2.0.4, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (~> 1.0)
+      tzinfo (~> 2.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
     google-api-client (0.30.8)
@@ -57,9 +57,9 @@ GEM
     google-cloud-core (1.5.0)
       google-cloud-env (~> 1.0)
       google-cloud-errors (~> 1.0)
-    google-cloud-env (1.3.1)
+    google-cloud-env (1.3.2)
       faraday (>= 0.17.3, < 2.0)
-    google-cloud-errors (1.0.0)
+    google-cloud-errors (1.0.1)
     google-cloud-logging (1.6.6)
       concurrent-ruby (~> 1.1)
       google-cloud-core (~> 1.2)
@@ -116,14 +116,14 @@ GEM
       google-cloud-monitoring (~> 0.32)
       google-cloud-trace (~> 0.35)
       opencensus (~> 0.5)
-    os (1.0.1)
-    parser (2.7.0.4)
+    os (1.1.0)
+    parser (2.7.1.3)
       ast (~> 2.4.0)
-    power_assert (1.1.6)
+    power_assert (1.2.0)
     powerpack (0.1.2)
     prometheus-client (0.9.0)
       quantile (~> 0.2.1)
-    public_suffix (4.0.3)
+    public_suffix (4.0.5)
     quantile (0.2.1)
     rainbow (2.2.2)
       rake
@@ -145,7 +145,7 @@ GEM
     serverengine (2.2.1)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    signet (0.13.0)
+    signet (0.14.0)
       addressable (~> 2.3)
       faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
@@ -157,22 +157,21 @@ GEM
     simplecov-html (0.10.2)
     stackdriver-core (1.4.0)
       google-cloud-core (~> 1.2)
-    strptime (0.2.3)
+    strptime (0.2.4)
     sync (0.5.0)
     term-ansicolor (1.7.1)
       tins (~> 1.0)
     test-unit (3.3.3)
       power_assert
     thor (1.0.1)
-    thread_safe (0.3.6)
-    tins (1.24.1)
+    tins (1.25.0)
       sync
-    tzinfo (1.2.6)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2019.3)
+    tzinfo (2.0.2)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2020.1)
       tzinfo (>= 1.0.0)
     uber (0.1.0)
-    unicode-display_width (1.6.1)
+    unicode-display_width (1.7.0)
     webmock (3.6.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
@@ -193,4 +192,4 @@ DEPENDENCIES
   webmock (= 3.6.2)
 
 BUNDLED WITH
-   2.1.1
+   2.1.4

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.8.1'
+  gem.version       = '0.8.6'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
@@ -19,7 +19,7 @@ eos
   gem.test_files    = gem.files.grep(/^(test)/)
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency 'fluentd', '1.6.3'
+  gem.add_runtime_dependency 'fluentd', '1.7.4'
   gem.add_runtime_dependency 'googleapis-common-protos', '1.3.9'
   gem.add_runtime_dependency 'googleauth', '0.9.0'
   gem.add_runtime_dependency 'google-api-client', '0.30.8'

data/lib/fluent/plugin/filter_analyze_config.rb

@@ -0,0 +1,306 @@
+# Copyright 2020 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fileutils'
+require 'fluent/config'
+require 'fluent/config/v1_parser'
+require 'set'
+
+require_relative 'monitoring'
+
+module Fluent
+  # Fluentd filter plugin to analyze configuration usage.
+  #
+  # For documentation on inspecting parsed configuration elements, see
+  # https://www.rubydoc.info/github/fluent/fluentd/Fluent/Config/Element
+  class AnalyzeConfigFilter < Filter
+    include Fluent::Config
+    Fluent::Plugin.register_filter('analyze_config', self)
+
+    module Constants
+      # Built-in plugins that are ok to reference in metrics.
+      KNOWN_PLUGINS = {
+        'filter' => Set[
+          'geoip',
+          'grep',
+          'parser',
+          'record_transformer',
+          'stdout',
+        ],
+        'match' => Set[
+          'copy',
+          'elasticsearch',
+          'exec',
+          'exec_filter',
+          'file',
+          'forward',
+          'http',
+          'kafka',
+          'mongo',
+          'mongo_replset',
+          'null',
+          'relabel',
+          'rewrite_tag_filter',
+          'roundrobin',
+          's3',
+          'secondary_file',
+          'stdout',
+          'webhdfs',
+        ],
+        'source' => Set[
+          'dummy',
+          'exec',
+          'forward',
+          'http',
+          'monitor_agent',
+          'syslog',
+          'tail',
+          'tcp',
+          'udp',
+          'unix',
+          'windows_eventlog',
+        ]
+      }.freeze
+
+      # For Google plugins, we collect metrics on the params listed here.
+      GOOGLE_PLUGIN_PARAMS = {
+        'google_cloud' => %w(
+          adjust_invalid_timestamps
+          auth_method
+          autoformat_stackdriver_trace
+          coerce_to_utf8
+          detect_json
+          enable_monitoring
+          gcm_service_address
+          grpc_compression_algorithm
+          http_request_key
+          insert_id_key
+          label_map
+          labels
+          labels_key
+          logging_api_url
+          monitoring_type
+          non_utf8_replacement_string
+          operation_key
+          private_key_email
+          private_key_passphrase
+          private_key_path
+          project_id
+          source_location_key
+          span_id_key
+          statusz_port
+          trace_key
+          trace_sampled_key
+          use_grpc
+          use_metadata_service
+          vm_id
+          vm_name
+          zone
+        ),
+        'detect_exceptions' => %w(
+          languages
+          max_bytes
+          max_lines
+          message
+          multiline_flush_interval
+          remove_tag_prefix
+          stream
+        )
+      }.freeze
+    end
+
+    include self::Constants
+
+    # The root configuration file of google-fluentd package.
+    # This only applies to Linux.
+    config_param :google_fluentd_config_path,
+                 :string,
+                 default: '/etc/google-fluentd/google-fluentd.conf'
+    # Baseline configuration for comparing with local
+    # customizations.
+    config_param :google_fluentd_baseline_config_path,
+                 :string,
+                 default: '/etc/google-fluentd/baseline/google-fluentd.conf'
+
+    def start
+      super
+      @log = $log # rubocop:disable Style/GlobalVars
+
+      # Initialize the insertID.
+      @log.info 'Started the analyze_config plugin to analyze configuration.'
+    end
+
+    def parse_config(path)
+      data = File.open(path, 'r', &:read)
+      fname = File.basename(path)
+      basepath = File.dirname(path)
+      eval_context = Kernel.binding
+      # Override instance_eval so that LiteralParser does not actually
+      # evaluate the embedded Ruby, but instead just returns the
+      # source string.  See
+      # https://github.com/fluent/fluentd/blob/master/lib/fluent/config/literal_parser.rb
+      def eval_context.instance_eval(code)
+        code
+      end
+      Fluent::Config::V1Parser.parse(data, fname, basepath, eval_context)
+    end
+
+    # Returns a name for identifying plugins we ship by default.
+    def default_plugin_name(e)
+      case e['@type']
+      when 'syslog'
+        "#{e.name}/syslog/#{e['protocol_type']}"
+      when 'tail'
+        "#{e.name}/tail/#{File.basename(e['pos_file'], '.pos')}"
+      else
+        "#{e.name}/#{e['@type']}"
+      end
+    end
+
+    # Returns a name for identifying plugins not in our default
+    # config.  This should not contain arbitrary user-supplied data.
+    def custom_plugin_name(e)
+      if KNOWN_PLUGINS.key?(e.name) &&
+         KNOWN_PLUGINS[e.name].include?(e['@type'])
+        "#{e.name}/#{e['@type']}"
+      else
+        e.name.to_s
+      end
+    end
+
+    def embedded_ruby?(e)
+      (e.arg.include?('#{') ||
+       e.any? { |_, v| v.include?('#{') } ||
+       e.elements.any? { |ee| embedded_ruby?(ee) })
+    end
+
+    def configure(conf)
+      super
+      if File.file?(@google_fluentd_config_path) &&
+         File.file?(@google_fluentd_baseline_config_path)
+        @log.info(
+          'google-fluentd configuration file found at' \
+          " #{@google_fluentd_config_path}. " \
+          'google-fluentd baseline configuration file found at' \
+          " #{@google_fluentd_baseline_config_path}. " \
+          'google-fluentd Analyzing configuration.')
+
+        # TODO: Add OpenCensus support.
+        registry = Monitoring::MonitoringRegistryFactory.create(
+          Monitoring::PrometheusMonitoringRegistry.name, nil, nil, nil)
+
+        plugin_usage = registry.counter(
+          :stackdriver_enabled_plugins,
+          [:plugin_name, :is_default_plugin, :has_default_value],
+          'Enabled plugins')
+        config_usage = registry.counter(
+          :stackdriver_config_usage,
+          [:plugin_name, :param, :is_present, :has_default_value],
+          'Parameter usage for Google Cloud plugins')
+        config_bool_values = registry.counter(
+          :stackdriver_config_bool_values,
+          [:plugin_name, :param, :value],
+          'Values for bool parameters in Google Cloud plugins')
+
+        config = parse_config(@google_fluentd_config_path)
+        baseline_config = parse_config(@google_fluentd_baseline_config_path)
+
+        # Create hash of all baseline elements by their plugin names.
+        baseline_elements = Hash[baseline_config.elements.collect do |e|
+                                   [default_plugin_name(e), e]
+                                 end]
+        baseline_google_element = baseline_config.elements.find do |e|
+          e['@type'] == 'google_cloud'
+        end
+
+        # Look at each top-level config element and see whether it
+        # matches the baseline value.
+        #
+        # Note on custom configurations: If the plugin has a custom
+        # value (e.g. if a tail plugin has pos_file
+        # /var/lib/google-fluentd/pos/my-custom-value.pos), then the
+        # default_plugin_name (e.g. source/tail/my-custom-value) won't
+        # be a key in baseline_elements below, so it won't be
+        # used.  Instead it will use the custom_plugin_name
+        # (e.g. source/tail).
+        config.elements.each do |e|
+          plugin_name = default_plugin_name(e)
+          if baseline_elements.key?(plugin_name)
+            is_default_plugin = true
+            has_default_value = (baseline_elements[plugin_name] == e)
+          else
+            plugin_name = custom_plugin_name(e)
+            is_default_plugin = false
+            has_default_value = false
+          end
+          plugin_usage.increment(
+            labels: {
+              plugin_name: plugin_name,
+              is_default_plugin: is_default_plugin,
+              has_default_value: has_default_value,
+              has_ruby_snippet: embedded_ruby?(e)
+            },
+            by: 1)
+
+          # Additional metric for Google plugins (google_cloud and
+          # detect_exceptions).
+          next unless GOOGLE_PLUGIN_PARAMS.key?(e['@type'])
+          GOOGLE_PLUGIN_PARAMS[e['@type']].each do |p|
+            config_usage.increment(
+              labels: {
+                plugin_name: e['@type'],
+                param: p,
+                is_present: e.key?(p),
+                has_default_value: (e.key?(p) &&
+                                    baseline_google_element.key?(p) &&
+                                    e[p] == baseline_google_element[p])
+              },
+              by: 1)
+            next unless e.key?(p) && %w(true false).include?(e[p])
+            config_bool_values.increment(
+              labels: {
+                plugin_name: e['@type'],
+                param: p,
+                value: e[p] == 'true'
+              },
+              by: 1)
+          end
+        end
+      else
+        @log.info(
+          'google-fluentd configuration file does not exist at' \
+          " #{@google_fluentd_config_path} or " \
+          'google-fluentd baseline configuration file does not exist at' \
+          " #{@google_fluentd_baseline_config_path} or " \
+          '. Skipping configuration analysis.')
+      end
+    rescue => e
+      # Do not crash the agent due to configuration analysis failures.
+      @log.warn(
+        'Failed to optionally analyze the google-fluentd configuration' \
+        " file. Proceeding anyway. Error: #{e}")
+    end
+
+    def shutdown
+      super
+    end
+
+    # rubocop:disable Lint/UnusedMethodArgument
+    def filter(tag, time, record)
+      # Skip the actual filtering process.
+      record
+    end
+    # rubocop:enable Lint/UnusedMethodArgument
+  end
+end

data/lib/fluent/plugin/out_google_cloud.rb

@@ -103,7 +103,7 @@ module Fluent
       }.freeze
       GKE_CONSTANTS = {
         service: 'container.googleapis.com',
-        resource_type: 'container',
+        resource_type: 'gke_container',
         extra_resource_labels: %w(namespace_id pod_id container_name),
         extra_common_labels: %w(namespace_name pod_name),
         metadata_attributes: %w(cluster-name cluster-location),
@@ -1575,7 +1575,7 @@ module Fluent
       # Recognizes IAM format (account@project-name.iam.gserviceaccount.com)
       # as well as the legacy format with a project number at the front of the
       # string, terminated by a dash (-) which is not part of the ID, i.e.:
-      # 270694816269-1l1r2hb813leuppurdeik0apglbs80sv.apps.googleusercontent.com
+      # <PROJECT_ID>-<OTHER_PARTS>.apps.googleusercontent.com
       def self.extract_project_id(str)
         [/^.*@(?<project_id>.+)\.iam\.gserviceaccount\.com/,
          /^(?<project_id>\d+)-/].each do |exp|
@@ -1909,7 +1909,11 @@ module Fluent
     end
 
     def format(tag, time, record)
-      Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s
+      Fluent::MessagePackFactory
+        .engine_factory
+        .packer
+        .write([tag, time, record])
+        .to_s
     end
 
     # Given a tag, returns the corresponding valid tag if possible, or nil if

data/test/plugin/asserts.rb

@@ -0,0 +1,75 @@
+# Copyright 2020 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Additional assert functions.
+module Asserts
+  # For an optional field with default values, Protobuf omits the field when it
+  # is deserialized to json. So we need to add an extra check for gRPC which
+  # uses Protobuf.
+  #
+  # An optional block can be passed in if we need to assert something other than
+  # a plain equal. e.g. assert_in_delta.
+  def assert_equal_with_default(_field, _expected_value, _default_value, _entry)
+    _undefined
+  end
+
+  # Compare the timestamp seconds and nanoseconds with the expected timestamp.
+  def assert_timestamp_matches(expected_ts, ts_secs, ts_nanos, entry)
+    assert_equal expected_ts.tv_sec, ts_secs, entry
+    # Fluentd v0.14 onwards supports nanosecond timestamp values.
+    # Added in 600 ns delta to avoid flaky tests introduced
+    # due to rounding error in double-precision floating-point numbers
+    # (to account for the missing 9 bits of precision ~ 512 ns).
+    # See http://wikipedia.org/wiki/Double-precision_floating-point_format.
+    assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
+  end
+
+  def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
+    metric = Prometheus::Client.registry.get(metric_name)
+    assert_not_nil(metric)
+    metric_value = if labels == :aggregate
+                     # Sum up all metric values regardless of the labels.
+                     metric.values.values.reduce(0.0, :+)
+                   else
+                     metric.get(labels)
+                   end
+    assert_equal(expected_value, metric_value)
+  end
+
+  def assert_opencensus_metric_value(metric_name, expected_value, labels = {})
+    translator = Monitoring::MetricTranslator.new(metric_name, labels)
+    metric_name = translator.name
+    labels = translator.translate_labels(labels)
+    # The next line collapses the labels to assert against the aggregated data,
+    # which can have some labels removed. Without this, it would test against
+    # the raw data. The view is more representative of the user experience, even
+    # though both tests should work because currently we only aggregate away one
+    # label that never changes during runtime.
+    labels.select! { |k, _| translator.view_labels.include? k }
+    labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
+    stats_recorder = OpenCensus::Stats.ensure_recorder
+    view_data = stats_recorder.view_data metric_name
+    assert_not_nil(view_data)
+    # For now assume all metrics are counters.
+    assert_kind_of(OpenCensus::Stats::Aggregation::Sum,
+                   view_data.view.aggregation)
+    assert_true(view_data.view.measure.int64?)
+    tag_values = view_data.view.columns.map { |column| labels[column] }
+    metric_value = 0
+    if view_data.data.key? tag_values
+      metric_value = view_data.data[tag_values].value
+    end
+    assert_equal(expected_value, metric_value)
+  end
+end

data/test/plugin/base_test.rb

@@ -22,6 +22,7 @@ require 'mocha/test_unit'
 require 'webmock/test_unit'
 require 'prometheus/client'
 
+require_relative 'asserts'
 require_relative 'constants'
 
 module Monitoring
@@ -35,6 +36,7 @@ end
 
 # Unit tests for Google Cloud Logging plugin
 module BaseTest
+  include Asserts
   include Constants
 
   def setup
@@ -2534,65 +2536,6 @@ module BaseTest
     _undefined
   end
 
-  # For an optional field with default values, Protobuf omits the field when it
-  # is deserialized to json. So we need to add an extra check for gRPC which
-  # uses Protobuf.
-  #
-  # An optional block can be passed in if we need to assert something other than
-  # a plain equal. e.g. assert_in_delta.
-  def assert_equal_with_default(_field, _expected_value, _default_value, _entry)
-    _undefined
-  end
-
-  # Compare the timestamp seconds and nanoseconds with the expected timestamp.
-  def assert_timestamp_matches(expected_ts, ts_secs, ts_nanos, entry)
-    assert_equal expected_ts.tv_sec, ts_secs, entry
-    # Fluentd v0.14 onwards supports nanosecond timestamp values.
-    # Added in 600 ns delta to avoid flaky tests introduced
-    # due to rounding error in double-precision floating-point numbers
-    # (to account for the missing 9 bits of precision ~ 512 ns).
-    # See http://wikipedia.org/wiki/Double-precision_floating-point_format.
-    assert_in_delta expected_ts.tv_nsec, ts_nanos, 600, entry
-  end
-
-  def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
-    metric = Prometheus::Client.registry.get(metric_name)
-    assert_not_nil(metric)
-    metric_value = if labels == :aggregate
-                     # Sum up all metric values regardless of the labels.
-                     metric.values.values.reduce(0.0, :+)
-                   else
-                     metric.get(labels)
-                   end
-    assert_equal(expected_value, metric_value)
-  end
-
-  def assert_opencensus_metric_value(metric_name, expected_value, labels = {})
-    translator = Monitoring::MetricTranslator.new(metric_name, labels)
-    metric_name = translator.name
-    labels = translator.translate_labels(labels)
-    # The next line collapses the labels to assert against the aggregated data,
-    # which can have some labels removed. Without this, it would test against
-    # the raw data. The view is more representative of the user experience, even
-    # though both tests should work because currently we only aggregate away one
-    # label that never changes during runtime.
-    labels.select! { |k, _| translator.view_labels.include? k }
-    labels = labels.map { |k, v| [k.to_s, v.to_s] }.to_h
-    stats_recorder = OpenCensus::Stats.ensure_recorder
-    view_data = stats_recorder.view_data metric_name
-    assert_not_nil(view_data)
-    # For now assume all metrics are counters.
-    assert_kind_of(OpenCensus::Stats::Aggregation::Sum,
-                   view_data.view.aggregation)
-    assert_true(view_data.view.measure.int64?)
-    tag_values = view_data.view.columns.map { |column| labels[column] }
-    metric_value = 0
-    if view_data.data.key? tag_values
-      metric_value = view_data.data[tag_values].value
-    end
-    assert_equal(expected_value, metric_value)
-  end
-
   # Defined in specific gRPC or REST files.
   def expected_operation_message2
     _undefined

data/test/plugin/constants.rb

@@ -471,6 +471,14 @@ module Constants
     zone                          asia-east2
   ).freeze
 
+  # For analyze_config.
+  CONFIG_ANALYZE_CONFIG = %(
+    google_fluentd_config_path          \
+      test/plugin/data/google-fluentd-custom.conf
+    google_fluentd_baseline_config_path \
+      test/plugin/data/google-fluentd-baseline.conf
+  ).freeze
+
   # Service configurations for various services.
 
   # GCE.

data/test/plugin/data/google-fluentd-baseline.conf

@@ -0,0 +1,24 @@
+<source>
+  @type syslog
+  port 514
+  protocol_type tcp
+  bind 127.0.0.1
+  tag syslog
+</source>
+
+<source>
+  @type tail
+  path /var/log/apache*/access.log,/var/log/apache*/access_log,/var/log/httpd/access.log,/var/log/httpd/access_log
+  pos_file /var/lib/google-fluentd/pos/apache-access.pos
+  tag apache-access
+</source>
+
+<filter **>
+  @type add_insert_ids
+</filter>
+
+<match **>
+  @type google_cloud
+  adjust_invalid_timestamps    true
+  autoformat_stackdriver_trace true
+</match>

data/test/plugin/data/google-fluentd-custom.conf

@@ -0,0 +1,40 @@
+<source>
+  @type syslog
+  port 514
+  protocol_type tcp
+  bind 127.0.0.1
+  tag syslog
+</source>
+
+<source>
+  @type tail
+  path /var/log/apache*/access.log,/var/log/apache*/access_log,/var/log/httpd/access.log,/var/log/httpd/access_log
+  pos_file /var/lib/google-fluentd/pos/apache-access.pos
+  tag apache-access
+</source>
+
+<filter **>
+  @type add_insert_ids
+</filter>
+
+<match **>
+  @type google_cloud
+  adjust_invalid_timestamps    true
+  autoformat_stackdriver_trace false
+  coerce_to_utf8               true
+</match>
+
+<filter **>
+  @type some_custom_filter
+</filter>
+
+<filter **>
+  @type record_transformer
+  <record>
+    host_param "#{Socket.gethostname}"
+  </record>
+</filter>
+
+<match "app.#{ENV['FLUENTD_TAG']}">
+  @type stdout
+</match>

data/test/plugin/test_filter_analyze_config.rb

@@ -0,0 +1,187 @@
+# Copyright 2020 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative '../helper'
+require_relative 'asserts'
+require_relative 'constants'
+
+require 'fluent/test/driver/filter'
+require 'fluent/plugin/filter_analyze_config'
+
+# Unit tests for filter_analyze_config plugin.
+class FilterAnalyzeConfigTest < Test::Unit::TestCase
+  include Asserts
+  include Constants
+  include Fluent::AnalyzeConfigFilter::Constants
+
+  APPLICATION_DEFAULT_CONFIG = ''.freeze
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def test_config_file_does_not_exist
+    # By default, the FilterTestDriver.new does not set up a config file at:
+    # /etc/google-fluentd/google-fluentd.conf. The plugin should still proceed.
+    create_driver
+    # No exceptions were thrown.
+  end
+
+  def test_analyze_config
+    create_driver(CONFIG_ANALYZE_CONFIG)
+
+    # Default plugins, with default config.
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'source/syslog/tcp',
+      is_default_plugin: true,
+      has_default_value: true,
+      has_ruby_snippet: false)
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'source/tail/apache-access',
+      is_default_plugin: true,
+      has_default_value: true,
+      has_ruby_snippet: false)
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'filter/add_insert_ids',
+      is_default_plugin: true,
+      has_default_value: true,
+      has_ruby_snippet: false)
+
+    # Default plugins, with custom config.
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'match/google_cloud',
+      is_default_plugin: true,
+      has_default_value: false,
+      has_ruby_snippet: false)
+
+    # Custom plugins, some with embedded Ruby.
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'filter',
+      is_default_plugin: false,
+      has_default_value: false,
+      has_ruby_snippet: false)
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'filter/record_transformer',
+      is_default_plugin: false,
+      has_default_value: false,
+      has_ruby_snippet: true)
+    assert_prometheus_metric_value(
+      :stackdriver_enabled_plugins,
+      1,
+      plugin_name: 'match/stdout',
+      is_default_plugin: false,
+      has_default_value: false,
+      has_ruby_snippet: true)
+
+    # For out_google_cloud, 3 params are present.
+    assert_prometheus_metric_value(
+      :stackdriver_config_usage,
+      1,
+      plugin_name: 'google_cloud',
+      param: 'adjust_invalid_timestamps',
+      is_present: true,
+      has_default_value: true)
+    assert_prometheus_metric_value(
+      :stackdriver_config_usage,
+      1,
+      plugin_name: 'google_cloud',
+      param: 'autoformat_stackdriver_trace',
+      is_present: true,
+      has_default_value: false)
+    assert_prometheus_metric_value(
+      :stackdriver_config_usage,
+      1,
+      plugin_name: 'google_cloud',
+      param: 'coerce_to_utf8',
+      is_present: true,
+      has_default_value: false)
+    # The remaining "google_cloud" params are not present.
+    # The are no params for "detect_exceptions".
+    %w(
+      auth_method
+      detect_json
+      enable_monitoring
+      gcm_service_address
+      grpc_compression_algorithm
+      http_request_key
+      insert_id_key
+      label_map
+      labels
+      labels_key
+      logging_api_url
+      monitoring_type
+      non_utf8_replacement_string
+      operation_key
+      private_key_email
+      private_key_passphrase
+      private_key_path
+      project_id
+      source_location_key
+      span_id_key
+      statusz_port
+      trace_key
+      trace_sampled_key
+      use_grpc
+      use_metadata_service
+      vm_id
+      vm_name
+      zone
+    ).each do |p|
+      assert_prometheus_metric_value(
+        :stackdriver_config_usage,
+        1,
+        plugin_name: 'google_cloud',
+        param: p,
+        is_present: false,
+        has_default_value: false)
+    end
+
+    # We also export values for the bools.
+    assert_prometheus_metric_value(
+      :stackdriver_config_bool_values,
+      1,
+      plugin_name: 'google_cloud',
+      param: 'adjust_invalid_timestamps',
+      value: true)
+    assert_prometheus_metric_value(
+      :stackdriver_config_bool_values,
+      1,
+      plugin_name: 'google_cloud',
+      param: 'autoformat_stackdriver_trace',
+      value: false)
+    assert_prometheus_metric_value(
+      :stackdriver_config_bool_values,
+      1,
+      plugin_name: 'google_cloud',
+      param: 'coerce_to_utf8',
+      value: true)
+  end
+
+  def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
+    Fluent::Test::FilterTestDriver.new(
+      Fluent::AnalyzeConfigFilter).configure(conf, true)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.8.6
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-06 00:00:00.000000000 Z
+date: 2020-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.3
+        version: 1.7.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.3
+        version: 1.7.4
 - !ruby/object:Gem::Dependency
   name: googleapis-common-protos
   requirement: !ruby/object:Gem::Requirement
@@ -267,20 +267,25 @@ files:
 - Rakefile
 - fluent-plugin-google-cloud.gemspec
 - lib/fluent/plugin/filter_add_insert_ids.rb
+- lib/fluent/plugin/filter_analyze_config.rb
 - lib/fluent/plugin/in_object_space_dump.rb
 - lib/fluent/plugin/monitoring.rb
 - lib/fluent/plugin/out_google_cloud.rb
 - lib/fluent/plugin/statusz.rb
 - test/helper.rb
+- test/plugin/asserts.rb
 - test/plugin/base_test.rb
 - test/plugin/constants.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
 - test/plugin/data/credentials.json
+- test/plugin/data/google-fluentd-baseline.conf
+- test/plugin/data/google-fluentd-custom.conf
 - test/plugin/data/iam-credentials.json
 - test/plugin/data/invalid_credentials.json
 - test/plugin/data/new-style-credentials.json
 - test/plugin/test_driver.rb
 - test/plugin/test_filter_add_insert_ids.rb
+- test/plugin/test_filter_analyze_config.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb
 homepage: https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
@@ -302,20 +307,24 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: fluentd plugins for the Stackdriver Logging API
 test_files:
 - test/helper.rb
+- test/plugin/asserts.rb
 - test/plugin/base_test.rb
 - test/plugin/constants.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
 - test/plugin/data/credentials.json
+- test/plugin/data/google-fluentd-baseline.conf
+- test/plugin/data/google-fluentd-custom.conf
 - test/plugin/data/iam-credentials.json
 - test/plugin/data/invalid_credentials.json
 - test/plugin/data/new-style-credentials.json
 - test/plugin/test_driver.rb
 - test/plugin/test_filter_add_insert_ids.rb
+- test/plugin/test_filter_analyze_config.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb