fluent-plugin-google-cloud 0.10.2 → 0.10.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 202c2767566b8903d03b55b211a5e29c55d77c08b2d8ab1b07c3c4a6c16ea4a9
-  data.tar.gz: 9175afb12ae142d19ad72c80384b0e5ea22766524d599d1343f0a1c55a564a27
+  metadata.gz: 04bd17bad7dbead504f9f9e3e78fa36cb4ecf44409e0b6d817b16e8d80498631
+  data.tar.gz: 12d9f4b26e4876dbea235043e2fa1bb59707a5f66024c8becfa1f9ed2e037a5b
 SHA512:
-  metadata.gz: 1bc3b60cac80908283a91c80f4db157117579119bfc13c7d233de706256a3e458d8d6c8c3fa5aea99ef840123caabb6fe45ae270f795a66720ab0742792107a6
-  data.tar.gz: b987203a4b12ac2c1a0679362ae1fd6c1e332ae258a0fb8094d6a15f88d84cce9a15d8f76e3129cffa8a2f70f607a8b5f746f8840b67fd708b2ade66462d426d
+  metadata.gz: 94f7657e972697e89acda8d1a78663561a42e084170aed6cb03a2f606e1706ccce0c721495cd0270a005f1a2aa8bc0c59914fb66ccf9c03c4cd20fd71588b198
+  data.tar.gz: 2077ecebdfcf73f02594ab93e8f96bfd4c0dc61bf27b2c9f4768a875ed9419945375423309dadd41630610732f5734218cf8304411edaae79af5c4d6d3e34c03

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.10.2)
+    fluent-plugin-google-cloud (0.10.3)
       fluentd (= 1.11.2)
       google-api-client (= 0.30.8)
       google-cloud-logging (= 1.6.6)
@@ -20,15 +20,14 @@ GEM
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.1)
     concurrent-ruby (1.1.7)
-    cool.io (1.6.0)
+    cool.io (1.7.0)
     coveralls (0.8.23)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
       term-ansicolor (~> 1.3)
       thor (>= 0.19.4, < 2.0)
       tins (~> 1.6)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
+    crack (0.4.4)
     declarative (0.0.20)
     declarative-option (0.1.0)
     docile (1.3.2)
@@ -129,7 +128,7 @@ GEM
       google-cloud-trace (~> 0.35)
       opencensus (~> 0.5)
     os (1.1.1)
-    parser (2.7.1.4)
+    parser (2.7.1.5)
       ast (~> 2.4.1)
     power_assert (1.2.0)
     powerpack (0.1.2)
@@ -153,7 +152,6 @@ GEM
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.10.1)
-    safe_yaml (1.0.5)
     serverengine (2.2.1)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
@@ -169,7 +167,7 @@ GEM
     simplecov-html (0.10.2)
     stackdriver-core (1.4.0)
       google-cloud-core (~> 1.2)
-    strptime (0.2.4)
+    strptime (0.2.5)
     sync (0.5.0)
     term-ansicolor (1.7.1)
       tins (~> 1.0)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.10.2'
+  gem.version       = '0.10.3'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')

data/lib/fluent/plugin/filter_analyze_config.rb

@@ -15,6 +15,9 @@
 require 'fileutils'
 require 'fluent/config'
 require 'fluent/config/v1_parser'
+require 'googleauth'
+require 'google/apis/logging_v2'
+require 'open-uri'
 require 'set'
 
 require_relative 'common'
@@ -155,8 +158,8 @@ module Fluent
       super
       @log = $log # rubocop:disable Style/GlobalVars
 
-      # Initialize the insertID.
-      @log.info 'Started the analyze_config plugin to analyze configuration.'
+      @log.info(
+        'analyze_config plugin: Started the plugin to analyze configuration.')
     end
 
     def parse_config(path)
@@ -205,6 +208,7 @@ module Fluent
 
     def configure(conf)
       super
+      @log.info('analyze_config plugin: Starting to configure the plugin.')
       if File.file?(@google_fluentd_config_path) &&
          File.file?(@google_fluentd_baseline_config_path)
         @log.info(
@@ -232,24 +236,28 @@ module Fluent
 
         unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
           @monitoring_type)
-          @log.warn "monitoring_type '#{@monitoring_type}' is unknown; "\
-                    'there will be no metrics'
+          @log.warn(
+            "analyze_config plugin: monitoring_type #{@monitoring_type} is " \
+            'unknown; there will be no metrics.')
         end
         registry = Monitoring::MonitoringRegistryFactory.create(
           @monitoring_type, project_id, resource, @gcm_service_address)
 
         plugin_usage = registry.counter(
-          :stackdriver_enabled_plugins,
-          [:plugin_name, :is_default_plugin, :has_default_value],
-          'Enabled plugins')
+          :enabled_plugins,
+          [:plugin_name, :is_default_plugin, :has_default_config],
+          'Enabled plugins',
+          'agent.googleapis.com/agent/internal/logging/config')
         config_usage = registry.counter(
-          :stackdriver_config_usage,
-          [:plugin_name, :param, :is_present, :has_default_value],
-          'Parameter usage for Google Cloud plugins')
+          :plugin_config,
+          [:plugin_name, :param, :is_present, :has_default_config],
+          'Configuration parameter usage for plugins relevant to Google Cloud.',
+          'agent.googleapis.com/agent/internal/logging/config')
         config_bool_values = registry.counter(
-          :stackdriver_config_bool_values,
+          :config_bool_values,
           [:plugin_name, :param, :value],
-          'Values for bool parameters in Google Cloud plugins')
+          'Values for bool parameters in Google Cloud plugins',
+          'agent.googleapis.com/agent/internal/logging/config')
 
         config = parse_config(@google_fluentd_config_path)
         baseline_config = parse_config(@google_fluentd_baseline_config_path)
@@ -276,17 +284,17 @@ module Fluent
           plugin_name = default_plugin_name(e)
           if baseline_elements.key?(plugin_name)
             is_default_plugin = true
-            has_default_value = (baseline_elements[plugin_name] == e)
+            has_default_config = (baseline_elements[plugin_name] == e)
           else
             plugin_name = custom_plugin_name(e)
             is_default_plugin = false
-            has_default_value = false
+            has_default_config = false
           end
           plugin_usage.increment(
             labels: {
               plugin_name: plugin_name,
               is_default_plugin: is_default_plugin,
-              has_default_value: has_default_value,
+              has_default_config: has_default_config,
               has_ruby_snippet: embedded_ruby?(e)
             },
             by: 1)
@@ -300,7 +308,7 @@ module Fluent
                 plugin_name: e['@type'],
                 param: p,
                 is_present: e.key?(p),
-                has_default_value: (e.key?(p) &&
+                has_default_config: (e.key?(p) &&
                                     baseline_google_element.key?(p) &&
                                     e[p] == baseline_google_element[p])
               },
@@ -317,17 +325,18 @@ module Fluent
         end
       else
         @log.info(
-          'google-fluentd configuration file does not exist at' \
-          " #{@google_fluentd_config_path} or " \
-          'google-fluentd baseline configuration file does not exist at' \
-          " #{@google_fluentd_baseline_config_path} or " \
-          '. Skipping configuration analysis.')
+          'analyze_config plugin: google-fluentd configuration file does not ' \
+          "exist at #{@google_fluentd_config_path} or google-fluentd " \
+          'baseline configuration file does not exist at' \
+          " #{@google_fluentd_baseline_config_path}. Skipping configuration " \
+          'analysis.')
       end
     rescue => e
       # Do not crash the agent due to configuration analysis failures.
       @log.warn(
-        'Failed to optionally analyze the google-fluentd configuration' \
-        " file. Proceeding anyway. Error: #{e}")
+        'analyze_config plugin: Failed to optionally analyze the ' \
+        "google-fluentd configuration file. Proceeding anyway. Error: #{e}. " \
+        "Trace: #{e.backtrace}")
     end
 
     def shutdown

data/lib/fluent/plugin/monitoring.rb

@@ -52,7 +52,7 @@ module Monitoring
     def initialize(_project_id, _monitored_resource, _gcm_service_address)
     end
 
-    def counter(_name, _labels, _docstring)
+    def counter(_name, _labels, _docstring, _prefix)
       BaseCounter.new
     end
 
@@ -75,7 +75,7 @@ module Monitoring
     end
 
     # Exception-driven behavior to avoid synchronization errors.
-    def counter(name, _labels, docstring)
+    def counter(name, _labels, docstring, _prefix)
       # When we upgrade to Prometheus client 0.10.0 or higher, pass the
       # labels in the metric constructor. The 'labels' field in
       # Prometheus client 0.9.0 has a different function and will not
@@ -97,21 +97,14 @@ module Monitoring
       require 'opencensus'
       require 'opencensus-stackdriver'
       @log = $log # rubocop:disable Style/GlobalVars
-      @recorder = OpenCensus::Stats.ensure_recorder
-      @exporter = OpenCensus::Stats::Exporters::Stackdriver.new(
-        project_id: project_id,
-        metric_prefix: 'agent.googleapis.com/agent',
-        resource_type: monitored_resource.type,
-        resource_labels: monitored_resource.labels,
-        gcm_service_address: gcm_service_address
-      )
-      OpenCensus.configure do |c|
-        c.stats.exporter = @exporter
-      end
-      @log.debug "OpenCensus config=#{OpenCensus.config}"
+      @project_id = project_id
+      @metrics_monitored_resource = monitored_resource
+      @gcm_service_address = gcm_service_address
+      @recorders = {}
+      @exporters = {}
     end
 
-    def counter(name, labels, docstring)
+    def counter(name, labels, docstring, prefix)
       translator = MetricTranslator.new(name, labels)
       measure = OpenCensus::Stats::MeasureRegistry.get(translator.name)
       if measure.nil?
@@ -121,6 +114,17 @@ module Monitoring
           description: docstring
         )
       end
+      unless @exporters.keys.include?(prefix)
+        @recorders[prefix] = OpenCensus::Stats.ensure_recorder
+        @exporters[prefix] = \
+          OpenCensus::Stats::Exporters::Stackdriver.new(
+            project_id: @project_id,
+            metric_prefix: prefix,
+            resource_type: @metrics_monitored_resource.type,
+            resource_labels: @metrics_monitored_resource.labels,
+            gcm_service_address: @gcm_service_address
+          )
+      end
       OpenCensus::Stats.create_and_register_view(
         name: translator.name,
         measure: measure,
@@ -128,11 +132,19 @@ module Monitoring
         description: docstring,
         columns: translator.view_labels.map(&:to_s)
       )
-      OpenCensusCounter.new(@recorder, measure, translator)
+      OpenCensusCounter.new(@recorders[prefix], measure, translator)
+    rescue StandardError => e
+      @log.warn "Failed to count metrics for #{name}.", error: e
+      raise e
     end
 
     def export
-      @exporter.export @recorder.views_data
+      @exporters.keys.each do |prefix|
+        @exporters[prefix].export @recorders[prefix].views_data
+      end
+    rescue StandardError => e
+      @log.warn 'Failed to export some metrics.', error: e
+      raise e
     end
   end
 

data/lib/fluent/plugin/out_google_cloud.rb

@@ -563,32 +563,38 @@ module Fluent
         # Uptime should be a gauge, but the metric definition is a counter and
         # we can't change it.
         @uptime_metric = @registry.counter(
-          :uptime, [:version], 'Uptime of Logging agent')
+          :uptime, [:version], 'Uptime of Logging agent',
+          'agent.googleapis.com/agent')
         update_uptime
         timer_execute(:update_uptime, 1) { update_uptime }
         @successful_requests_count = @registry.counter(
           :stackdriver_successful_requests_count,
           [:grpc, :code],
-          'A number of successful requests to the Stackdriver Logging API')
+          'A number of successful requests to the Stackdriver Logging API',
+          'agent.googleapis.com/agent')
         @failed_requests_count = @registry.counter(
           :stackdriver_failed_requests_count,
           [:grpc, :code],
           'A number of failed requests to the Stackdriver Logging '\
-          'API, broken down by the error code')
+          'API, broken down by the error code',
+          'agent.googleapis.com/agent')
         @ingested_entries_count = @registry.counter(
           :stackdriver_ingested_entries_count,
           [:grpc, :code],
-          'A number of log entries ingested by Stackdriver Logging')
+          'A number of log entries ingested by Stackdriver Logging',
+          'agent.googleapis.com/agent')
         @dropped_entries_count = @registry.counter(
           :stackdriver_dropped_entries_count,
           [:grpc, :code],
-          'A number of log entries dropped by the Stackdriver output plugin')
+          'A number of log entries dropped by the Stackdriver output plugin',
+          'agent.googleapis.com/agent')
         @retried_entries_count = @registry.counter(
           :stackdriver_retried_entries_count,
           [:grpc, :code],
           'The number of log entries that failed to be ingested by '\
           'the Stackdriver output plugin due to a transient error '\
-          'and were retried')
+          'and were retried',
+          'agent.googleapis.com/agent')
         @ok_code = @use_grpc ? GRPC::Core::StatusCodes::OK : 200
       end
 

data/test/plugin/base_test.rb

@@ -169,10 +169,11 @@ module BaseTest
       assert_true d.instance.instance_variable_get(:@enable_monitoring)
       registry = d.instance.instance_variable_get(:@registry)
       assert_not_nil registry
-      exporter = registry.instance_variable_get(:@exporter)
-      assert_equal 'custom_resource', exporter.resource_type, "Index #{index}"
+      monitored_resource = registry.instance_variable_get(
+        :@metrics_monitored_resource)
+      assert_equal('custom_resource', monitored_resource.type, "Index #{index}")
       assert_equal({ 'label1' => '123', 'label2' => 'abc' },
-                   exporter.resource_labels, "Index #{index}")
+                   monitored_resource.labels, "Index #{index}")
     end
   end
 

data/test/plugin/test_filter_analyze_config.rb

@@ -55,81 +55,81 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
 
       # Default plugins, with default config.
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
         plugin_name: 'source/syslog/tcp',
         is_default_plugin: true,
-        has_default_value: true,
+        has_default_config: true,
         has_ruby_snippet: false)
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
         plugin_name: 'source/tail/apache-access',
         is_default_plugin: true,
-        has_default_value: true,
+        has_default_config: true,
         has_ruby_snippet: false)
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
         plugin_name: 'filter/add_insert_ids',
         is_default_plugin: true,
-        has_default_value: true,
+        has_default_config: true,
         has_ruby_snippet: false)
 
       # Default plugins, with custom config.
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
         plugin_name: 'match/google_cloud',
         is_default_plugin: true,
-        has_default_value: false,
+        has_default_config: false,
         has_ruby_snippet: false)
 
       # Custom plugins, some with embedded Ruby.
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
         plugin_name: 'filter',
         is_default_plugin: false,
-        has_default_value: false,
+        has_default_config: false,
         has_ruby_snippet: false)
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
         plugin_name: 'filter/record_transformer',
         is_default_plugin: false,
-        has_default_value: false,
+        has_default_config: false,
         has_ruby_snippet: true)
       assert_metric_value.call(
-        :stackdriver_enabled_plugins,
+        :enabled_plugins,
         1,
         plugin_name: 'match/stdout',
         is_default_plugin: false,
-        has_default_value: false,
+        has_default_config: false,
         has_ruby_snippet: true)
 
       # For out_google_cloud, 3 params are present.
       assert_metric_value.call(
-        :stackdriver_config_usage,
+        :plugin_config,
         1,
         plugin_name: 'google_cloud',
         param: 'adjust_invalid_timestamps',
         is_present: true,
-        has_default_value: true)
+        has_default_config: true)
       assert_metric_value.call(
-        :stackdriver_config_usage,
+        :plugin_config,
         1,
         plugin_name: 'google_cloud',
         param: 'autoformat_stackdriver_trace',
         is_present: true,
-        has_default_value: false)
+        has_default_config: false)
       assert_metric_value.call(
-        :stackdriver_config_usage,
+        :plugin_config,
         1,
         plugin_name: 'google_cloud',
         param: 'coerce_to_utf8',
         is_present: true,
-        has_default_value: false)
+        has_default_config: false)
       # The remaining "google_cloud" params are not present.
       # The are no params for "detect_exceptions".
       %w(
@@ -163,29 +163,29 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
         zone
       ).each do |p|
         assert_metric_value.call(
-          :stackdriver_config_usage,
+          :plugin_config,
           1,
           plugin_name: 'google_cloud',
           param: p,
           is_present: false,
-          has_default_value: false)
+          has_default_config: false)
       end
 
       # We also export values for the bools.
       assert_metric_value.call(
-        :stackdriver_config_bool_values,
+        :config_bool_values,
         1,
         plugin_name: 'google_cloud',
         param: 'adjust_invalid_timestamps',
         value: true)
       assert_metric_value.call(
-        :stackdriver_config_bool_values,
+        :config_bool_values,
         1,
         plugin_name: 'google_cloud',
         param: 'autoformat_stackdriver_trace',
         value: false)
       assert_metric_value.call(
-        :stackdriver_config_bool_values,
+        :config_bool_values,
         1,
         plugin_name: 'google_cloud',
         param: 'coerce_to_utf8',

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.10.3
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-10 00:00:00.000000000 Z
+date: 2020-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd