fluent-plugin-google-cloud 0.10.1 → 0.10.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 343edd2c59a22d1f35757f12629e19b7e01314dd55bf78bbbfbe6914530dcf27
-  data.tar.gz: d811dfea9e90007dd3dee571df75428a50e2d7b9ce9ab61d6581c21c4d9d5423
+  metadata.gz: d962a86da4d815fd57a1c6bb8f6462995d2b5bce96de8cf69869d59e08d44f92
+  data.tar.gz: f092dca63d15d65ff0983cd719b544093a2865bfd67f7d88eb1f80fb9f1c15c4
 SHA512:
-  metadata.gz: 8a8b30d92c16eeaea8a6608dccb8e97d72ca1b7e13126e5523582f8be56bd150a2b26bda4e56c247cbf6c3680f307bb71eb248588f6a7c2d87cf30379096f75d
-  data.tar.gz: ced9235516d07cba8af6a045481c969bf8a2abe9b67b891b14904cc5e150ebb82f888d0c7f2d2c3ea55686c1cc3bff53ff29bace8d455f6abf84a6264366e87d
+  metadata.gz: cdfdc357fea1964db223fd2ee5df419c6c2bb8e17171915368b94a5518a35b8904c986f80175c328bf57080e019a370fd882f575bca5731538eeede2f2d25f1a
+  data.tar.gz: ae6ef51d1fc54c8e8c65b351ce15409ccab8d6285123640d54cfea7840e1bfb84e0209b9cd2dd6b41dde752203b4ff29b9541fcf87036bf2bf715aa371d0e75f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.10.1)
+    fluent-plugin-google-cloud (0.10.6)
       fluentd (= 1.11.2)
       google-api-client (= 0.30.8)
       google-cloud-logging (= 1.6.6)
@@ -18,20 +18,20 @@ GEM
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    ast (2.4.1)
-    concurrent-ruby (1.1.7)
-    cool.io (1.6.0)
+    ast (2.4.2)
+    concurrent-ruby (1.1.8)
+    cool.io (1.7.0)
     coveralls (0.8.23)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
       term-ansicolor (~> 1.3)
       thor (>= 0.19.4, < 2.0)
       tins (~> 1.6)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
+    crack (0.4.5)
+      rexml
     declarative (0.0.20)
     declarative-option (0.1.0)
-    docile (1.3.2)
+    docile (1.3.5)
     faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
     fluentd (1.11.2)
@@ -61,7 +61,7 @@ GEM
     google-cloud-core (1.5.0)
       google-cloud-env (~> 1.0)
       google-cloud-errors (~> 1.0)
-    google-cloud-env (1.3.3)
+    google-cloud-env (1.4.0)
       faraday (>= 0.17.3, < 2.0)
     google-cloud-errors (1.0.1)
     google-cloud-logging (1.6.6)
@@ -80,10 +80,10 @@ GEM
       google-cloud-trace-v1 (~> 0.0)
       google-cloud-trace-v2 (~> 0.0)
       stackdriver-core (~> 1.3)
-    google-cloud-trace-v1 (0.1.2)
+    google-cloud-trace-v1 (0.2.0)
       gapic-common (~> 0.3)
       google-cloud-errors (~> 1.0)
-    google-cloud-trace-v2 (0.1.2)
+    google-cloud-trace-v2 (0.2.0)
       gapic-common (~> 0.3)
       google-cloud-errors (~> 1.0)
     google-gax (1.8.1)
@@ -119,7 +119,7 @@ GEM
     mini_mime (1.0.2)
     mocha (1.9.0)
       metaclass (~> 0.0.1)
-    msgpack (1.3.3)
+    msgpack (1.4.2)
     multi_json (1.15.0)
     multipart-post (2.1.1)
     opencensus (0.5.0)
@@ -129,13 +129,13 @@ GEM
       google-cloud-trace (~> 0.35)
       opencensus (~> 0.5)
     os (1.1.1)
-    parser (2.7.1.4)
+    parser (2.7.2.0)
       ast (~> 2.4.1)
-    power_assert (1.2.0)
-    powerpack (0.1.2)
+    power_assert (2.0.0)
+    powerpack (0.1.3)
     prometheus-client (0.9.0)
       quantile (~> 0.2.1)
-    public_suffix (4.0.5)
+    public_suffix (4.0.6)
     quantile (0.2.1)
     rainbow (2.2.2)
       rake
@@ -145,6 +145,7 @@ GEM
       declarative-option (< 0.2.0)
       uber (< 0.2.0)
     retriable (3.1.2)
+    rexml (3.2.4)
     rly (0.2.3)
     rubocop (0.39.0)
       parser (>= 2.3.0.7, < 3.0)
@@ -152,12 +153,11 @@ GEM
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.10.1)
-    safe_yaml (1.0.5)
-    serverengine (2.2.1)
+    ruby-progressbar (1.11.0)
+    serverengine (2.2.2)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    signet (0.14.0)
+    signet (0.14.1)
       addressable (~> 2.3)
       faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
@@ -169,18 +169,18 @@ GEM
     simplecov-html (0.10.2)
     stackdriver-core (1.4.0)
       google-cloud-core (~> 1.2)
-    strptime (0.2.4)
+    strptime (0.2.5)
     sync (0.5.0)
     term-ansicolor (1.7.1)
       tins (~> 1.0)
     test-unit (3.3.3)
       power_assert
-    thor (1.0.1)
-    tins (1.25.0)
+    thor (1.1.0)
+    tins (1.28.0)
       sync
-    tzinfo (2.0.2)
+    tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2020.1)
+    tzinfo-data (1.2021.1)
       tzinfo (>= 1.0.0)
     uber (0.1.0)
     unicode-display_width (1.7.0)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.10.1'
+  gem.version       = '0.10.6'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')

data/lib/fluent/plugin/filter_analyze_config.rb

@@ -15,6 +15,10 @@
 require 'fileutils'
 require 'fluent/config'
 require 'fluent/config/v1_parser'
+require 'fluent/plugin_helper'
+require 'googleauth'
+require 'google/apis/logging_v2'
+require 'open-uri'
 require 'set'
 
 require_relative 'common'
@@ -29,6 +33,10 @@ module Fluent
     include Fluent::Config
     Fluent::Plugin.register_filter('analyze_config', self)
 
+    # Required for the timer_execute method below
+    include PluginHelper::Mixin
+    helpers :timer
+
     module Constants
       # Built-in plugins that are ok to reference in metrics.
       KNOWN_PLUGINS = {
@@ -155,8 +163,8 @@ module Fluent
       super
       @log = $log # rubocop:disable Style/GlobalVars
 
-      # Initialize the insertID.
-      @log.info 'Started the analyze_config plugin to analyze configuration.'
+      @log.info(
+        'analyze_config plugin: Started the plugin to analyze configuration.')
     end
 
     def parse_config(path)
@@ -205,10 +213,11 @@ module Fluent
 
     def configure(conf)
       super
+      @log.info('analyze_config plugin: Starting to configure the plugin.')
       if File.file?(@google_fluentd_config_path) &&
          File.file?(@google_fluentd_baseline_config_path)
         @log.info(
-          'google-fluentd configuration file found at' \
+          'analyze_config plugin: google-fluentd configuration file found at' \
           " #{@google_fluentd_config_path}. " \
           'google-fluentd baseline configuration file found at' \
           " #{@google_fluentd_baseline_config_path}. " \
@@ -232,27 +241,53 @@ module Fluent
 
         unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
           @monitoring_type)
-          @log.warn "monitoring_type '#{@monitoring_type}' is unknown; "\
-                    'there will be no metrics'
+          @log.warn(
+            "analyze_config plugin: monitoring_type #{@monitoring_type} is " \
+            'unknown; there will be no metrics.')
         end
-        registry = Monitoring::MonitoringRegistryFactory.create(
+
+        @registry = Monitoring::MonitoringRegistryFactory.create(
           @monitoring_type, project_id, resource, @gcm_service_address)
+        # Export metrics every 60 seconds.
+        timer_execute(:export_config_analysis_metrics, 60) { @registry.export }
 
-        plugin_usage = registry.counter(
-          :stackdriver_enabled_plugins,
-          [:plugin_name, :is_default_plugin, :has_default_value],
-          'Enabled plugins')
-        config_usage = registry.counter(
-          :stackdriver_config_usage,
-          [:plugin_name, :param, :is_present, :has_default_value],
-          'Parameter usage for Google Cloud plugins')
-        config_bool_values = registry.counter(
-          :stackdriver_config_bool_values,
+        @log.info('analyze_config plugin: Registering counters.')
+        enabled_plugins_counter = @registry.counter(
+          :enabled_plugins,
+          [:plugin_name, :is_default_plugin,
+           :has_default_config, :has_ruby_snippet],
+          'Enabled plugins',
+          'agent.googleapis.com/agent/internal/logging/config',
+          'GAUGE')
+        @log.info(
+          'analyze_config plugin: registered enable_plugins counter. ' \
+          "#{enabled_plugins_counter}")
+        plugin_config_counter = @registry.counter(
+          :plugin_config,
+          [:plugin_name, :param, :is_present, :has_default_config],
+          'Configuration parameter usage for plugins relevant to Google Cloud.',
+          'agent.googleapis.com/agent/internal/logging/config',
+          'GAUGE')
+        @log.info('analyze_config plugin: registered plugin_config counter. ' \
+          "#{plugin_config_counter}")
+        config_bool_values_counter = @registry.counter(
+          :config_bool_values,
           [:plugin_name, :param, :value],
-          'Values for bool parameters in Google Cloud plugins')
+          'Values for bool parameters in Google Cloud plugins',
+          'agent.googleapis.com/agent/internal/logging/config',
+          'GAUGE')
+        @log.info('analyze_config plugin: registered config_bool_values ' \
+          "counter. #{config_bool_values_counter}")
 
         config = parse_config(@google_fluentd_config_path)
+        @log.debug(
+          'analyze_config plugin: successfully parsed google-fluentd' \
+          " configuration file at #{@google_fluentd_config_path}. #{config}")
         baseline_config = parse_config(@google_fluentd_baseline_config_path)
+        @log.debug(
+          'analyze_config plugin: successfully parsed google-fluentd' \
+          ' baseline configuration file at' \
+          " #{@google_fluentd_baseline_config_path}: #{baseline_config}")
 
         # Create hash of all baseline elements by their plugin names.
         baseline_elements = Hash[baseline_config.elements.collect do |e|
@@ -276,17 +311,17 @@ module Fluent
           plugin_name = default_plugin_name(e)
           if baseline_elements.key?(plugin_name)
             is_default_plugin = true
-            has_default_value = (baseline_elements[plugin_name] == e)
+            has_default_config = (baseline_elements[plugin_name] == e)
           else
             plugin_name = custom_plugin_name(e)
             is_default_plugin = false
-            has_default_value = false
+            has_default_config = false
           end
-          plugin_usage.increment(
+          enabled_plugins_counter.increment(
             labels: {
               plugin_name: plugin_name,
               is_default_plugin: is_default_plugin,
-              has_default_value: has_default_value,
+              has_default_config: has_default_config,
               has_ruby_snippet: embedded_ruby?(e)
             },
             by: 1)
@@ -295,18 +330,18 @@ module Fluent
           # detect_exceptions).
           next unless GOOGLE_PLUGIN_PARAMS.key?(e['@type'])
           GOOGLE_PLUGIN_PARAMS[e['@type']].each do |p|
-            config_usage.increment(
+            plugin_config_counter.increment(
               labels: {
                 plugin_name: e['@type'],
                 param: p,
                 is_present: e.key?(p),
-                has_default_value: (e.key?(p) &&
+                has_default_config: (e.key?(p) &&
                                     baseline_google_element.key?(p) &&
                                     e[p] == baseline_google_element[p])
               },
               by: 1)
             next unless e.key?(p) && %w(true false).include?(e[p])
-            config_bool_values.increment(
+            config_bool_values_counter.increment(
               labels: {
                 plugin_name: e['@type'],
                 param: p,
@@ -315,23 +350,29 @@ module Fluent
               by: 1)
           end
         end
+        @log.info(
+          'analyze_config plugin: Successfully finished analyzing config.')
       else
         @log.info(
-          'google-fluentd configuration file does not exist at' \
-          " #{@google_fluentd_config_path} or " \
-          'google-fluentd baseline configuration file does not exist at' \
-          " #{@google_fluentd_baseline_config_path} or " \
-          '. Skipping configuration analysis.')
+          'analyze_config plugin: google-fluentd configuration file does not ' \
+          "exist at #{@google_fluentd_config_path} or google-fluentd " \
+          'baseline configuration file does not exist at' \
+          " #{@google_fluentd_baseline_config_path}. Skipping configuration " \
+          'analysis.')
       end
     rescue => e
       # Do not crash the agent due to configuration analysis failures.
       @log.warn(
-        'Failed to optionally analyze the google-fluentd configuration' \
-        " file. Proceeding anyway. Error: #{e}")
+        'analyze_config plugin: Failed to optionally analyze the ' \
+        "google-fluentd configuration file. Proceeding anyway. Error: #{e}. " \
+        "Trace: #{e.backtrace}")
     end
 
     def shutdown
       super
+      # Export metrics on shutdown. This is a best-effort attempt, and it might
+      # fail, for instance if there was a recent write to the same time series.
+      @registry.export unless @registry.nil?
     end
 
     # rubocop:disable Lint/UnusedMethodArgument

data/lib/fluent/plugin/monitoring.rb

@@ -52,7 +52,7 @@ module Monitoring
     def initialize(_project_id, _monitored_resource, _gcm_service_address)
     end
 
-    def counter(_name, _labels, _docstring)
+    def counter(_name, _labels, _docstring, _prefix, _aggregation)
       BaseCounter.new
     end
 
@@ -75,7 +75,7 @@ module Monitoring
     end
 
     # Exception-driven behavior to avoid synchronization errors.
-    def counter(name, _labels, docstring)
+    def counter(name, _labels, docstring, _prefix, _aggregation)
       # When we upgrade to Prometheus client 0.10.0 or higher, pass the
       # labels in the metric constructor. The 'labels' field in
       # Prometheus client 0.9.0 has a different function and will not
@@ -97,42 +97,85 @@ module Monitoring
       require 'opencensus'
       require 'opencensus-stackdriver'
       @log = $log # rubocop:disable Style/GlobalVars
-      @recorder = OpenCensus::Stats.ensure_recorder
-      @exporter = OpenCensus::Stats::Exporters::Stackdriver.new(
-        project_id: project_id,
-        metric_prefix: 'agent.googleapis.com/agent',
-        resource_type: monitored_resource.type,
-        resource_labels: monitored_resource.labels,
-        gcm_service_address: gcm_service_address
-      )
-      OpenCensus.configure do |c|
-        c.stats.exporter = @exporter
-      end
-      @log.debug "OpenCensus config=#{OpenCensus.config}"
+      @project_id = project_id
+      @metrics_monitored_resource = monitored_resource
+      @gcm_service_address = gcm_service_address
+      @recorders = {}
+      @exporters = {}
+      @log.info(
+        'monitoring module: Successfully initialized Open Census monitoring ' \
+        'registry.')
     end
 
-    def counter(name, labels, docstring)
+    def counter(name, labels, docstring, prefix, aggregation)
       translator = MetricTranslator.new(name, labels)
       measure = OpenCensus::Stats::MeasureRegistry.get(translator.name)
       if measure.nil?
+        @log.info(
+          'monitoring module: Registering a new measure registry for ' \
+          "#{translator.name}")
         measure = OpenCensus::Stats.create_measure_int(
           name: translator.name,
           unit: OpenCensus::Stats::Measure::UNIT_NONE,
           description: docstring
         )
       end
-      OpenCensus::Stats.create_and_register_view(
-        name: translator.name,
-        measure: measure,
-        aggregation: OpenCensus::Stats.create_sum_aggregation,
-        description: docstring,
-        columns: translator.view_labels.map(&:to_s)
+      unless @exporters.keys.include?(prefix)
+        @log.info(
+          'monitoring module: Registering a new exporter for ' \
+          "#{prefix}")
+        @recorders[prefix] = OpenCensus::Stats::Recorder.new
+        @exporters[prefix] = \
+          OpenCensus::Stats::Exporters::Stackdriver.new(
+            project_id: @project_id,
+            metric_prefix: prefix,
+            resource_type: @metrics_monitored_resource.type,
+            resource_labels: @metrics_monitored_resource.labels,
+            gcm_service_address: @gcm_service_address
+          )
+        @log.info(
+          'monitoring module: Registered recorders and exporters for ' \
+          "#{prefix}.\n#{@exporters[prefix]}")
+      end
+      if aggregation == 'GAUGE'
+        stats_aggregation = OpenCensus::Stats.create_last_value_aggregation
+      else
+        stats_aggregation = OpenCensus::Stats.create_sum_aggregation
+      end
+      @recorders[prefix].register_view(
+        OpenCensus::Stats::View.new(
+          name: translator.name,
+          measure: measure,
+          aggregation: stats_aggregation,
+          description: docstring,
+          columns: translator.view_labels.map(&:to_s)
+        )
       )
-      OpenCensusCounter.new(@recorder, measure, translator)
+      counter = OpenCensusCounter.new(@recorders[prefix], measure, translator)
+      @log.info(
+        'monitoring module: Successfully initialized Open Census counter for ' \
+        "#{prefix}/#{name}.")
+      counter
+    rescue StandardError => e
+      @log.warn "Failed to count metrics for #{name}.", error: e
+      raise e
     end
 
     def export
-      @exporter.export @recorder.views_data
+      @log.debug(
+        "monitoring module: Exporting metrics for #{@exporters.keys}.")
+      @exporters.keys.each do |prefix|
+        @log.debug(
+          "monitoring module: Exporting metrics for #{prefix}. " \
+          "#{@recorders[prefix].views_data}")
+        @exporters[prefix].export @recorders[prefix].views_data
+      end
+    rescue StandardError => e
+      # TODO(lingshi): Fix the error handling here. Seems like the export is
+      # done asynchronously. So any failure happens silently. More details at
+      # https://github.com/census-ecosystem/opencensus-ruby-exporter-stackdriver/blob/f8de506204972548ca535eff6010d15f328df6c3/lib/opencensus/stats/exporters/stackdriver.rb#L156
+      @log.warn 'Failed to export some metrics.', error: e
+      raise e
     end
   end