fluent-plugin-google-cloud 0.10.2 → 0.10.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 202c2767566b8903d03b55b211a5e29c55d77c08b2d8ab1b07c3c4a6c16ea4a9
-  data.tar.gz: 9175afb12ae142d19ad72c80384b0e5ea22766524d599d1343f0a1c55a564a27
+  metadata.gz: 6cc4f838b0d129461db3849d1b089ce76b24a3c19baf8b984d2ec00bf2f268d0
+  data.tar.gz: 43109d8b31ea5bdd15e3e21f10a7786eb7e197393ab3a61edc66ee9009c4e848
 SHA512:
-  metadata.gz: 1bc3b60cac80908283a91c80f4db157117579119bfc13c7d233de706256a3e458d8d6c8c3fa5aea99ef840123caabb6fe45ae270f795a66720ab0742792107a6
-  data.tar.gz: b987203a4b12ac2c1a0679362ae1fd6c1e332ae258a0fb8094d6a15f88d84cce9a15d8f76e3129cffa8a2f70f607a8b5f746f8840b67fd708b2ade66462d426d
+  metadata.gz: 51ac3c4dfa9034eab5bb282112a90352b96cfc34cdeda5563df60bc8f9baff015e384bc4c838389b08556e1007c83890b5be34def35dd1a6826bc35b36351dfe
+  data.tar.gz: 2579e12ee867d18466644c8f8d695059831f643b9b50da3e359ddaae31ee429c449620827cc25acdc1509f809f6b2352aa103e53b80b0367bc915e10f4497b51

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.10.2)
+    fluent-plugin-google-cloud (0.10.7)
       fluentd (= 1.11.2)
       google-api-client (= 0.30.8)
       google-cloud-logging (= 1.6.6)
-      google-protobuf (= 3.12.2)
-      googleapis-common-protos (= 1.3.9)
+      google-protobuf (= 3.14.0)
+      googleapis-common-protos (= 1.3.10)
       googleauth (= 0.9.0)
       grpc (= 1.31.1)
       json (= 2.2.0)
@@ -18,21 +18,21 @@ GEM
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    ast (2.4.1)
-    concurrent-ruby (1.1.7)
-    cool.io (1.6.0)
+    ast (2.4.2)
+    concurrent-ruby (1.1.8)
+    cool.io (1.7.1)
     coveralls (0.8.23)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
       term-ansicolor (~> 1.3)
       thor (>= 0.19.4, < 2.0)
       tins (~> 1.6)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
+    crack (0.4.5)
+      rexml
     declarative (0.0.20)
     declarative-option (0.1.0)
-    docile (1.3.2)
-    faraday (0.17.3)
+    docile (1.3.5)
+    faraday (0.17.4)
       multipart-post (>= 1.2, < 3)
     fluentd (1.11.2)
       cool.io (>= 1.4.5, < 2.0.0)
@@ -58,12 +58,12 @@ GEM
       representable (~> 3.0)
       retriable (>= 2.0, < 4.0)
       signet (~> 0.10)
-    google-cloud-core (1.5.0)
+    google-cloud-core (1.6.0)
       google-cloud-env (~> 1.0)
       google-cloud-errors (~> 1.0)
-    google-cloud-env (1.3.3)
+    google-cloud-env (1.5.0)
       faraday (>= 0.17.3, < 2.0)
-    google-cloud-errors (1.0.1)
+    google-cloud-errors (1.1.0)
     google-cloud-logging (1.6.6)
       concurrent-ruby (~> 1.1)
       google-cloud-core (~> 1.2)
@@ -74,16 +74,16 @@ GEM
       google-gax (~> 1.8)
       googleapis-common-protos (>= 1.3.9, < 2.0)
       googleapis-common-protos-types (>= 1.0.4, < 2.0)
-    google-cloud-trace (0.40.0)
+    google-cloud-trace (0.41.0)
       concurrent-ruby (~> 1.1)
       google-cloud-core (~> 1.5)
       google-cloud-trace-v1 (~> 0.0)
       google-cloud-trace-v2 (~> 0.0)
       stackdriver-core (~> 1.3)
-    google-cloud-trace-v1 (0.1.2)
+    google-cloud-trace-v1 (0.3.0)
       gapic-common (~> 0.3)
       google-cloud-errors (~> 1.0)
-    google-cloud-trace-v2 (0.1.2)
+    google-cloud-trace-v2 (0.3.0)
       gapic-common (~> 0.3)
       google-cloud-errors (~> 1.0)
     google-gax (1.8.1)
@@ -92,13 +92,13 @@ GEM
       googleauth (~> 0.9)
       grpc (~> 1.24)
       rly (~> 0.2.3)
-    google-protobuf (3.12.2)
-    googleapis-common-protos (1.3.9)
-      google-protobuf (~> 3.0)
-      googleapis-common-protos-types (~> 1.0)
-      grpc (~> 1.0)
-    googleapis-common-protos-types (1.0.5)
+    google-protobuf (3.14.0)
+    googleapis-common-protos (1.3.10)
       google-protobuf (~> 3.11)
+      googleapis-common-protos-types (>= 1.0.5, < 2.0)
+      grpc (~> 1.27)
+    googleapis-common-protos-types (1.0.6)
+      google-protobuf (~> 3.14)
     googleauth (0.9.0)
       faraday (~> 0.12)
       jwt (>= 1.4, < 3.0)
@@ -119,7 +119,7 @@ GEM
     mini_mime (1.0.2)
     mocha (1.9.0)
       metaclass (~> 0.0.1)
-    msgpack (1.3.3)
+    msgpack (1.4.2)
     multi_json (1.15.0)
     multipart-post (2.1.1)
     opencensus (0.5.0)
@@ -129,10 +129,10 @@ GEM
       google-cloud-trace (~> 0.35)
       opencensus (~> 0.5)
     os (1.1.1)
-    parser (2.7.1.4)
+    parser (2.7.2.0)
       ast (~> 2.4.1)
-    power_assert (1.2.0)
-    powerpack (0.1.2)
+    power_assert (2.0.0)
+    powerpack (0.1.3)
     prometheus-client (0.9.0)
       quantile (~> 0.2.1)
     public_suffix (4.0.6)
@@ -145,6 +145,7 @@ GEM
       declarative-option (< 0.2.0)
       uber (< 0.2.0)
     retriable (3.1.2)
+    rexml (3.2.4)
     rly (0.2.3)
     rubocop (0.39.0)
       parser (>= 2.3.0.7, < 3.0)
@@ -152,12 +153,11 @@ GEM
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.10.1)
-    safe_yaml (1.0.5)
-    serverengine (2.2.1)
+    ruby-progressbar (1.11.0)
+    serverengine (2.2.3)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    signet (0.14.0)
+    signet (0.15.0)
       addressable (~> 2.3)
       faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
@@ -167,20 +167,20 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    stackdriver-core (1.4.0)
+    stackdriver-core (1.5.0)
       google-cloud-core (~> 1.2)
-    strptime (0.2.4)
+    strptime (0.2.5)
     sync (0.5.0)
     term-ansicolor (1.7.1)
       tins (~> 1.0)
     test-unit (3.3.3)
       power_assert
-    thor (1.0.1)
-    tins (1.25.0)
+    thor (1.1.0)
+    tins (1.28.0)
       sync
-    tzinfo (2.0.2)
+    tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2020.1)
+    tzinfo-data (1.2021.1)
       tzinfo (>= 1.0.0)
     uber (0.1.0)
     unicode-display_width (1.7.0)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.10.2'
+  gem.version       = '0.10.7'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
@@ -20,11 +20,11 @@ eos
   gem.require_paths = ['lib']
 
   gem.add_runtime_dependency 'fluentd', '1.11.2'
-  gem.add_runtime_dependency 'googleapis-common-protos', '1.3.9'
+  gem.add_runtime_dependency 'googleapis-common-protos', '1.3.10'
   gem.add_runtime_dependency 'googleauth', '0.9.0'
   gem.add_runtime_dependency 'google-api-client', '0.30.8'
   gem.add_runtime_dependency 'google-cloud-logging', '1.6.6'
-  gem.add_runtime_dependency 'google-protobuf', '3.12.2'
+  gem.add_runtime_dependency 'google-protobuf', '3.14.0'
   gem.add_runtime_dependency 'grpc', '1.31.1'
   gem.add_runtime_dependency 'json', '2.2.0'
   gem.add_runtime_dependency 'opencensus', '0.5.0'

data/lib/fluent/plugin/filter_analyze_config.rb

@@ -15,6 +15,10 @@
 require 'fileutils'
 require 'fluent/config'
 require 'fluent/config/v1_parser'
+require 'fluent/plugin_helper'
+require 'googleauth'
+require 'google/apis/logging_v2'
+require 'open-uri'
 require 'set'
 
 require_relative 'common'
@@ -29,6 +33,10 @@ module Fluent
     include Fluent::Config
     Fluent::Plugin.register_filter('analyze_config', self)
 
+    # Required for the timer_execute method below
+    include PluginHelper::Mixin
+    helpers :timer
+
     module Constants
       # Built-in plugins that are ok to reference in metrics.
       KNOWN_PLUGINS = {
@@ -155,8 +163,8 @@ module Fluent
       super
       @log = $log # rubocop:disable Style/GlobalVars
 
-      # Initialize the insertID.
-      @log.info 'Started the analyze_config plugin to analyze configuration.'
+      @log.info(
+        'analyze_config plugin: Started the plugin to analyze configuration.')
     end
 
     def parse_config(path)
@@ -205,10 +213,11 @@ module Fluent
 
     def configure(conf)
       super
+      @log.info('analyze_config plugin: Starting to configure the plugin.')
       if File.file?(@google_fluentd_config_path) &&
          File.file?(@google_fluentd_baseline_config_path)
         @log.info(
-          'google-fluentd configuration file found at' \
+          'analyze_config plugin: google-fluentd configuration file found at' \
           " #{@google_fluentd_config_path}. " \
           'google-fluentd baseline configuration file found at' \
           " #{@google_fluentd_baseline_config_path}. " \
@@ -232,27 +241,53 @@ module Fluent
 
         unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
           @monitoring_type)
-          @log.warn "monitoring_type '#{@monitoring_type}' is unknown; "\
-                    'there will be no metrics'
+          @log.warn(
+            "analyze_config plugin: monitoring_type #{@monitoring_type} is " \
+            'unknown; there will be no metrics.')
         end
-        registry = Monitoring::MonitoringRegistryFactory.create(
+
+        @registry = Monitoring::MonitoringRegistryFactory.create(
           @monitoring_type, project_id, resource, @gcm_service_address)
+        # Export metrics every 60 seconds.
+        timer_execute(:export_config_analysis_metrics, 60) { @registry.export }
 
-        plugin_usage = registry.counter(
-          :stackdriver_enabled_plugins,
-          [:plugin_name, :is_default_plugin, :has_default_value],
-          'Enabled plugins')
-        config_usage = registry.counter(
-          :stackdriver_config_usage,
-          [:plugin_name, :param, :is_present, :has_default_value],
-          'Parameter usage for Google Cloud plugins')
-        config_bool_values = registry.counter(
-          :stackdriver_config_bool_values,
+        @log.info('analyze_config plugin: Registering counters.')
+        enabled_plugins_counter = @registry.counter(
+          :enabled_plugins,
+          [:plugin_name, :is_default_plugin,
+           :has_default_config, :has_ruby_snippet],
+          'Enabled plugins',
+          'agent.googleapis.com/agent/internal/logging/config',
+          'GAUGE')
+        @log.info(
+          'analyze_config plugin: registered enable_plugins counter. ' \
+          "#{enabled_plugins_counter}")
+        plugin_config_counter = @registry.counter(
+          :plugin_config,
+          [:plugin_name, :param, :is_present, :has_default_config],
+          'Configuration parameter usage for plugins relevant to Google Cloud.',
+          'agent.googleapis.com/agent/internal/logging/config',
+          'GAUGE')
+        @log.info('analyze_config plugin: registered plugin_config counter. ' \
+          "#{plugin_config_counter}")
+        config_bool_values_counter = @registry.counter(
+          :config_bool_values,
           [:plugin_name, :param, :value],
-          'Values for bool parameters in Google Cloud plugins')
+          'Values for bool parameters in Google Cloud plugins',
+          'agent.googleapis.com/agent/internal/logging/config',
+          'GAUGE')
+        @log.info('analyze_config plugin: registered config_bool_values ' \
+          "counter. #{config_bool_values_counter}")
 
         config = parse_config(@google_fluentd_config_path)
+        @log.debug(
+          'analyze_config plugin: successfully parsed google-fluentd' \
+          " configuration file at #{@google_fluentd_config_path}. #{config}")
         baseline_config = parse_config(@google_fluentd_baseline_config_path)
+        @log.debug(
+          'analyze_config plugin: successfully parsed google-fluentd' \
+          ' baseline configuration file at' \
+          " #{@google_fluentd_baseline_config_path}: #{baseline_config}")
 
         # Create hash of all baseline elements by their plugin names.
         baseline_elements = Hash[baseline_config.elements.collect do |e|
@@ -276,17 +311,17 @@ module Fluent
           plugin_name = default_plugin_name(e)
           if baseline_elements.key?(plugin_name)
             is_default_plugin = true
-            has_default_value = (baseline_elements[plugin_name] == e)
+            has_default_config = (baseline_elements[plugin_name] == e)
           else
             plugin_name = custom_plugin_name(e)
             is_default_plugin = false
-            has_default_value = false
+            has_default_config = false
           end
-          plugin_usage.increment(
+          enabled_plugins_counter.increment(
             labels: {
               plugin_name: plugin_name,
               is_default_plugin: is_default_plugin,
-              has_default_value: has_default_value,
+              has_default_config: has_default_config,
               has_ruby_snippet: embedded_ruby?(e)
             },
             by: 1)
@@ -295,18 +330,18 @@ module Fluent
           # detect_exceptions).
           next unless GOOGLE_PLUGIN_PARAMS.key?(e['@type'])
           GOOGLE_PLUGIN_PARAMS[e['@type']].each do |p|
-            config_usage.increment(
+            plugin_config_counter.increment(
               labels: {
                 plugin_name: e['@type'],
                 param: p,
                 is_present: e.key?(p),
-                has_default_value: (e.key?(p) &&
+                has_default_config: (e.key?(p) &&
                                     baseline_google_element.key?(p) &&
                                     e[p] == baseline_google_element[p])
               },
               by: 1)
             next unless e.key?(p) && %w(true false).include?(e[p])
-            config_bool_values.increment(
+            config_bool_values_counter.increment(
               labels: {
                 plugin_name: e['@type'],
                 param: p,
@@ -315,23 +350,29 @@ module Fluent
               by: 1)
           end
         end
+        @log.info(
+          'analyze_config plugin: Successfully finished analyzing config.')
       else
         @log.info(
-          'google-fluentd configuration file does not exist at' \
-          " #{@google_fluentd_config_path} or " \
-          'google-fluentd baseline configuration file does not exist at' \
-          " #{@google_fluentd_baseline_config_path} or " \
-          '. Skipping configuration analysis.')
+          'analyze_config plugin: google-fluentd configuration file does not ' \
+          "exist at #{@google_fluentd_config_path} or google-fluentd " \
+          'baseline configuration file does not exist at' \
+          " #{@google_fluentd_baseline_config_path}. Skipping configuration " \
+          'analysis.')
       end
     rescue => e
       # Do not crash the agent due to configuration analysis failures.
       @log.warn(
-        'Failed to optionally analyze the google-fluentd configuration' \
-        " file. Proceeding anyway. Error: #{e}")
+        'analyze_config plugin: Failed to optionally analyze the ' \
+        "google-fluentd configuration file. Proceeding anyway. Error: #{e}. " \
+        "Trace: #{e.backtrace}")
     end
 
     def shutdown
       super
+      # Export metrics on shutdown. This is a best-effort attempt, and it might
+      # fail, for instance if there was a recent write to the same time series.
+      @registry.export unless @registry.nil?
     end
 
     # rubocop:disable Lint/UnusedMethodArgument

data/lib/fluent/plugin/monitoring.rb

@@ -52,7 +52,7 @@ module Monitoring
     def initialize(_project_id, _monitored_resource, _gcm_service_address)
     end
 
-    def counter(_name, _labels, _docstring)
+    def counter(_name, _labels, _docstring, _prefix, _aggregation)
       BaseCounter.new
     end
 
@@ -75,7 +75,7 @@ module Monitoring
     end
 
     # Exception-driven behavior to avoid synchronization errors.
-    def counter(name, _labels, docstring)
+    def counter(name, _labels, docstring, _prefix, _aggregation)
       # When we upgrade to Prometheus client 0.10.0 or higher, pass the
       # labels in the metric constructor. The 'labels' field in
       # Prometheus client 0.9.0 has a different function and will not
@@ -97,42 +97,85 @@ module Monitoring
       require 'opencensus'
       require 'opencensus-stackdriver'
       @log = $log # rubocop:disable Style/GlobalVars
-      @recorder = OpenCensus::Stats.ensure_recorder
-      @exporter = OpenCensus::Stats::Exporters::Stackdriver.new(
-        project_id: project_id,
-        metric_prefix: 'agent.googleapis.com/agent',
-        resource_type: monitored_resource.type,
-        resource_labels: monitored_resource.labels,
-        gcm_service_address: gcm_service_address
-      )
-      OpenCensus.configure do |c|
-        c.stats.exporter = @exporter
-      end
-      @log.debug "OpenCensus config=#{OpenCensus.config}"
+      @project_id = project_id
+      @metrics_monitored_resource = monitored_resource
+      @gcm_service_address = gcm_service_address
+      @recorders = {}
+      @exporters = {}
+      @log.info(
+        'monitoring module: Successfully initialized Open Census monitoring ' \
+        'registry.')
     end
 
-    def counter(name, labels, docstring)
+    def counter(name, labels, docstring, prefix, aggregation)
       translator = MetricTranslator.new(name, labels)
       measure = OpenCensus::Stats::MeasureRegistry.get(translator.name)
       if measure.nil?
+        @log.info(
+          'monitoring module: Registering a new measure registry for ' \
+          "#{translator.name}")
         measure = OpenCensus::Stats.create_measure_int(
           name: translator.name,
           unit: OpenCensus::Stats::Measure::UNIT_NONE,
           description: docstring
         )
       end
-      OpenCensus::Stats.create_and_register_view(
-        name: translator.name,
-        measure: measure,
-        aggregation: OpenCensus::Stats.create_sum_aggregation,
-        description: docstring,
-        columns: translator.view_labels.map(&:to_s)
+      unless @exporters.keys.include?(prefix)
+        @log.info(
+          'monitoring module: Registering a new exporter for ' \
+          "#{prefix}")
+        @recorders[prefix] = OpenCensus::Stats::Recorder.new
+        @exporters[prefix] = \
+          OpenCensus::Stats::Exporters::Stackdriver.new(
+            project_id: @project_id,
+            metric_prefix: prefix,
+            resource_type: @metrics_monitored_resource.type,
+            resource_labels: @metrics_monitored_resource.labels,
+            gcm_service_address: @gcm_service_address
+          )
+        @log.info(
+          'monitoring module: Registered recorders and exporters for ' \
+          "#{prefix}.\n#{@exporters[prefix]}")
+      end
+      if aggregation == 'GAUGE'
+        stats_aggregation = OpenCensus::Stats.create_last_value_aggregation
+      else
+        stats_aggregation = OpenCensus::Stats.create_sum_aggregation
+      end
+      @recorders[prefix].register_view(
+        OpenCensus::Stats::View.new(
+          name: translator.name,
+          measure: measure,
+          aggregation: stats_aggregation,
+          description: docstring,
+          columns: translator.view_labels.map(&:to_s)
+        )
       )
-      OpenCensusCounter.new(@recorder, measure, translator)
+      counter = OpenCensusCounter.new(@recorders[prefix], measure, translator)
+      @log.info(
+        'monitoring module: Successfully initialized Open Census counter for ' \
+        "#{prefix}/#{name}.")
+      counter
+    rescue StandardError => e
+      @log.warn "Failed to count metrics for #{name}.", error: e
+      raise e
     end
 
     def export
-      @exporter.export @recorder.views_data
+      @log.debug(
+        "monitoring module: Exporting metrics for #{@exporters.keys}.")
+      @exporters.keys.each do |prefix|
+        @log.debug(
+          "monitoring module: Exporting metrics for #{prefix}. " \
+          "#{@recorders[prefix].views_data}")
+        @exporters[prefix].export @recorders[prefix].views_data
+      end
+    rescue StandardError => e
+      # TODO(lingshi): Fix the error handling here. Seems like the export is
+      # done asynchronously. So any failure happens silently. More details at
+      # https://github.com/census-ecosystem/opencensus-ruby-exporter-stackdriver/blob/f8de506204972548ca535eff6010d15f328df6c3/lib/opencensus/stats/exporters/stackdriver.rb#L156
+      @log.warn 'Failed to export some metrics.', error: e
+      raise e
     end
   end