fluent-plugin-google-cloud 0.10.9 → 0.13.1

data/lib/fluent/plugin/filter_add_insert_ids.rb

@@ -51,9 +51,6 @@ module Fluent
     desc 'The field name for insertIds in the log record.'
     config_param :insert_id_key, :string, default: DEFAULT_INSERT_ID_KEY
 
-    # Expose attr_readers for testing.
-    attr_reader :insert_id_key
-
     def start
       super
       @log = $log # rubocop:disable Style/GlobalVars
@@ -65,21 +62,11 @@ module Fluent
       @log.info "Initialized the insert ID key to #{@insert_id}."
     end
 
-    def configure(conf)
-      super
-    end
-
-    def shutdown
-      super
-    end
-
     # rubocop:disable Lint/UnusedMethodArgument
     def filter(tag, time, record)
       # Only generate and add an insertId field if the record is a hash and
       # the insert ID field is not already set (or set to an empty string).
-      if record.is_a?(Hash) && record[@insert_id_key].to_s.empty?
-        record[@insert_id_key] = increment_insert_id
-      end
+      record[@insert_id_key] = increment_insert_id if record.is_a?(Hash) && record[@insert_id_key].to_s.empty?
       record
     end
     # rubocop:enable Lint/UnusedMethodArgument

data/lib/fluent/plugin/filter_analyze_config.rb

@@ -38,6 +38,8 @@ module Fluent
     helpers :timer
 
     module Constants
+      PREFIX = 'agent.googleapis.com/agent/internal/logging/config'.freeze
+
       # Built-in plugins that are ok to reference in metrics.
       KNOWN_PLUGINS = {
         'filter' => Set[
@@ -84,7 +86,7 @@ module Fluent
 
       # For Google plugins, we collect metrics on the params listed here.
       GOOGLE_PLUGIN_PARAMS = {
-        'google_cloud' => %w(
+        'google_cloud' => %w[
           adjust_invalid_timestamps
           auth_method
           autoformat_stackdriver_trace
@@ -116,8 +118,8 @@ module Fluent
           vm_id
           vm_name
           zone
-        ),
-        'detect_exceptions' => %w(
+        ],
+        'detect_exceptions' => %w[
           languages
           max_bytes
           max_lines
@@ -125,7 +127,7 @@ module Fluent
           multiline_flush_interval
           remove_tag_prefix
           stream
-        )
+        ]
       }.freeze
     end
 
@@ -164,7 +166,8 @@ module Fluent
       @log = $log # rubocop:disable Style/GlobalVars
 
       @log.info(
-        'analyze_config plugin: Started the plugin to analyze configuration.')
+        'analyze_config plugin: Started the plugin to analyze configuration.'
+      )
     end
 
     def parse_config(path)
@@ -183,32 +186,32 @@ module Fluent
     end
 
     # Returns a name for identifying plugins we ship by default.
-    def default_plugin_name(e)
-      case e['@type']
+    def default_plugin_name(conf_element)
+      case conf_element['@type']
       when 'syslog'
-        "#{e.name}/syslog/#{e['protocol_type']}"
+        "#{conf_element.name}/syslog/#{conf_element['protocol_type']}"
       when 'tail'
-        "#{e.name}/tail/#{File.basename(e['pos_file'], '.pos')}"
+        "#{conf_element.name}/tail/#{File.basename(conf_element['pos_file'], '.pos')}"
       else
-        "#{e.name}/#{e['@type']}"
+        "#{conf_element.name}/#{conf_element['@type']}"
       end
     end
 
     # Returns a name for identifying plugins not in our default
     # config.  This should not contain arbitrary user-supplied data.
-    def custom_plugin_name(e)
-      if KNOWN_PLUGINS.key?(e.name) &&
-         KNOWN_PLUGINS[e.name].include?(e['@type'])
-        "#{e.name}/#{e['@type']}"
+    def custom_plugin_name(conf_element)
+      if KNOWN_PLUGINS.key?(conf_element.name) &&
+         KNOWN_PLUGINS[conf_element.name].include?(conf_element['@type'])
+        "#{conf_element.name}/#{conf_element['@type']}"
       else
-        e.name.to_s
+        conf_element.name.to_s
       end
     end
 
-    def embedded_ruby?(e)
-      (e.arg.include?('#{') ||
-       e.any? { |_, v| v.include?('#{') } ||
-       e.elements.any? { |ee| embedded_ruby?(ee) })
+    def embedded_ruby?(conf_element)
+      (conf_element.arg.include?('#{') ||
+       conf_element.any? { |_, v| v.include?('#{') } ||
+       conf_element.elements.any? { |e| embedded_ruby?(e) })
     end
 
     def configure(conf)
@@ -221,7 +224,8 @@ module Fluent
           " #{@google_fluentd_config_path}. " \
           'google-fluentd baseline configuration file found at' \
           " #{@google_fluentd_baseline_config_path}. " \
-          'google-fluentd Analyzing configuration.')
+          'google-fluentd Analyzing configuration.'
+        )
 
         utils = Common::Utils.new(@log)
         platform = utils.detect_platform(true)
@@ -231,63 +235,76 @@ module Fluent
 
         # All metadata parameters must now be set.
         utils.check_required_metadata_variables(
-          platform, project_id, zone, vm_id)
+          platform, project_id, zone, vm_id
+        )
 
         # Retrieve monitored resource.
         # Fail over to retrieve monitored resource via the legacy path if we
         # fail to get it from Metadata Agent.
         resource = utils.determine_agent_level_monitored_resource_via_legacy(
-          platform, nil, false, vm_id, zone)
+          platform, nil, false, vm_id, zone
+        )
 
         unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
-          @monitoring_type)
+          @monitoring_type
+        )
           @log.warn(
             "analyze_config plugin: monitoring_type #{@monitoring_type} is " \
-            'unknown; there will be no metrics.')
+            'unknown; there will be no metrics.'
+          )
         end
 
         @registry = Monitoring::MonitoringRegistryFactory.create(
-          @monitoring_type, project_id, resource, @gcm_service_address)
+          @monitoring_type, project_id, resource, @gcm_service_address
+        )
         # Export metrics every 60 seconds.
-        timer_execute(:export_config_analysis_metrics, 60) { @registry.export }
+        timer_execute(:export_config_analysis_metrics, 60) do
+          @registry.update_timestamps(PREFIX) if @registry.respond_to? :update_timestamps
+          @registry.export
+        end
 
         @log.info('analyze_config plugin: Registering counters.')
         enabled_plugins_counter = @registry.counter(
           :enabled_plugins,
-          [:plugin_name, :is_default_plugin,
-           :has_default_config, :has_ruby_snippet],
+          %i[plugin_name is_default_plugin has_default_config has_ruby_snippet],
           'Enabled plugins',
-          'agent.googleapis.com/agent/internal/logging/config',
-          'GAUGE')
+          PREFIX,
+          'GAUGE'
+        )
         @log.info(
           'analyze_config plugin: registered enable_plugins counter. ' \
-          "#{enabled_plugins_counter}")
+          "#{enabled_plugins_counter}"
+        )
         plugin_config_counter = @registry.counter(
           :plugin_config,
-          [:plugin_name, :param, :is_present, :has_default_config],
+          %i[plugin_name param is_present has_default_config],
           'Configuration parameter usage for plugins relevant to Google Cloud.',
-          'agent.googleapis.com/agent/internal/logging/config',
-          'GAUGE')
+          PREFIX,
+          'GAUGE'
+        )
         @log.info('analyze_config plugin: registered plugin_config counter. ' \
           "#{plugin_config_counter}")
         config_bool_values_counter = @registry.counter(
           :config_bool_values,
-          [:plugin_name, :param, :value],
+          %i[plugin_name param value],
           'Values for bool parameters in Google Cloud plugins',
-          'agent.googleapis.com/agent/internal/logging/config',
-          'GAUGE')
+          PREFIX,
+          'GAUGE'
+        )
         @log.info('analyze_config plugin: registered config_bool_values ' \
           "counter. #{config_bool_values_counter}")
 
         config = parse_config(@google_fluentd_config_path)
         @log.debug(
           'analyze_config plugin: successfully parsed google-fluentd' \
-          " configuration file at #{@google_fluentd_config_path}. #{config}")
+          " configuration file at #{@google_fluentd_config_path}. #{config}"
+        )
         baseline_config = parse_config(@google_fluentd_baseline_config_path)
         @log.debug(
           'analyze_config plugin: successfully parsed google-fluentd' \
           ' baseline configuration file at' \
-          " #{@google_fluentd_baseline_config_path}: #{baseline_config}")
+          " #{@google_fluentd_baseline_config_path}: #{baseline_config}"
+        )
 
         # Create hash of all baseline elements by their plugin names.
         baseline_elements = Hash[baseline_config.elements.collect do |e|
@@ -324,11 +341,13 @@ module Fluent
               has_default_config: has_default_config,
               has_ruby_snippet: embedded_ruby?(e)
             },
-            by: 1)
+            by: 1
+          )
 
           # Additional metric for Google plugins (google_cloud and
           # detect_exceptions).
           next unless GOOGLE_PLUGIN_PARAMS.key?(e['@type'])
+
           GOOGLE_PLUGIN_PARAMS[e['@type']].each do |p|
             plugin_config_counter.increment(
               labels: {
@@ -339,40 +358,46 @@ module Fluent
                                     baseline_google_element.key?(p) &&
                                     e[p] == baseline_google_element[p])
               },
-              by: 1)
-            next unless e.key?(p) && %w(true false).include?(e[p])
+              by: 1
+            )
+            next unless e.key?(p) && %w[true false].include?(e[p])
+
             config_bool_values_counter.increment(
               labels: {
                 plugin_name: e['@type'],
                 param: p,
                 value: e[p] == 'true'
               },
-              by: 1)
+              by: 1
+            )
           end
         end
         @log.info(
-          'analyze_config plugin: Successfully finished analyzing config.')
+          'analyze_config plugin: Successfully finished analyzing config.'
+        )
       else
         @log.info(
           'analyze_config plugin: google-fluentd configuration file does not ' \
           "exist at #{@google_fluentd_config_path} or google-fluentd " \
           'baseline configuration file does not exist at' \
           " #{@google_fluentd_baseline_config_path}. Skipping configuration " \
-          'analysis.')
+          'analysis.'
+        )
       end
-    rescue => e
+    rescue StandardError => e
       # Do not crash the agent due to configuration analysis failures.
       @log.warn(
         'analyze_config plugin: Failed to optionally analyze the ' \
         "google-fluentd configuration file. Proceeding anyway. Error: #{e}. " \
-        "Trace: #{e.backtrace}")
+        "Trace: #{e.backtrace}"
+      )
     end
 
     def shutdown
       super
       # Export metrics on shutdown. This is a best-effort attempt, and it might
       # fail, for instance if there was a recent write to the same time series.
-      @registry.export unless @registry.nil?
+      @registry&.export
     end
 
     # rubocop:disable Lint/UnusedMethodArgument

data/lib/fluent/plugin/in_object_space_dump.rb

@@ -48,7 +48,7 @@ module Fluent
     def on_timer
       GC.start
       # Use Tempfile.create to open the file, in order to preserve the file.
-      file = Tempfile.create(['heap-' + fluentd_worker_id.to_s + '-', '.json'])
+      file = Tempfile.create(["heap-#{fluentd_worker_id}-", '.json'])
       begin
         log.info 'dumping object space to',
                  filepath: file.path,

data/lib/fluent/plugin/monitoring.rb

@@ -15,13 +15,15 @@
 module Monitoring
   # Base class for the counter.
   class BaseCounter
-    def increment(*)
+    def increment(by: 1, labels: {})
+      # No default behavior
     end
   end
 
   # Prometheus implementation of counters.
   class PrometheusCounter < BaseCounter
     def initialize(prometheus_counter)
+      super()
       @counter = prometheus_counter
     end
 
@@ -33,7 +35,9 @@ module Monitoring
   # OpenCensus implementation of counters.
   class OpenCensusCounter < BaseCounter
     def initialize(recorder, measure, translator)
+      super()
       raise ArgumentError, 'measure must not be nil' if measure.nil?
+
       @recorder = recorder
       @measure = measure
       @translator = translator
@@ -42,7 +46,8 @@ module Monitoring
     def increment(by: 1, labels: {})
       labels = @translator.translate_labels(labels)
       tag_map = OpenCensus::Tags::TagMap.new(
-        labels.map { |k, v| [k.to_s, v.to_s] }.to_h)
+        labels.map { |k, v| [k.to_s, v.to_s] }.to_h
+      )
       @recorder.record(@measure.create_measurement(value: by, tags: tag_map))
     end
   end
@@ -50,6 +55,7 @@ module Monitoring
   # Base class for the monitoring registry.
   class BaseMonitoringRegistry
     def initialize(_project_id, _monitored_resource, _gcm_service_address)
+      # no default behavior
     end
 
     def counter(_name, _labels, _docstring, _prefix, _aggregation)
@@ -80,9 +86,9 @@ module Monitoring
       # labels in the metric constructor. The 'labels' field in
       # Prometheus client 0.9.0 has a different function and will not
       # work as intended.
-      return PrometheusCounter.new(@registry.counter(name, docstring))
+      PrometheusCounter.new(@registry.counter(name, docstring))
     rescue Prometheus::Client::Registry::AlreadyRegisteredError
-      return PrometheusCounter.new(@registry.get(name))
+      PrometheusCounter.new(@registry.get(name))
     end
   end
 
@@ -104,7 +110,8 @@ module Monitoring
       @exporters = {}
       @log.info(
         'monitoring module: Successfully initialized Open Census monitoring ' \
-        'registry.')
+        'registry.'
+      )
     end
 
     def counter(name, labels, docstring, prefix, aggregation)
@@ -113,7 +120,8 @@ module Monitoring
       if measure.nil?
         @log.info(
           'monitoring module: Registering a new measure registry for ' \
-          "#{translator.name}")
+          "#{translator.name}"
+        )
         measure = OpenCensus::Stats.create_measure_int(
           name: translator.name,
           unit: OpenCensus::Stats::Measure::UNIT_NONE,
@@ -123,7 +131,8 @@ module Monitoring
       unless @exporters.keys.include?(prefix)
         @log.info(
           'monitoring module: Registering a new exporter for ' \
-          "#{prefix}")
+          "#{prefix}"
+        )
         @recorders[prefix] = OpenCensus::Stats::Recorder.new
         @exporters[prefix] = \
           OpenCensus::Stats::Exporters::Stackdriver.new(
@@ -135,13 +144,14 @@ module Monitoring
           )
         @log.info(
           'monitoring module: Registered recorders and exporters for ' \
-          "#{prefix}.\n#{@exporters[prefix]}")
-      end
-      if aggregation == 'GAUGE'
-        stats_aggregation = OpenCensus::Stats.create_last_value_aggregation
-      else
-        stats_aggregation = OpenCensus::Stats.create_sum_aggregation
+          "#{prefix}.\n#{@exporters[prefix]}"
+        )
       end
+      stats_aggregation = if aggregation == 'GAUGE'
+                            OpenCensus::Stats.create_last_value_aggregation
+                          else
+                            OpenCensus::Stats.create_sum_aggregation
+                          end
       @recorders[prefix].register_view(
         OpenCensus::Stats::View.new(
           name: translator.name,
@@ -154,20 +164,39 @@ module Monitoring
       counter = OpenCensusCounter.new(@recorders[prefix], measure, translator)
       @log.info(
         'monitoring module: Successfully initialized Open Census counter for ' \
-        "#{prefix}/#{name}.")
+        "#{prefix}/#{name}."
+      )
       counter
     rescue StandardError => e
       @log.warn "Failed to count metrics for #{name}.", error: e
       raise e
     end
 
+    # Update timestamps for each existing AggregationData without altering tags
+    # or values.
+    # This is currently only used for config analysis metrics, because we want
+    # to repeatedly send the exact same metrics as created at start-up.
+    def update_timestamps(prefix)
+      new_time = Time.now.utc
+      recorder = @recorders[prefix]
+      recorder.views_data.each do |view_data|
+        view_data.data.each_value do |aggr_data|
+          # Apply this only to GAUGE metrics. This could fail if the metric uses
+          # Distribution or other fancier aggregators.
+          aggr_data.add aggr_data.value, new_time if aggr_data.is_a? OpenCensus::Stats::AggregationData::LastValue
+        end
+      end
+    end
+
     def export
       @log.debug(
-        "monitoring module: Exporting metrics for #{@exporters.keys}.")
-      @exporters.keys.each do |prefix|
+        "monitoring module: Exporting metrics for #{@exporters.keys}."
+      )
+      @exporters.each_key do |prefix|
         @log.debug(
           "monitoring module: Exporting metrics for #{prefix}. " \
-          "#{@recorders[prefix].views_data}")
+          "#{@recorders[prefix].views_data}"
+        )
         @exporters[prefix].export @recorders[prefix].views_data
       end
     rescue StandardError => e
@@ -205,8 +234,7 @@ module Monitoring
   # Avoid this mechanism for new metrics by defining them in their final form,
   # so they don't need translation.
   class MetricTranslator
-    attr_reader :name
-    attr_reader :view_labels
+    attr_reader :name, :view_labels
 
     def initialize(name, metric_labels)
       @legacy = true
@@ -229,8 +257,9 @@ module Monitoring
 
     def translate_labels(labels)
       return labels unless @legacy
+
       translation = { code: :response_code, grpc: :grpc }
-      labels.map { |k, v| [translation[k], v] }.to_h
+      labels.transform_keys { |k| translation[k] }
     end
   end
 end