fluent-plugin-google-cloud 0.10.4 → 0.10.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1e4077eb78db81677fa78de507e0aa13acd11a5a07a94004a4df57b3b3ee09a
-  data.tar.gz: 03602f3a4196b0527a164954c1015e595b7364728513843304c098f7ce440d20
+  metadata.gz: cfb8318e26804ad6f09d4e16777e3bbef7448a671f6a81dc3c2c2e30fd5e8a92
+  data.tar.gz: 4410e23d39d9dacf8fad1b9053be2fc9da055c37e3be1fcd13916a04f0ae3758
 SHA512:
-  metadata.gz: 93388ca7c2229d29dd42e6f9d35972841a4024d9e16fc5403993289c6e48f8911d7a12ff8944b73812b15b0fc2311dbf1919e11581343c90e0dfde599ad6c8a3
-  data.tar.gz: c04221d3ac69bfaa0a5831830a8138352a3956a3ad9adb9b99df027b1788252b8e1085e58b7ff180d574511f91cf2b408b0405c5310e3fd11982ad9f36f6d1c1
+  metadata.gz: f28ea72b58f2ed3d8eedea65b002fad2dda4ffc8a62b4b0a5c5c26fd54e04a058d680ea9745199fa8dcfbae5b1b984400a3cae5e88332b563453eb0c6415967e
+  data.tar.gz: 2992d107c0617ba61f6637d6f0e57d997b6b19068d9c67bbd7794adef43489a8bf7dc133601ddfe59d1a150f47ce6b41388fb0decbded25a4d49b35dfabf2b00

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.10.4)
+    fluent-plugin-google-cloud (0.10.9)
       fluentd (= 1.11.2)
       google-api-client (= 0.30.8)
       google-cloud-logging (= 1.6.6)
-      google-protobuf (= 3.12.2)
-      googleapis-common-protos (= 1.3.9)
+      google-protobuf (= 3.15.8)
+      googleapis-common-protos (= 1.3.10)
       googleauth (= 0.9.0)
       grpc (= 1.31.1)
       json (= 2.2.0)
@@ -18,20 +18,20 @@ GEM
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    ast (2.4.1)
-    concurrent-ruby (1.1.7)
-    cool.io (1.7.0)
+    ast (2.4.2)
+    concurrent-ruby (1.1.8)
+    cool.io (1.7.1)
     coveralls (0.8.23)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
       term-ansicolor (~> 1.3)
       thor (>= 0.19.4, < 2.0)
       tins (~> 1.6)
-    crack (0.4.4)
+    crack (0.4.5)
+      rexml
     declarative (0.0.20)
-    declarative-option (0.1.0)
-    docile (1.3.2)
-    faraday (0.17.3)
+    docile (1.3.5)
+    faraday (0.17.4)
       multipart-post (>= 1.2, < 3)
     fluentd (1.11.2)
       cool.io (>= 1.4.5, < 2.0.0)
@@ -57,12 +57,12 @@ GEM
       representable (~> 3.0)
       retriable (>= 2.0, < 4.0)
       signet (~> 0.10)
-    google-cloud-core (1.5.0)
+    google-cloud-core (1.6.0)
       google-cloud-env (~> 1.0)
       google-cloud-errors (~> 1.0)
-    google-cloud-env (1.4.0)
+    google-cloud-env (1.5.0)
       faraday (>= 0.17.3, < 2.0)
-    google-cloud-errors (1.0.1)
+    google-cloud-errors (1.1.0)
     google-cloud-logging (1.6.6)
       concurrent-ruby (~> 1.1)
       google-cloud-core (~> 1.2)
@@ -73,16 +73,16 @@ GEM
       google-gax (~> 1.8)
       googleapis-common-protos (>= 1.3.9, < 2.0)
       googleapis-common-protos-types (>= 1.0.4, < 2.0)
-    google-cloud-trace (0.40.0)
+    google-cloud-trace (0.41.0)
       concurrent-ruby (~> 1.1)
       google-cloud-core (~> 1.5)
       google-cloud-trace-v1 (~> 0.0)
       google-cloud-trace-v2 (~> 0.0)
       stackdriver-core (~> 1.3)
-    google-cloud-trace-v1 (0.1.2)
+    google-cloud-trace-v1 (0.3.0)
       gapic-common (~> 0.3)
       google-cloud-errors (~> 1.0)
-    google-cloud-trace-v2 (0.1.2)
+    google-cloud-trace-v2 (0.3.0)
       gapic-common (~> 0.3)
       google-cloud-errors (~> 1.0)
     google-gax (1.8.1)
@@ -91,13 +91,13 @@ GEM
       googleauth (~> 0.9)
       grpc (~> 1.24)
       rly (~> 0.2.3)
-    google-protobuf (3.12.2)
-    googleapis-common-protos (1.3.9)
-      google-protobuf (~> 3.0)
-      googleapis-common-protos-types (~> 1.0)
-      grpc (~> 1.0)
-    googleapis-common-protos-types (1.0.5)
+    google-protobuf (3.15.8)
+    googleapis-common-protos (1.3.10)
       google-protobuf (~> 3.11)
+      googleapis-common-protos-types (>= 1.0.5, < 2.0)
+      grpc (~> 1.27)
+    googleapis-common-protos-types (1.0.6)
+      google-protobuf (~> 3.14)
     googleauth (0.9.0)
       faraday (~> 0.12)
       jwt (>= 1.4, < 3.0)
@@ -112,13 +112,13 @@ GEM
     http_parser.rb (0.6.0)
     httpclient (2.8.3)
     json (2.2.0)
-    jwt (2.2.2)
+    jwt (2.2.3)
     memoist (0.16.2)
     metaclass (0.0.4)
-    mini_mime (1.0.2)
+    mini_mime (1.1.0)
     mocha (1.9.0)
       metaclass (~> 0.0.1)
-    msgpack (1.3.3)
+    msgpack (1.4.2)
     multi_json (1.15.0)
     multipart-post (2.1.1)
     opencensus (0.5.0)
@@ -130,8 +130,8 @@ GEM
     os (1.1.1)
     parser (2.7.2.0)
       ast (~> 2.4.1)
-    power_assert (1.2.0)
-    powerpack (0.1.2)
+    power_assert (2.0.0)
+    powerpack (0.1.3)
     prometheus-client (0.9.0)
       quantile (~> 0.2.1)
     public_suffix (4.0.6)
@@ -139,11 +139,12 @@ GEM
     rainbow (2.2.2)
       rake
     rake (10.5.0)
-    representable (3.0.4)
+    representable (3.1.1)
       declarative (< 0.1.0)
-      declarative-option (< 0.2.0)
+      trailblazer-option (>= 0.1.1, < 0.2.0)
       uber (< 0.2.0)
     retriable (3.1.2)
+    rexml (3.2.5)
     rly (0.2.3)
     rubocop (0.39.0)
       parser (>= 2.3.0.7, < 3.0)
@@ -151,11 +152,11 @@ GEM
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.10.1)
-    serverengine (2.2.1)
+    ruby-progressbar (1.11.0)
+    serverengine (2.2.3)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    signet (0.14.0)
+    signet (0.15.0)
       addressable (~> 2.3)
       faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
@@ -165,7 +166,7 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    stackdriver-core (1.4.0)
+    stackdriver-core (1.5.0)
       google-cloud-core (~> 1.2)
     strptime (0.2.5)
     sync (0.5.0)
@@ -173,12 +174,13 @@ GEM
       tins (~> 1.0)
     test-unit (3.3.3)
       power_assert
-    thor (1.0.1)
-    tins (1.26.0)
+    thor (1.1.0)
+    tins (1.28.0)
       sync
-    tzinfo (2.0.2)
+    trailblazer-option (0.1.1)
+    tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2020.4)
+    tzinfo-data (1.2021.1)
       tzinfo (>= 1.0.0)
     uber (0.1.0)
     unicode-display_width (1.7.0)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.10.4'
+  gem.version       = '0.10.9'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
@@ -20,11 +20,11 @@ eos
   gem.require_paths = ['lib']
 
   gem.add_runtime_dependency 'fluentd', '1.11.2'
-  gem.add_runtime_dependency 'googleapis-common-protos', '1.3.9'
+  gem.add_runtime_dependency 'googleapis-common-protos', '1.3.10'
   gem.add_runtime_dependency 'googleauth', '0.9.0'
   gem.add_runtime_dependency 'google-api-client', '0.30.8'
   gem.add_runtime_dependency 'google-cloud-logging', '1.6.6'
-  gem.add_runtime_dependency 'google-protobuf', '3.12.2'
+  gem.add_runtime_dependency 'google-protobuf', '3.15.8'
   gem.add_runtime_dependency 'grpc', '1.31.1'
   gem.add_runtime_dependency 'json', '2.2.0'
   gem.add_runtime_dependency 'opencensus', '0.5.0'

data/lib/fluent/plugin/filter_analyze_config.rb

@@ -15,6 +15,7 @@
 require 'fileutils'
 require 'fluent/config'
 require 'fluent/config/v1_parser'
+require 'fluent/plugin_helper'
 require 'googleauth'
 require 'google/apis/logging_v2'
 require 'open-uri'
@@ -32,6 +33,10 @@ module Fluent
     include Fluent::Config
     Fluent::Plugin.register_filter('analyze_config', self)
 
+    # Required for the timer_execute method below
+    include PluginHelper::Mixin
+    helpers :timer
+
     module Constants
       # Built-in plugins that are ok to reference in metrics.
       KNOWN_PLUGINS = {
@@ -212,7 +217,7 @@ module Fluent
       if File.file?(@google_fluentd_config_path) &&
          File.file?(@google_fluentd_baseline_config_path)
         @log.info(
-          'google-fluentd configuration file found at' \
+          'analyze_config plugin: google-fluentd configuration file found at' \
           " #{@google_fluentd_config_path}. " \
           'google-fluentd baseline configuration file found at' \
           " #{@google_fluentd_baseline_config_path}. " \
@@ -240,27 +245,49 @@ module Fluent
             "analyze_config plugin: monitoring_type #{@monitoring_type} is " \
             'unknown; there will be no metrics.')
         end
-        registry = Monitoring::MonitoringRegistryFactory.create(
+
+        @registry = Monitoring::MonitoringRegistryFactory.create(
           @monitoring_type, project_id, resource, @gcm_service_address)
+        # Export metrics every 60 seconds.
+        timer_execute(:export_config_analysis_metrics, 60) { @registry.export }
 
-        plugin_usage = registry.counter(
+        @log.info('analyze_config plugin: Registering counters.')
+        enabled_plugins_counter = @registry.counter(
           :enabled_plugins,
-          [:plugin_name, :is_default_plugin, :has_default_config],
+          [:plugin_name, :is_default_plugin,
+           :has_default_config, :has_ruby_snippet],
           'Enabled plugins',
-          'agent.googleapis.com/agent/internal/logging/config')
-        config_usage = registry.counter(
+          'agent.googleapis.com/agent/internal/logging/config',
+          'GAUGE')
+        @log.info(
+          'analyze_config plugin: registered enable_plugins counter. ' \
+          "#{enabled_plugins_counter}")
+        plugin_config_counter = @registry.counter(
           :plugin_config,
           [:plugin_name, :param, :is_present, :has_default_config],
           'Configuration parameter usage for plugins relevant to Google Cloud.',
-          'agent.googleapis.com/agent/internal/logging/config')
-        config_bool_values = registry.counter(
+          'agent.googleapis.com/agent/internal/logging/config',
+          'GAUGE')
+        @log.info('analyze_config plugin: registered plugin_config counter. ' \
+          "#{plugin_config_counter}")
+        config_bool_values_counter = @registry.counter(
           :config_bool_values,
           [:plugin_name, :param, :value],
           'Values for bool parameters in Google Cloud plugins',
-          'agent.googleapis.com/agent/internal/logging/config')
+          'agent.googleapis.com/agent/internal/logging/config',
+          'GAUGE')
+        @log.info('analyze_config plugin: registered config_bool_values ' \
+          "counter. #{config_bool_values_counter}")
 
         config = parse_config(@google_fluentd_config_path)
+        @log.debug(
+          'analyze_config plugin: successfully parsed google-fluentd' \
+          " configuration file at #{@google_fluentd_config_path}. #{config}")
         baseline_config = parse_config(@google_fluentd_baseline_config_path)
+        @log.debug(
+          'analyze_config plugin: successfully parsed google-fluentd' \
+          ' baseline configuration file at' \
+          " #{@google_fluentd_baseline_config_path}: #{baseline_config}")
 
         # Create hash of all baseline elements by their plugin names.
         baseline_elements = Hash[baseline_config.elements.collect do |e|
@@ -290,7 +317,7 @@ module Fluent
             is_default_plugin = false
             has_default_config = false
           end
-          plugin_usage.increment(
+          enabled_plugins_counter.increment(
             labels: {
               plugin_name: plugin_name,
               is_default_plugin: is_default_plugin,
@@ -303,7 +330,7 @@ module Fluent
           # detect_exceptions).
           next unless GOOGLE_PLUGIN_PARAMS.key?(e['@type'])
           GOOGLE_PLUGIN_PARAMS[e['@type']].each do |p|
-            config_usage.increment(
+            plugin_config_counter.increment(
               labels: {
                 plugin_name: e['@type'],
                 param: p,
@@ -314,7 +341,7 @@ module Fluent
               },
               by: 1)
             next unless e.key?(p) && %w(true false).include?(e[p])
-            config_bool_values.increment(
+            config_bool_values_counter.increment(
               labels: {
                 plugin_name: e['@type'],
                 param: p,
@@ -323,6 +350,8 @@ module Fluent
               by: 1)
           end
         end
+        @log.info(
+          'analyze_config plugin: Successfully finished analyzing config.')
       else
         @log.info(
           'analyze_config plugin: google-fluentd configuration file does not ' \
@@ -341,6 +370,9 @@ module Fluent
 
     def shutdown
       super
+      # Export metrics on shutdown. This is a best-effort attempt, and it might
+      # fail, for instance if there was a recent write to the same time series.
+      @registry.export unless @registry.nil?
     end
 
     # rubocop:disable Lint/UnusedMethodArgument

data/lib/fluent/plugin/monitoring.rb

@@ -52,7 +52,7 @@ module Monitoring
     def initialize(_project_id, _monitored_resource, _gcm_service_address)
     end
 
-    def counter(_name, _labels, _docstring, _prefix)
+    def counter(_name, _labels, _docstring, _prefix, _aggregation)
       BaseCounter.new
     end
 
@@ -75,7 +75,7 @@ module Monitoring
     end
 
     # Exception-driven behavior to avoid synchronization errors.
-    def counter(name, _labels, docstring, _prefix)
+    def counter(name, _labels, docstring, _prefix, _aggregation)
       # When we upgrade to Prometheus client 0.10.0 or higher, pass the
       # labels in the metric constructor. The 'labels' field in
       # Prometheus client 0.9.0 has a different function and will not
@@ -102,12 +102,18 @@ module Monitoring
       @gcm_service_address = gcm_service_address
       @recorders = {}
       @exporters = {}
+      @log.info(
+        'monitoring module: Successfully initialized Open Census monitoring ' \
+        'registry.')
     end
 
-    def counter(name, labels, docstring, prefix)
+    def counter(name, labels, docstring, prefix, aggregation)
       translator = MetricTranslator.new(name, labels)
       measure = OpenCensus::Stats::MeasureRegistry.get(translator.name)
       if measure.nil?
+        @log.info(
+          'monitoring module: Registering a new measure registry for ' \
+          "#{translator.name}")
         measure = OpenCensus::Stats.create_measure_int(
           name: translator.name,
           unit: OpenCensus::Stats::Measure::UNIT_NONE,
@@ -115,7 +121,10 @@ module Monitoring
         )
       end
       unless @exporters.keys.include?(prefix)
-        @recorders[prefix] = OpenCensus::Stats.ensure_recorder
+        @log.info(
+          'monitoring module: Registering a new exporter for ' \
+          "#{prefix}")
+        @recorders[prefix] = OpenCensus::Stats::Recorder.new
         @exporters[prefix] = \
           OpenCensus::Stats::Exporters::Stackdriver.new(
             project_id: @project_id,
@@ -124,25 +133,47 @@ module Monitoring
             resource_labels: @metrics_monitored_resource.labels,
             gcm_service_address: @gcm_service_address
           )
+        @log.info(
+          'monitoring module: Registered recorders and exporters for ' \
+          "#{prefix}.\n#{@exporters[prefix]}")
       end
-      OpenCensus::Stats.create_and_register_view(
-        name: translator.name,
-        measure: measure,
-        aggregation: OpenCensus::Stats.create_sum_aggregation,
-        description: docstring,
-        columns: translator.view_labels.map(&:to_s)
+      if aggregation == 'GAUGE'
+        stats_aggregation = OpenCensus::Stats.create_last_value_aggregation
+      else
+        stats_aggregation = OpenCensus::Stats.create_sum_aggregation
+      end
+      @recorders[prefix].register_view(
+        OpenCensus::Stats::View.new(
+          name: translator.name,
+          measure: measure,
+          aggregation: stats_aggregation,
+          description: docstring,
+          columns: translator.view_labels.map(&:to_s)
+        )
       )
-      OpenCensusCounter.new(@recorders[prefix], measure, translator)
+      counter = OpenCensusCounter.new(@recorders[prefix], measure, translator)
+      @log.info(
+        'monitoring module: Successfully initialized Open Census counter for ' \
+        "#{prefix}/#{name}.")
+      counter
     rescue StandardError => e
       @log.warn "Failed to count metrics for #{name}.", error: e
       raise e
     end
 
     def export
+      @log.debug(
+        "monitoring module: Exporting metrics for #{@exporters.keys}.")
       @exporters.keys.each do |prefix|
+        @log.debug(
+          "monitoring module: Exporting metrics for #{prefix}. " \
+          "#{@recorders[prefix].views_data}")
         @exporters[prefix].export @recorders[prefix].views_data
       end
     rescue StandardError => e
+      # TODO(lingshi): Fix the error handling here. Seems like the export is
+      # done asynchronously. So any failure happens silently. More details at
+      # https://github.com/census-ecosystem/opencensus-ruby-exporter-stackdriver/blob/f8de506204972548ca535eff6010d15f328df6c3/lib/opencensus/stats/exporters/stackdriver.rb#L156
       @log.warn 'Failed to export some metrics.', error: e
       raise e
     end

data/lib/fluent/plugin/out_google_cloud.rb

@@ -564,37 +564,37 @@ module Fluent
         # we can't change it.
         @uptime_metric = @registry.counter(
           :uptime, [:version], 'Uptime of Logging agent',
-          'agent.googleapis.com/agent')
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         update_uptime
         timer_execute(:update_uptime, 1) { update_uptime }
         @successful_requests_count = @registry.counter(
           :stackdriver_successful_requests_count,
           [:grpc, :code],
           'A number of successful requests to the Stackdriver Logging API',
-          'agent.googleapis.com/agent')
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @failed_requests_count = @registry.counter(
           :stackdriver_failed_requests_count,
           [:grpc, :code],
           'A number of failed requests to the Stackdriver Logging '\
           'API, broken down by the error code',
-          'agent.googleapis.com/agent')
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @ingested_entries_count = @registry.counter(
           :stackdriver_ingested_entries_count,
           [:grpc, :code],
           'A number of log entries ingested by Stackdriver Logging',
-          'agent.googleapis.com/agent')
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @dropped_entries_count = @registry.counter(
           :stackdriver_dropped_entries_count,
           [:grpc, :code],
           'A number of log entries dropped by the Stackdriver output plugin',
-          'agent.googleapis.com/agent')
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @retried_entries_count = @registry.counter(
           :stackdriver_retried_entries_count,
           [:grpc, :code],
           'The number of log entries that failed to be ingested by '\
           'the Stackdriver output plugin due to a transient error '\
           'and were retried',
-          'agent.googleapis.com/agent')
+          'agent.googleapis.com/agent', 'CUMULATIVE')
         @ok_code = @use_grpc ? GRPC::Core::StatusCodes::OK : 200
       end