fluent-plugin-google-cloud 0.7.15 → 0.7.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7deed85b438bb17342217bdf9b4c849577217a70702005b8912f07e854e2b62f
-  data.tar.gz: 37f6366b74c5f59d1a5ded992cb16d79eeb16ed61e7f99d50b28d68527e488f5
+  metadata.gz: cfb019175c35271448646b370774c129b48096dac9a1dfb5a1904776ba0af63c
+  data.tar.gz: 07ffca87bc262f3a29b964b47ccefd69551ad3b11384b54295f1270d101d4633
 SHA512:
-  metadata.gz: 9ec2439d5df4db5196e11680d2d28da15a7602e234272536622bc24389c69c46361a07694b76dd77c2a7fb285c2bdfaf18153b9c0b692e99bc2c931d498042ba
-  data.tar.gz: 0be1859cd6518819843caf5ab391e3e406f13a60d7c91672bcad79ecbd9242f45875bd1199a47758a6bfed8a957687c9d977771a07e0afe389ec7533bac55672
+  metadata.gz: f4f2584f87ab91bd87d75b7139555928e3d5daaa073170258a7125468a5a2b24bcce968b2f3120f13f8f6ef5328029c2a5ac6171fab0e0b2b8654eb3436f6941
+  data.tar.gz: 7d917d3ecbd2d946e8c712a2c686a5322465b3154fc2c5b889f87fa299b53dd17a5396509315d8cb89a40b55c8960dffe41995c342df01baa50cc3c1a287bc7d

data/Gemfile.lock

@@ -1,15 +1,15 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.7.15)
-      fluentd (= 1.4.2)
-      google-api-client (= 0.28.4)
-      google-cloud-logging (= 1.6.0)
-      google-protobuf (= 3.6.1)
-      googleapis-common-protos (= 1.3.7)
-      googleauth (= 0.8.1)
-      grpc (= 1.14.2)
-      json (= 2.1.0)
+    fluent-plugin-google-cloud (0.7.16)
+      fluentd (= 1.6.3)
+      google-api-client (= 0.30.8)
+      google-cloud-logging (= 1.6.6)
+      google-protobuf (= 3.9.0)
+      googleapis-common-protos (= 1.3.9)
+      googleauth (= 0.9.0)
+      grpc (= 1.22.0)
+      json (= 2.2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -26,7 +26,7 @@ GEM
     dig_rb (1.0.1)
     faraday (0.15.4)
       multipart-post (>= 1.2, < 3)
-    fluentd (1.4.2)
+    fluentd (1.6.3)
       cool.io (>= 1.4.5, < 2.0.0)
       dig_rb (~> 1.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
@@ -37,11 +37,11 @@ GEM
       tzinfo (~> 1.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
-    google-api-client (0.28.4)
+    google-api-client (0.30.8)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (>= 0.5, < 0.10.0)
       httpclient (>= 2.8.1, < 3.0)
-      mime-types (~> 3.0)
+      mini_mime (~> 1.0)
       representable (~> 3.0)
       retriable (>= 2.0, < 4.0)
       signet (~> 0.10)
@@ -49,10 +49,10 @@ GEM
       google-cloud-env (~> 1.0)
     google-cloud-env (1.2.0)
       faraday (~> 0.11)
-    google-cloud-logging (1.6.0)
-      concurrent-ruby (~> 1.0)
+    google-cloud-logging (1.6.6)
+      concurrent-ruby (~> 1.1)
       google-cloud-core (~> 1.2)
-      google-gax (~> 1.3)
+      google-gax (~> 1.7)
       googleapis-common-protos-types (>= 1.0.2)
       stackdriver-core (~> 1.3)
     google-gax (1.7.0)
@@ -61,45 +61,43 @@ GEM
       googleauth (>= 0.6.2, < 0.10.0)
       grpc (>= 1.7.2, < 2.0)
       rly (~> 0.2.3)
-    google-protobuf (3.6.1)
-    googleapis-common-protos (1.3.7)
+    google-protobuf (3.9.0)
+    googleapis-common-protos (1.3.9)
       google-protobuf (~> 3.0)
       googleapis-common-protos-types (~> 1.0)
       grpc (~> 1.0)
     googleapis-common-protos-types (1.0.4)
       google-protobuf (~> 3.0)
-    googleauth (0.8.1)
+    googleauth (0.9.0)
       faraday (~> 0.12)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.7)
-    grpc (1.14.2)
-      google-protobuf (~> 3.1)
-      googleapis-common-protos-types (~> 1.0.0)
-    hashdiff (0.4.0)
+    grpc (1.22.0)
+      google-protobuf (~> 3.8)
+      googleapis-common-protos-types (~> 1.0)
+    hashdiff (1.0.0)
     http_parser.rb (0.6.0)
     httpclient (2.8.3)
-    json (2.1.0)
+    json (2.2.0)
     jwt (2.2.1)
     memoist (0.16.0)
     metaclass (0.0.4)
-    mime-types (3.2.2)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.0331)
+    mini_mime (1.0.2)
     mocha (1.9.0)
       metaclass (~> 0.0.1)
-    msgpack (1.3.0)
+    msgpack (1.3.1)
     multi_json (1.13.1)
     multipart-post (2.1.1)
     os (1.0.1)
     parser (2.6.3.0)
       ast (~> 2.4.0)
-    power_assert (1.1.4)
+    power_assert (1.1.5)
     powerpack (0.1.2)
-    prometheus-client (0.7.1)
-      quantile (~> 0.2.0)
+    prometheus-client (0.9.0)
+      quantile (~> 0.2.1)
     public_suffix (3.1.1)
     quantile (0.2.1)
     rainbow (2.2.2)
@@ -139,10 +137,10 @@ GEM
       tzinfo (>= 1.0.0)
     uber (0.1.0)
     unicode-display_width (1.6.0)
-    webmock (2.3.2)
+    webmock (3.6.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
-      hashdiff
+      hashdiff (>= 0.4.0, < 2.0.0)
     yajl-ruby (1.4.1)
 
 PLATFORMS
@@ -150,12 +148,12 @@ PLATFORMS
 
 DEPENDENCIES
   fluent-plugin-google-cloud!
-  mocha (~> 1.1)
-  prometheus-client (~> 0.7.1)
-  rake (~> 10.3)
-  rubocop (~> 0.39.0)
-  test-unit (~> 3.0)
-  webmock (~> 2.3.1)
+  mocha (= 1.9.0)
+  prometheus-client (= 0.9.0)
+  rake (= 10.5.0)
+  rubocop (= 0.39.0)
+  test-unit (= 3.3.3)
+  webmock (= 3.6.2)
 
 BUNDLED WITH
    1.16.6

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.7.15'
+  gem.version       = '0.7.16'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
@@ -19,19 +19,24 @@ eos
   gem.test_files    = gem.files.grep(/^(test)/)
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency 'fluentd', '1.4.2'
-  gem.add_runtime_dependency 'googleapis-common-protos', '1.3.7'
-  gem.add_runtime_dependency 'googleauth', '0.8.1'
-  gem.add_runtime_dependency 'google-api-client', '0.28.4'
-  gem.add_runtime_dependency 'google-cloud-logging', '1.6.0'
-  gem.add_runtime_dependency 'google-protobuf', '3.6.1'
-  gem.add_runtime_dependency 'grpc', '1.14.2'
-  gem.add_runtime_dependency 'json', '2.1.0'
+  gem.add_runtime_dependency 'fluentd', '1.6.3'
+  gem.add_runtime_dependency 'googleapis-common-protos', '1.3.9'
+  gem.add_runtime_dependency 'googleauth', '0.9.0'
+  gem.add_runtime_dependency 'google-api-client', '0.30.8'
+  gem.add_runtime_dependency 'google-cloud-logging', '1.6.6'
+  gem.add_runtime_dependency 'google-protobuf', '3.9.0'
+  gem.add_runtime_dependency 'grpc', '1.22.0'
+  gem.add_runtime_dependency 'json', '2.2.0'
 
-  gem.add_development_dependency 'mocha', '~> 1.1'
-  gem.add_development_dependency 'prometheus-client', '~> 0.7.1'
-  gem.add_development_dependency 'rake', '~> 10.3'
-  gem.add_development_dependency 'rubocop', '~> 0.39.0'
-  gem.add_development_dependency 'test-unit', '~> 3.0'
-  gem.add_development_dependency 'webmock', '~> 2.3.1'
+  gem.add_development_dependency 'mocha', '1.9.0'
+  gem.add_development_dependency 'prometheus-client', '0.9.0'
+  # TODO(qingling128): Upgrade rake to 11.0+ after the following issues are
+  # fixed because rake (11.0+) requires ALL variables to be explicitly
+  # initialized.
+  # https://github.com/googleapis/google-auth-library-ruby/issues/227
+  # https://github.com/farcaller/rly/issues/2
+  gem.add_development_dependency 'rake', '10.5.0'
+  gem.add_development_dependency 'rubocop', '0.39.0'
+  gem.add_development_dependency 'test-unit', '3.3.3'
+  gem.add_development_dependency 'webmock', '3.6.2'
 end

data/lib/fluent/plugin/monitoring.rb

@@ -16,7 +16,7 @@ module Monitoring
   # Base class for the monitoring registry.
   class BaseMonitoringRegistry
     def counter(_name, _desc)
-      _undefined
+      nil
     end
   end
 
@@ -48,6 +48,10 @@ module Monitoring
         PrometheusMonitoringRegistry
     }
 
+    def self.supports_monitoring_type(name)
+      @known_registry_types.key?(name)
+    end
+
     def self.create(name)
       (@known_registry_types[name] || BaseMonitoringRegistry).new
     end

data/lib/fluent/plugin/out_google_cloud.rb

@@ -36,6 +36,7 @@ module Google
     # Alias the has_key? method to have the same interface as a regular map.
     class Map
       alias key? has_key?
+      alias to_hash to_h
     end
   end
 end
@@ -460,6 +461,14 @@ module Fluent
       super
       # use the global logger
       @log = $log # rubocop:disable Style/GlobalVars
+
+      @failed_requests_count = nil
+      @successful_requests_count = nil
+      @dropped_entries_count = nil
+      @ingested_entries_count = nil
+      @retried_entries_count = nil
+
+      @ok_code = nil
     end
 
     def configure(conf)
@@ -491,6 +500,11 @@ module Fluent
       # If monitoring is enabled, register metrics in the default registry
       # and store metric objects for future use.
       if @enable_monitoring
+        unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
+          @monitoring_type)
+          @log.warn "monitoring_type '#{@monitoring_type}' is unknown; "\
+                    'there will be no metrics'
+        end
         registry = Monitoring::MonitoringRegistryFactory.create @monitoring_type
         @successful_requests_count = registry.counter(
           :stackdriver_successful_requests_count,
@@ -592,7 +606,7 @@ module Fluent
                       @statusz_port,
                       bind: '127.0.0.1') do |data, conn|
           if data.split(' ')[1] == '/statusz'
-            write_html_response(data, conn, 200, Statusz.response)
+            write_html_response(data, conn, 200, Statusz.response(self))
           else
             write_html_response(data, conn, 404, "Not found\n")
           end
@@ -624,8 +638,14 @@ module Fluent
             # Save the following fields if available, then clear them out to
             # allow for determining whether we should parse the log or message
             # field.
+            # This list should be in sync with
+            # https://cloud.google.com/logging/docs/agent/configuration#special-fields.
             preserved_keys = [
               'time',
+              'timeNanos',
+              'timestamp',
+              'timestampNanos',
+              'timestampSeconds',
               'severity',
               @http_request_key,
               @insert_id_key,
@@ -1031,7 +1051,7 @@ module Fluent
       end
 
       begin
-        open('http://' + METADATA_SERVICE_ADDR) do |f|
+        open('http://' + METADATA_SERVICE_ADDR, proxy: false) do |f|
           if f.meta['metadata-flavor'] == 'Google'
             @log.info 'Detected GCE platform'
             return Platform::GCE
@@ -1054,7 +1074,8 @@ module Fluent
         @platform == Platform::GCE
       # See https://cloud.google.com/compute/docs/metadata
       open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
-           metadata_path, 'Metadata-Flavor' => 'Google', &:read)
+           metadata_path, 'Metadata-Flavor' => 'Google', :proxy => false,
+           &:read)
     end
 
     # EC2 Metadata server returns everything in one call. Store it after the
@@ -1065,7 +1086,7 @@ module Fluent
       unless @ec2_metadata
         # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
         open('http://' + METADATA_SERVICE_ADDR +
-             '/latest/dynamic/instance-identity/document') do |f|
+             '/latest/dynamic/instance-identity/document', proxy: false) do |f|
           contents = f.read
           @ec2_metadata = JSON.parse(contents)
         end
@@ -2242,7 +2263,6 @@ module Fluent
     def construct_error_details_map_grpc(gax_error)
       return {} unless @partial_success
       error_details_map = Hash.new { |h, k| h[k] = [] }
-
       error_details = ensure_array(gax_error.status_details)
       raise JSON::ParserError, 'The error details are empty.' if
         error_details.empty?
@@ -2330,10 +2350,12 @@ module Fluent
       constructed_resource
     end
 
+    # Convert the value to a Ruby array.
     def ensure_array(value)
       Array.try_convert(value) || (raise JSON::ParserError, value.class.to_s)
     end
 
+    # Convert the value to a Ruby hash.
     def ensure_hash(value)
       Hash.try_convert(value) || (raise JSON::ParserError, value.class.to_s)
     end

data/lib/fluent/plugin/statusz.rb

@@ -12,20 +12,116 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require 'erb'
+
 # Module for collecting diagnostic information and formatting it as an
 # HTML page to serve on the /statusz endpoint.
 module Statusz
   module_function
 
-  def response
-    # TODO(davidbtucker): Add more status information here.
-    [
-      '<html>',
-      '<body>',
-      '<h1>Status</h1>',
-      "<b>Command-line flags:</b> #{CGI.escapeHTML(ARGV.join(' '))}",
-      '</body>',
-      '</html>'
-    ].join("\n") + "\n"
+  # Note: The plugin parameter is referenced in STATUSZ_TMPL.
+  def response(plugin)
+    uptime = Time.now - SERVER_START
+    uptime_str = format('%d hr %02d min %02d sec',
+                        uptime / 3600,
+                        (uptime / 60) % 60,
+                        uptime % 60)
+    ERB.new(STATUSZ_TMPL).result(binding)
   end
 end
+
+SERVER_START = Time.now
+
+# Does not include the following deprecated config params:
+# auth_method, private_key_email, private_key_passphrase, private_key_path
+CONFIG_KEYS = %w(
+  adjust_invalid_timestamps
+  autoformat_stackdriver_trace
+  coerce_to_utf8
+  detect_json
+  detect_subservice
+  enable_metadata_agent
+  enable_monitoring
+  http_request_key
+  insert_id_key
+  k8s_cluster_location
+  k8s_cluster_name
+  kubernetes_tag_regexp
+  label_map
+  labels_key
+  labels
+  logging_api_url
+  metadata_agent_url
+  monitoring_type
+  non_utf8_replacement_string
+  operation_key
+  partial_success
+  project_id
+  require_valid_tags
+  source_location_key
+  span_id_key
+  split_logs_by_tag
+  statusz_port
+  subservice_name
+  trace_key
+  trace_sampled_key
+  use_aws_availability_zone
+  use_grpc
+  use_metadata_service
+  vm_id
+  vm_name
+  zone
+).freeze
+
+# rubocop:disable LineLength
+STATUSZ_TMPL = %(\
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Status for <%= File.basename($PROGRAM_NAME) %></title>
+    <style>
+      body {
+        font-family: sans-serif;
+      }
+      h1 {
+        clear: both;
+        width: 100%;
+        text-align: center;
+        font-size: 120%;
+        background: #eef;
+      }
+      .lefthand {
+        float: left;
+        width: 80%;
+      }
+      .righthand {
+        text-align: right;
+      }
+      td, th {
+        background-color: rgba(0, 0, 0, 0.05);
+      }
+      th {
+        text-align: left;
+      }
+    </style>
+  </head>
+
+  <body>
+    <h1>Status for <%= File.basename($PROGRAM_NAME) %></h1>
+
+    <div>
+      <div class="lefthand">
+        Started: <%= SERVER_START %><br>
+        Up <%= uptime_str %><br>
+      </div>
+    </div>
+
+    <h1>Parsed configuration</h1>
+
+    <table>
+#{CONFIG_KEYS.map { |k| "      <tr><th>#{k}</th><td><%= plugin.#{k} %></td></tr>" }.join("\n")}
+    </table>
+  </body>
+</html>
+).freeze
+# rubocop:enable LineLength