fluent-plugin-google-cloud 0.6.8.pre.1 → 0.6.8.pre.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f013270937ffeb8ac1addeb0b50cbd04b9434d8f
-  data.tar.gz: a19be3b8f5b49266e6055be20c6a8c693fae1fea
+  metadata.gz: 5f2c640c0609198df6d95e33e06d442806ecb5f2
+  data.tar.gz: 842c10c9b0d881cec2bfcd272b13ac3e7ab915b6
 SHA512:
-  metadata.gz: cfc74c7b17e028776423d30fbc438d1d5d146d532991bb2331044a35f54e1a6acfd6f14e6bc4b234f924ac5e6d62b268931b8c856d07532b9c610c85579f3906
-  data.tar.gz: bb14fdffce9b35b5c1ff4d9609d5ec473d19d69b74972b3ba20bc843d28e1118be19b89b112b961d7b903e246612eca551780c3e9c202fa852ad8bdec6f50f2d
+  metadata.gz: d1e3a20210418858232b925e2d35255e78564170d0d8f391c20d7d8700bf4d3a52bec63e8c82fa02ff5fccbbad117097672637b30803d1e5b3348f7ac1b248ff
+  data.tar.gz: e9787a02f9be79bbc2d37594d3718446d8c7a197455201b2c772fd36223783c9bbe5a77e4c69b387b1dbc8e30afd739d88c001281f759e62f033c0eedd3e3431

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.6.8.pre.1)
+    fluent-plugin-google-cloud (0.6.8.pre.2)
       fluentd (~> 0.10)
       google-api-client (~> 0.9.0)
       google-cloud-logging (= 0.24.1)
@@ -23,7 +23,7 @@ GEM
       safe_yaml (~> 1.0.0)
     faraday (0.13.1)
       multipart-post (>= 1.2, < 3)
-    fluentd (0.14.20)
+    fluentd (0.14.21)
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       msgpack (>= 0.7.0, < 2.0.0)
@@ -49,16 +49,19 @@ GEM
       google-cloud-core (~> 0.21.1)
       google-gax (~> 0.8.0)
       stackdriver-core (~> 0.21.0)
-    google-gax (0.8.6)
+    google-gax (0.8.7)
       google-protobuf (~> 3.2)
       googleapis-common-protos (~> 1.3.5)
       googleauth (~> 0.5.1)
       grpc (~> 1.0)
       rly (~> 0.2.3)
-    google-protobuf (3.4.0.2)
-    googleapis-common-protos (1.3.5)
-      google-protobuf (~> 3.2)
+    google-protobuf (3.4.1.1-x86_64-linux)
+    googleapis-common-protos (1.3.6)
+      google-protobuf (~> 3.0)
+      googleapis-common-protos-types (~> 1.0)
       grpc (~> 1.0)
+    googleapis-common-protos-types (1.0.0)
+      google-protobuf (~> 3.0)
     googleauth (0.5.1)
       faraday (~> 0.9)
       jwt (~> 1.4)
@@ -67,7 +70,7 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
-    grpc (1.2.5)
+    grpc (1.2.5-x86_64-linux)
       google-protobuf (~> 3.1)
       googleauth (~> 0.5.1)
     hashdiff (0.3.6)
@@ -113,7 +116,7 @@ GEM
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.7)
       tins (<= 1.6.0)
-    ruby-progressbar (1.8.1)
+    ruby-progressbar (1.8.3)
     ruby_dig (0.0.2)
     safe_yaml (1.0.4)
     serverengine (2.0.5)
@@ -126,7 +129,7 @@ GEM
       multi_json (~> 1.10)
     stackdriver-core (0.21.0)
     strptime (0.1.9)
-    test-unit (3.2.5)
+    test-unit (3.2.6)
       power_assert
     thread_safe (0.3.6)
     tins (1.6.0)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.8.pre.1'
+  gem.version       = '0.6.8.pre.2'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/lib/fluent/plugin/out_google_cloud.rb

@@ -114,10 +114,6 @@ module Fluent
 
     # Constants for log entry field extraction.
     module InternalConstants
-      # Use empty string as request path when the local_resource_id of monitored
-      # resource can be implicitly inferred by Metadata Agent.
-      IMPLICIT_LOCAL_RESOURCE_ID = ''
-
       # The label name of local_resource_id in the json payload. When a record
       # has this field in the payload, we will use the value to retrieve
       # monitored resource from Stackdriver Metadata agent.
@@ -181,7 +177,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
-    PLUGIN_VERSION = '0.6.8.pre.1'
+    PLUGIN_VERSION = '0.6.8.pre.2'
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
@@ -359,6 +355,11 @@ module Fluent
         @dropped_entries_count = registry.counter(
           :stackdriver_dropped_entries_count,
           'A number of log entries dropped by the Stackdriver output plugin')
+        @retried_entries_count = registry.counter(
+          :stackdriver_retried_entries_count,
+          'The number of log entries that failed to be ingested by the'\
+            ' Stackdriver output plugin due to a transient error and were'\
+            ' retried')
       end
 
       # Alert on old authentication configuration.
@@ -380,19 +381,6 @@ module Fluent
 
       @platform = detect_platform
 
-      # Set agent-level monitored resource. This monitored resource is initiated
-      # as the logging agent starts up. It will be inherited by all log entries
-      # processed by this agent. First try to retrieve it via Metadata Agent.
-      if @enable_metadata_agent
-        # The local_resource_id for this should be the instance id. Since this
-        # can be implicitly inferred by Metadata Agent, we do not need to
-        # explicitly send the key.
-        # TODO(qingling128): Remove this logic once the resource is retrieved at
-        # a proper time (b/65175256).
-        @resource = query_metadata_agent_for_monitored_resource(
-          IMPLICIT_LOCAL_RESOURCE_ID)
-      end
-
       # Set required variables: @project_id, @vm_id, @vm_name and @zone.
       set_required_metadata_variables
 
@@ -575,9 +563,10 @@ module Fluent
               entries: entries
             )
 
+            entries_count = entries.length
             client.write_log_entries(write_request)
             increment_successful_requests_count
-            increment_ingested_entries_count(entries.length)
+            increment_ingested_entries_count(entries_count)
 
             # Let the user explicitly know when the first call succeeded, to aid
             # with verification and troubleshooting.
@@ -588,7 +577,10 @@ module Fluent
 
           rescue GRPC::Cancelled => error
             increment_failed_requests_count(GRPC::Core::StatusCodes::CANCELLED)
+            increment_retried_entries_count(entries_count, error.code)
             # RPC cancelled, so retry via re-raising the error.
+            @log.debug "Retrying #{entries_count} log message(s) later.",
+                       error: error.to_s, error_code: error.code.to_s
             raise error
 
           rescue GRPC::BadStatus => error
@@ -599,32 +591,31 @@ module Fluent
                  GRPC::Core::StatusCodes::DEADLINE_EXCEEDED,
                  GRPC::Core::StatusCodes::INTERNAL,
                  GRPC::Core::StatusCodes::UNKNOWN
-              # TODO
               # Server error, so retry via re-raising the error.
+              increment_retried_entries_count(entries.length, error.code)
+              @log.debug "Retrying #{entries_count} log message(s) later.",
+                         error: error.to_s, error_code: error.code.to_s
               raise error
             when GRPC::Core::StatusCodes::UNIMPLEMENTED,
                  GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED
               # Most client errors indicate a problem with the request itself
               # and should not be retried.
-              dropped = entries.length
-              increment_dropped_entries_count(dropped)
-              @log.warn "Dropping #{dropped} log message(s)",
+              increment_dropped_entries_count(entries_count)
+              @log.warn "Dropping #{entries_count} log message(s)",
                         error: error.to_s, error_code: error.code.to_s
             when GRPC::Core::StatusCodes::UNAUTHENTICATED
               # Authorization error.
               # These are usually solved via a `gcloud auth` call, or by
               # modifying the permissions on the Google Cloud project.
-              dropped = entries.length
-              increment_dropped_entries_count(dropped)
-              @log.warn "Dropping #{dropped} log message(s)",
+              increment_dropped_entries_count(entries_count)
+              @log.warn "Dropping #{entries_count} log message(s)",
                         error: error.to_s, error_code: error.code.to_s
             else
               # Assume this is a problem with the request itself and don't
               # retry.
-              dropped = entries.length
-              increment_dropped_entries_count(dropped)
+              increment_dropped_entries_count(entries_count)
               @log.error "Unknown response code #{error.code} from the "\
-                         "server, dropping #{dropped} log message(s)",
+                         "server, dropping #{entries_count} log message(s)",
                          error: error.to_s, error_code: error.code.to_s
             end
           end
@@ -636,6 +627,7 @@ module Fluent
                 resource: group_level_resource,
                 labels: group_level_common_labels,
                 entries: entries)
+            entries_count = entries.length
 
             # TODO: RequestOptions
             begin
@@ -645,7 +637,7 @@ module Fluent
               raise error
             end
             increment_successful_requests_count
-            increment_ingested_entries_count(entries.length)
+            increment_ingested_entries_count(entries_count)
 
             # Let the user explicitly know when the first call succeeded, to aid
             # with verification and troubleshooting.
@@ -656,23 +648,24 @@ module Fluent
 
           rescue Google::Apis::ServerError => error
             # Server error, so retry via re-raising the error.
+            increment_retried_entries_count(entries.length, error.status_code)
+            @log.debug "Retrying #{entries_count} log message(s) later.",
+                       error: error.to_s, error_code: error.status_code.to_s
             raise error
 
           rescue Google::Apis::AuthorizationError => error
             # Authorization error.
             # These are usually solved via a `gcloud auth` call, or by modifying
             # the permissions on the Google Cloud project.
-            dropped = entries.length
-            increment_dropped_entries_count(dropped)
-            @log.warn "Dropping #{dropped} log message(s)",
+            increment_dropped_entries_count(entries_count)
+            @log.warn "Dropping #{entries_count} log message(s)",
                       error_class: error.class.to_s, error: error.to_s
 
           rescue Google::Apis::ClientError => error
             # Most ClientErrors indicate a problem with the request itself and
             # should not be retried.
-            dropped = entries.length
-            increment_dropped_entries_count(dropped)
-            @log.warn "Dropping #{dropped} log message(s)",
+            increment_dropped_entries_count(entries_count)
+            @log.warn "Dropping #{entries_count} log message(s)",
                       error_class: error.class.to_s, error: error.to_s
           end
         end
@@ -1186,7 +1179,7 @@ module Fluent
       begin
         resource = Google::Api::MonitoredResource.decode_json(response.to_json)
       rescue Google::Protobuf::ParseError, ArgumentError => e
-        @log.error 'Error paring monitored resource from Metadata Agent. ' \
+        @log.error 'Error parsing monitored resource from Metadata Agent. ' \
                    "response: #{response.inspect}", error: e
         return nil
       end
@@ -1743,6 +1736,13 @@ module Fluent
       return unless @dropped_entries_count
       @dropped_entries_count.increment({}, count)
     end
+
+    # Increment the metric for the number of log entries that were dropped
+    # and not ingested by the Stackdriver Logging API.
+    def increment_retried_entries_count(count, code)
+      return unless @retried_entries_count
+      @retried_entries_count.increment({ code: code }, count)
+    end
   end
 end
 

data/test/plugin/base_test.rb

@@ -1123,25 +1123,6 @@ module BaseTest
       setup_metadata_agent_stubs
       d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
       assert_true d.instance.instance_variable_get(:@enable_metadata_agent)
-      assert_requested_metadata_agent_stub(IMPLICIT_LOCAL_RESOURCE_ID)
-    end
-  end
-
-  # Test that an implicit monitored resource can be retrieved from Metadata
-  # Agent with an empty string as the local_resource_id.
-  def test_retrieve_implicit_monitored_resource
-    # GCE metadata stubs has VM_ID and ZONE, while the Metadata Agent stub has
-    # METADATA_VM_ID and METADATA_ZONE.
-    new_stub_context do
-      setup_gce_metadata_stubs
-      setup_metadata_agent_stubs
-      setup_logging_stubs do
-        d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
-        d.emit('message' => log_entry(0))
-        d.run
-      end
-      verify_log_entries(1, COMPUTE_PARAMS_WITH_METADATA_VM_ID_AND_ZONE)
-      assert_requested_metadata_agent_stub(IMPLICIT_LOCAL_RESOURCE_ID)
     end
   end
 
@@ -1161,7 +1142,6 @@ module BaseTest
           d.run
         end
         verify_log_entries(n, DOCKER_CONTAINER_PARAMS)
-        assert_requested_metadata_agent_stub(IMPLICIT_LOCAL_RESOURCE_ID)
         assert_requested_metadata_agent_stub("container.#{DOCKER_CONTAINER_ID}")
       end
     end
@@ -1191,7 +1171,6 @@ module BaseTest
           assert_equal 'test', get_string(fields['tag2']), entry
           assert_equal 5000, get_number(fields['data']), entry
         end
-        assert_requested_metadata_agent_stub(IMPLICIT_LOCAL_RESOURCE_ID)
         assert_requested_metadata_agent_stub("container.#{DOCKER_CONTAINER_ID}")
       end
     end
@@ -1212,7 +1191,6 @@ module BaseTest
         d.run
       end
       verify_log_entries(1, DOCKER_CONTAINER_PARAMS_NO_STREAM)
-      assert_requested_metadata_agent_stub(IMPLICIT_LOCAL_RESOURCE_ID)
       assert_requested_metadata_agent_stub(
         "containerName.#{DOCKER_CONTAINER_NAME}")
     end
@@ -1237,7 +1215,6 @@ module BaseTest
         assert_equal DOCKER_CONTAINER_NANOS,
                      entry['timestamp']['nanos'], entry
       end
-      assert_requested_metadata_agent_stub(IMPLICIT_LOCAL_RESOURCE_ID)
       assert_requested_metadata_agent_stub(
         "containerName.#{DOCKER_CONTAINER_NAME}")
     end
@@ -1289,7 +1266,6 @@ module BaseTest
           d.run
         end
         verify_log_entries(n, CONTAINER_FROM_APPLICATION_PARAMS)
-        assert_requested_metadata_agent_stub(IMPLICIT_LOCAL_RESOURCE_ID)
         assert_requested_metadata_agent_stub(
           "gke_containerName.#{CONTAINER_NAMESPACE_ID}.#{CONTAINER_POD_NAME}." \
           "#{CONTAINER_CONTAINER_NAME}")

data/test/plugin/constants.rb

@@ -572,15 +572,6 @@ module Constants
   # Stub value for Monitored resources from Metadata Agent.
   # Map from the local_resource_id to the retrieved monitored resource.
   MONITORED_RESOURCE_STUBS = {
-    # Implicit GCE instance.
-    IMPLICIT_LOCAL_RESOURCE_ID =>
-      {
-        'type' => COMPUTE_CONSTANTS[:resource_type],
-        'labels' => {
-          'zone' => METADATA_ZONE,
-          'instance_id' => METADATA_VM_ID
-        }
-      }.to_json,
     # Docker container stderr / stdout logs.
     "container.#{DOCKER_CONTAINER_ID}" =>
       {

data/test/plugin/test_out_google_cloud.rb

@@ -77,15 +77,16 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     setup_gce_metadata_stubs
     [
       # Single successful request.
-      [200, 1, 1, [1, 0, 1, 0]],
+      [200, 1, 1, [1, 0, 1, 0, 0]],
       # Several successful requests.
-      [200, 2, 1, [2, 0, 2, 0]],
+      [200, 2, 1, [2, 0, 2, 0, 0]],
       # Single successful request with several entries.
-      [200, 1, 2, [1, 0, 2, 0]],
+      [200, 1, 2, [1, 0, 2, 0, 0]],
       # Single failed request that causes logs to be dropped.
-      [401, 1, 1, [0, 1, 0, 1]],
-      # Single failed request that escalates without logs being dropped.
-      [500, 1, 1, [0, 1, 0, 0]]
+      [401, 1, 1, [0, 1, 0, 1, 0]],
+      # Single failed request that escalates without logs being dropped with
+      # several entries.
+      [500, 1, 2, [0, 1, 0, 0, 2]]
     ].each do |code, request_count, entry_count, metric_values|
       setup_prometheus
       # TODO: Do this as part of setup_logging_stubs.
@@ -105,7 +106,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
         # rubocop:enable Lint/HandleExceptions
       end
       successful_requests_count, failed_requests_count,
-        ingested_entries_count, dropped_entries_count = metric_values
+        ingested_entries_count, dropped_entries_count,
+        retried_entries_count = metric_values
       assert_prometheus_metric_value(:stackdriver_successful_requests_count,
                                      successful_requests_count, grpc: false)
       assert_prometheus_metric_value(:stackdriver_failed_requests_count,
@@ -115,6 +117,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
                                      ingested_entries_count)
       assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
                                      dropped_entries_count)
+      assert_prometheus_metric_value(:stackdriver_retried_entries_count,
+                                     retried_entries_count, code: code)
     end
   end
 

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -79,15 +79,16 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     setup_gce_metadata_stubs
     [
       # Single successful request.
-      [false, 0, 1, 1, [1, 0, 1, 0]],
+      [false, 0, 1, 1, [1, 0, 1, 0, 0]],
       # Several successful requests.
-      [false, 0, 2, 1, [2, 0, 2, 0]],
+      [false, 0, 2, 1, [2, 0, 2, 0, 0]],
       # Single successful request with several entries.
-      [false, 0, 1, 2, [1, 0, 2, 0]],
+      [false, 0, 1, 2, [1, 0, 2, 0, 0]],
       # Single failed request that causes logs to be dropped.
-      [true, 16, 1, 1, [0, 1, 0, 1]],
-      # Single failed request that escalates without logs being dropped.
-      [true, 13, 1, 1, [0, 1, 0, 0]]
+      [true, 16, 1, 1, [0, 1, 0, 1, 0]],
+      # Single failed request that escalates without logs being dropped with
+      # several entries.
+      [true, 13, 1, 2, [0, 1, 0, 0, 2]]
     ].each do |should_fail, code, request_count, entry_count, metric_values|
       setup_prometheus
       setup_logging_stubs(should_fail, code, 'SomeMessage') do
@@ -106,7 +107,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
         end
       end
       successful_requests_count, failed_requests_count,
-        ingested_entries_count, dropped_entries_count = metric_values
+        ingested_entries_count, dropped_entries_count,
+        retried_entries_count = metric_values
       assert_prometheus_metric_value(:stackdriver_successful_requests_count,
                                      successful_requests_count, grpc: true)
       assert_prometheus_metric_value(:stackdriver_failed_requests_count,
@@ -116,6 +118,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
                                      ingested_entries_count)
       assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
                                      dropped_entries_count)
+      assert_prometheus_metric_value(:stackdriver_retried_entries_count,
+                                     retried_entries_count, code: code)
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.8.pre.1
+  version: 0.6.8.pre.2
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-06 00:00:00.000000000 Z
+date: 2017-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd