fluent-plugin-google-cloud 0.6.10 → 0.6.11.pre.memory.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d1d9303278baab1ded1484eaa6c2974c5dbb3bc5
-  data.tar.gz: 313c3e168ab212dd65e276b0e0a3562e718fcaa1
+  metadata.gz: fadfeb3397203502f2708012a6019ca93fbf4b28
+  data.tar.gz: d2ebfbc912f30cd1caa3120fe1d282570e98da6e
 SHA512:
-  metadata.gz: 2a67095d29a49fb9b264a23f78949d98f28614428381ae4f85b886fb14d5b7d34d4974649ca8ab1c740dbc172e2e036ab64f2b788d4818d94dc3d50ba0529a8a
-  data.tar.gz: 3ec89701d9d83e380e90eb06f4b3d9a25a7d7b223420a1384dd07b110af43c8fb4aeccd59bdfaffda4140ba74bfc4b0d5804d4dc74b627b7527d8981205a7bc5
+  metadata.gz: 5884686ae0eae5dad3f533d5399b171029bc03352b251c1690c20425071c141eddb34449bebcb8fa486d5fd6c6752ffe9dd207ec47889f7eed812781d01c9d12
+  data.tar.gz: feffa60a8bf9b93d83b9ec8762e12f74e6c2934fe57a4d10d219410bf8618ce97bc5c14a2deae8e20dbb5a1f0697f1bfec0f167b7cb9b1a2bbf61b2a916272a4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.6.10)
+    fluent-plugin-google-cloud (0.6.11.pre.memory.1)
       fluentd (~> 0.10)
       google-api-client (~> 0.14)
       google-cloud-logging (~> 1.2, >= 1.2.3)
@@ -16,14 +16,14 @@ GEM
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
     ast (2.3.0)
-    cool.io (1.5.1)
+    cool.io (1.5.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     declarative (0.0.10)
     declarative-option (0.1.0)
     faraday (0.13.1)
       multipart-post (>= 1.2, < 3)
-    fluentd (0.14.23)
+    fluentd (0.14.25)
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       msgpack (>= 0.7.0, < 2.0.0)
@@ -34,7 +34,7 @@ GEM
       tzinfo (~> 1.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
-    google-api-client (0.17.1)
+    google-api-client (0.17.4)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (>= 0.5, < 0.7.0)
       httpclient (>= 2.8.1, < 3.0)
@@ -45,11 +45,11 @@ GEM
       google-cloud-env (~> 1.0)
     google-cloud-env (1.0.1)
       faraday (~> 0.11)
-    google-cloud-logging (1.3.1)
+    google-cloud-logging (1.3.2)
       google-cloud-core (~> 1.1)
       google-gax (~> 0.10.1)
       stackdriver-core (~> 1.2)
-    google-gax (0.10.1)
+    google-gax (0.10.2)
       google-protobuf (~> 3.2)
       googleapis-common-protos (>= 1.3.5, < 2.0)
       googleauth (~> 0.6.2)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.10'
+  gem.version       = '0.6.11.pre.memory.1'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/lib/fluent/plugin/out_google_cloud.rb

@@ -20,6 +20,8 @@ require 'time'
 require 'yaml'
 require 'google/apis'
 require 'google/apis/logging_v2'
+require 'google/cloud/logging/v2'
+require 'google/gax'
 require 'google/logging/v2/logging_pb'
 require 'google/logging/v2/logging_services_pb'
 require 'google/logging/v2/log_entry_pb'
@@ -193,7 +195,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
-    PLUGIN_VERSION = '0.6.10'.freeze
+    PLUGIN_VERSION = '0.6.11.pre.memory.1'.freeze
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
@@ -391,6 +393,7 @@ module Fluent
           'The number of log entries that failed to be ingested by the'\
             ' Stackdriver output plugin due to a transient error and were'\
             ' retried')
+        @ok_code = @use_grpc ? 0 : 200
       end
 
       # Alert on old authentication configuration.
@@ -514,17 +517,6 @@ module Fluent
           severity = compute_severity(
             entry_level_resource.type, record, entry_level_common_labels)
 
-          ts_secs = begin
-                      Integer ts_secs
-                    rescue ArgumentError, TypeError
-                      ts_secs
-                    end
-          ts_nanos = begin
-                       Integer ts_nanos
-                     rescue ArgumentError, TypeError
-                       ts_nanos
-                     end
-
           if @use_grpc
             entry = Google::Logging::V2::LogEntry.new(
               labels: entry_level_common_labels,
@@ -583,18 +575,16 @@ module Fluent
               [k.encode('utf-8'), convert_to_utf8(v)]
             end
 
-            write_request = Google::Logging::V2::WriteLogEntriesRequest.new(
+            entries_count = entries.length
+            client.write_log_entries(
+              entries,
               log_name: log_name,
               resource: Google::Api::MonitoredResource.new(
                 type: group_level_resource.type,
                 labels: group_level_resource.labels.to_h
               ),
-              labels: labels_utf8_pairs.to_h,
-              entries: entries
+              labels: labels_utf8_pairs.to_h
             )
-
-            entries_count = entries.length
-            client.write_log_entries(write_request)
             increment_successful_requests_count
             increment_ingested_entries_count(entries_count)
 
@@ -605,45 +595,62 @@ module Fluent
               @log.info 'Successfully sent gRPC to Stackdriver Logging API.'
             end
 
-          rescue GRPC::Cancelled => error
-            increment_failed_requests_count(GRPC::Core::StatusCodes::CANCELLED)
-            increment_retried_entries_count(entries_count, error.code)
-            # RPC cancelled, so retry via re-raising the error.
-            @log.debug "Retrying #{entries_count} log message(s) later.",
-                       error: error.to_s, error_code: error.code.to_s
-            raise error
-
-          rescue GRPC::BadStatus => error
+          rescue Google::Gax::GaxError => gax_error
+            # GRPC::BadStatus is wrapped in error.cause.
+            error = gax_error.cause
             increment_failed_requests_count(error.code)
-            case error.code
-            when GRPC::Core::StatusCodes::CANCELLED,
-                 GRPC::Core::StatusCodes::UNAVAILABLE,
-                 GRPC::Core::StatusCodes::DEADLINE_EXCEEDED,
-                 GRPC::Core::StatusCodes::INTERNAL,
-                 GRPC::Core::StatusCodes::UNKNOWN
-              # Server error, so retry via re-raising the error.
-              increment_retried_entries_count(entries.length, error.code)
+
+            # See the mapping between HTTP status and gRPC status code at:
+            # https://github.com/grpc/grpc/blob/master/src/core/lib/transport/status_conversion.cc
+            case error
+            # Server error, so retry via re-raising the error.
+            when \
+                # HTTP status 500 (Internal Server Error).
+                GRPC::Internal,
+                # HTTP status 501 (Not Implemented).
+                GRPC::Unimplemented,
+                # HTTP status 503 (Service Unavailable).
+                GRPC::Unavailable,
+                # HTTP status 504 (Gateway Timeout).
+                GRPC::DeadlineExceeded
+              increment_retried_entries_count(entries_count, error.code)
               @log.debug "Retrying #{entries_count} log message(s) later.",
                          error: error.to_s, error_code: error.code.to_s
               raise error
-            when GRPC::Core::StatusCodes::UNIMPLEMENTED,
-                 GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED
-              # Most client errors indicate a problem with the request itself
-              # and should not be retried.
-              increment_dropped_entries_count(entries_count)
-              @log.warn "Dropping #{entries_count} log message(s)",
-                        error: error.to_s, error_code: error.code.to_s
-            when GRPC::Core::StatusCodes::UNAUTHENTICATED
-              # Authorization error.
-              # These are usually solved via a `gcloud auth` call, or by
-              # modifying the permissions on the Google Cloud project.
-              increment_dropped_entries_count(entries_count)
+
+            # Most client errors indicate a problem with the request itself and
+            # should not be retried.
+            when \
+                # HTTP status 400 (Bad Request).
+                GRPC::InvalidArgument,
+                # HTTP status 401 (Unauthorized).
+                # These are usually solved via a `gcloud auth` call, or by
+                # modifying the permissions on the Google Cloud project.
+                GRPC::Unauthenticated,
+                # HTTP status 403 (Forbidden).
+                GRPC::PermissionDenied,
+                # HTTP status 404 (Not Found).
+                GRPC::NotFound,
+                # HTTP status 409 (Conflict).
+                GRPC::Aborted,
+                # HTTP status 412 (Precondition Failed).
+                GRPC::FailedPrecondition,
+                # HTTP status 429 (Too Many Requests).
+                GRPC::ResourceExhausted,
+                # HTTP status 499 (Client Closed Request).
+                GRPC::Cancelled,
+                # the remaining http codes in both 4xx and 5xx category.
+                # It's debatable whether to retry or drop these log entries.
+                # This decision is made to avoid retrying forever due to client
+                # errors.
+                GRPC::Unknown
+              increment_dropped_entries_count(entries_count, error.code)
               @log.warn "Dropping #{entries_count} log message(s)",
                         error: error.to_s, error_code: error.code.to_s
+
             else
-              # Assume this is a problem with the request itself and don't
-              # retry.
-              increment_dropped_entries_count(entries_count)
+              # Assume it is a problem with the request itself and don't retry.
+              increment_dropped_entries_count(entries_count, error.code)
               @log.error "Unknown response code #{error.code} from the "\
                          "server, dropping #{entries_count} log message(s)",
                          error: error.to_s, error_code: error.code.to_s
@@ -681,36 +688,38 @@ module Fluent
             end
 
           rescue Google::Apis::ServerError => error
-            # Server error, so retry via re-raising the error.
-            increment_retried_entries_count(entries.length, error.status_code)
+            # 5xx server errors. Retry via re-raising the error.
+            increment_retried_entries_count(entries_count, error.status_code)
             @log.debug "Retrying #{entries_count} log message(s) later.",
                        error: error.to_s, error_code: error.status_code.to_s
             raise error
 
           rescue Google::Apis::AuthorizationError => error
-            # Authorization error.
+            # 401 authorization error.
             # These are usually solved via a `gcloud auth` call, or by modifying
             # the permissions on the Google Cloud project.
-            increment_dropped_entries_count(entries_count)
+            increment_dropped_entries_count(entries_count, error.status_code)
             @log.warn "Dropping #{entries_count} log message(s)",
                       error_class: error.class.to_s, error: error.to_s
 
           rescue Google::Apis::ClientError => error
-            # Most ClientErrors indicate a problem with the request itself and
-            # should not be retried.
+            # 4xx client errors. Most client errors indicate a problem with the
+            # request itself and should not be retried.
             error_details_map = construct_error_details_map(error)
             if error_details_map.empty?
-              increment_dropped_entries_count(entries_count)
+              increment_dropped_entries_count(entries_count, error.status_code)
               @log.warn "Dropping #{entries_count} log message(s)",
                         error_class: error.class.to_s, error: error.to_s
             else
               error_details_map.each do |(error_code, error_message), indexes|
                 partial_error_count = indexes.length
-                increment_dropped_entries_count(partial_error_count)
+                increment_dropped_entries_count(partial_error_count, error_code)
+                entries_count -= partial_error_count
                 @log.warn "Dropping #{partial_error_count} log message(s)",
                           error_code: "google.rpc.Code[#{error_code}]",
                           error: error_message
               end
+              increment_ingested_entries_count(entries_count)
             end
           end
         end
@@ -839,11 +848,9 @@ module Fluent
     # 2. If not, try to retrieve it by calling metadata server directly.
     # 3. If still not set, try to obtain it from the credentials.
     def set_project_id
+      @project_id ||= CredentialsInfo.project_id
       @project_id ||= fetch_gce_metadata('project/project-id') if
         @platform == Platform::GCE
-      @project_id ||= CredentialsInfo.project_id
-    rescue StandardError => e
-      @log.error 'Failed to obtain project id: ', error: e
     end
 
     # 1. Return the value if it is explicitly set in the config already.
@@ -1312,7 +1319,14 @@ module Fluent
       instance_prefix
     end
 
+    def time_or_nil(ts_secs, ts_nanos)
+      Time.at((Integer ts_secs), (Integer ts_nanos) / 1_000.0)
+    rescue ArgumentError, TypeError
+      nil
+    end
+
     def compute_timestamp(resource_type, record, time)
+      current_time = Time.now
       if record.key?('timestamp') &&
          record['timestamp'].is_a?(Hash) &&
          record['timestamp'].key?('seconds') &&
@@ -1320,10 +1334,12 @@ module Fluent
         ts_secs = record['timestamp']['seconds']
         ts_nanos = record['timestamp']['nanos']
         record.delete('timestamp')
+        timestamp = time_or_nil(ts_secs, ts_nanos)
       elsif record.key?('timestampSeconds') &&
             record.key?('timestampNanos')
         ts_secs = record.delete('timestampSeconds')
         ts_nanos = record.delete('timestampNanos')
+        timestamp = time_or_nil(ts_secs, ts_nanos)
       elsif record.key?('timeNanos')
         # This is deprecated since the precision is insufficient.
         # Use timestampSeconds/timestampNanos instead
@@ -1336,11 +1352,13 @@ module Fluent
           @log.warn 'timeNanos is deprecated - please use ' \
             'timestampSeconds and timestampNanos instead.'
         end
+        timestamp = time_or_nil(ts_secs, ts_nanos)
       elsif resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
             @cloudfunctions_log_match
-        timestamp = DateTime.parse(@cloudfunctions_log_match['timestamp'])
-        ts_secs = timestamp.strftime('%s').to_i
-        ts_nanos = timestamp.strftime('%N').to_i
+        timestamp = DateTime.parse(
+          @cloudfunctions_log_match['timestamp']).to_time
+        ts_secs = timestamp.tv_sec
+        ts_nanos = timestamp.tv_nsec
       elsif record.key?('time')
         # k8s ISO8601 timestamp
         begin
@@ -1355,6 +1373,44 @@ module Fluent
         ts_secs = timestamp.tv_sec
         ts_nanos = timestamp.tv_nsec
       end
+      ts_secs = begin
+                  Integer ts_secs
+                rescue ArgumentError, TypeError
+                  ts_secs
+                end
+      ts_nanos = begin
+                   Integer ts_nanos
+                 rescue ArgumentError, TypeError
+                   ts_nanos
+                 end
+
+      # Adjust timestamps from the future.
+      # There are two cases:
+      # 1. The parsed timestamp is later in the current year:
+      # This can happen when system log lines from previous years are missing
+      # the year, so the date parser assumes the current year.
+      # We treat these lines as coming from last year.  This could label
+      # 2-year-old logs incorrectly, but this probably isn't super important.
+      #
+      # 2. The parsed timestamp is past the end of the current year:
+      # Since the year is different from the current year, this isn't the
+      # missing year in system logs.  It is unlikely that users explicitly
+      # write logs at a future date.  This could result from an unsynchronized
+      # clock on a VM, or some random value being parsed as the timestamp.
+      # We reset the timestamp on those lines to the default value and let the
+      # downstream API handle it.
+      if timestamp
+        next_year = Time.mktime(current_time.year + 1)
+        one_day_later = current_time.to_datetime.next_day.to_time
+        if timestamp >= next_year # Case 2.
+          ts_secs = 0
+          ts_nanos = 0
+        elsif timestamp >= one_day_later # Case 1.
+          adjusted_timestamp = timestamp.to_datetime.prev_year.to_time
+          ts_secs = adjusted_timestamp.tv_sec
+          # The value of ts_nanos should not change when subtracting a year.
+        end
+      end
       [ts_secs, ts_nanos]
     end
 
@@ -1454,7 +1510,7 @@ module Fluent
 
     def parse_severity(severity_str)
       # The API is case insensitive, but uppercase to make things simpler.
-      severity = severity_str.upcase.strip
+      severity = severity_str.to_s.upcase.strip
 
       # If the severity is already valid, just return it.
       return severity if VALID_SEVERITIES.include?(severity)
@@ -1714,15 +1770,19 @@ module Fluent
     def api_client
       if @use_grpc
         ssl_creds = GRPC::Core::ChannelCredentials.new
-        authentication = Google::Auth.get_application_default
-        creds = GRPC::Core::CallCredentials.new(authentication.updater_proc)
+        authentication = Google::Auth.get_application_default(LOGGING_SCOPE)
+        creds = GRPC::Core::CallCredentials.new(lambda do |a_hash|
+          authentication.apply(a_hash, use_configured_scope: true)
+        end)
         creds = ssl_creds.compose(creds)
-        @client = Google::Logging::V2::LoggingServiceV2::Stub.new(
-          'logging.googleapis.com', creds)
+        @client = Google::Cloud::Logging::V2::LoggingServiceV2Client.new(
+          channel: GRPC::Core::Channel.new(
+            'logging.googleapis.com', nil, creds))
       else
         unless @client.authorization.expired?
           begin
-            @client.authorization.fetch_access_token!
+            @client.authorization.fetch_access_token!(
+              use_configured_scope: true)
           rescue MultiJson::ParseError
             # Workaround an issue in the API client; just re-raise a more
             # descriptive error for the user (which will still cause a retry).
@@ -1873,7 +1933,7 @@ module Fluent
     # Increment the metric for the number of successful requests.
     def increment_successful_requests_count
       return unless @successful_requests_count
-      @successful_requests_count.increment(grpc: @use_grpc)
+      @successful_requests_count.increment(grpc: @use_grpc, code: @ok_code)
     end
 
     # Increment the metric for the number of failed requests, labeled by
@@ -1887,21 +1947,22 @@ module Fluent
     # ingested by the Stackdriver Logging API.
     def increment_ingested_entries_count(count)
       return unless @ingested_entries_count
-      @ingested_entries_count.increment({}, count)
+      @ingested_entries_count.increment({ grpc: @use_grpc, code: @ok_code },
+                                        count)
     end
 
     # Increment the metric for the number of log entries that were dropped
     # and not ingested by the Stackdriver Logging API.
-    def increment_dropped_entries_count(count)
+    def increment_dropped_entries_count(count, code)
       return unless @dropped_entries_count
-      @dropped_entries_count.increment({}, count)
+      @dropped_entries_count.increment({ grpc: @use_grpc, code: code }, count)
     end
 
     # Increment the metric for the number of log entries that were dropped
     # and not ingested by the Stackdriver Logging API.
     def increment_retried_entries_count(count, code)
       return unless @retried_entries_count
-      @retried_entries_count.increment({ code: code }, count)
+      @retried_entries_count.increment({ grpc: @use_grpc, code: code }, count)
     end
   end
 end

data/test/plugin/base_test.rb

@@ -177,7 +177,6 @@ module BaseTest
   def test_ec2_metadata_requires_project_id
     setup_ec2_metadata_stubs
     exception_count = 0
-    Fluent::GoogleCloudOutput::CredentialsInfo.stubs(:project_id).returns(nil)
     begin
       create_driver
     rescue Fluent::ConfigError => error
@@ -188,13 +187,15 @@ module BaseTest
     assert_equal 1, exception_count
   end
 
-  def test_ec2_metadata_project_id_from_credentials
-    setup_ec2_metadata_stubs
-    [IAM_CREDENTIALS, LEGACY_CREDENTIALS].each do |creds|
-      ENV['GOOGLE_APPLICATION_CREDENTIALS'] = creds[:path]
-      d = create_driver
-      d.run
-      assert_equal creds[:project_id], d.instance.project_id
+  def test_project_id_from_credentials
+    %w(gce ec2).each do |platform|
+      send("setup_#{platform}_metadata_stubs")
+      [IAM_CREDENTIALS, LEGACY_CREDENTIALS].each do |creds|
+        ENV['GOOGLE_APPLICATION_CREDENTIALS'] = creds[:path]
+        d = create_driver
+        d.run
+        assert_equal creds[:project_id], d.instance.project_id
+      end
     end
   end
 
@@ -216,18 +217,17 @@ module BaseTest
       d.emit('message' => log_entry(0))
       d.run
     end
-    verify_log_entries(1, COMPUTE_PARAMS)
+    verify_log_entries(1, COMPUTE_PARAMS.merge(
+                            project_id: IAM_CREDENTIALS[:project_id]))
   end
 
-  def test_one_log_with_invalid_json_credentials
-    setup_gce_metadata_stubs
-    ENV['GOOGLE_APPLICATION_CREDENTIALS'] = INVALID_CREDENTIALS[:path]
-    setup_logging_stubs do
-      d = create_driver
-      d.emit('message' => log_entry(0))
+  def test_invalid_json_credentials
+    %w(gce_metadata ec2_metadata no_metadata_service).each do |platform|
+      send("setup_#{platform}_stubs")
       exception_count = 0
+      ENV['GOOGLE_APPLICATION_CREDENTIALS'] = INVALID_CREDENTIALS[:path]
       begin
-        d.run
+        create_driver
       rescue RuntimeError => error
         assert error.message.include? 'Unable to read the credential file'
         exception_count += 1
@@ -607,7 +607,22 @@ module BaseTest
 
   def test_timestamps
     setup_gce_metadata_stubs
-    [Time.at(123_456.789), Time.at(0), Time.now].each do |ts|
+    current_time = Time.now
+    next_year = Time.mktime(current_time.year + 1)
+    one_second_before_next_year = next_year - 1
+    adjusted_to_last_year =
+      one_second_before_next_year.to_datetime.prev_year.to_time
+    one_second_into_next_year = next_year + 1
+    one_day_into_next_year = next_year.to_date.next_day.to_time
+    {
+      Time.at(123_456.789) => Time.at(123_456.789),
+      Time.at(0) => Time.at(0),
+      current_time => current_time,
+      one_second_before_next_year => adjusted_to_last_year,
+      next_year => Time.at(0),
+      one_second_into_next_year => Time.at(0),
+      one_day_into_next_year => Time.at(0)
+    }.each do |ts, adjusted_ts|
       expected_ts = []
       emit_index = 0
       setup_logging_stubs do
@@ -615,24 +630,24 @@ module BaseTest
         d = create_driver
         # Test the "native" fluentd timestamp as well as our nanosecond tags.
         d.emit({ 'message' => log_entry(emit_index) }, ts.to_f)
-        expected_ts.push(ts)
+        expected_ts.push(adjusted_ts)
         emit_index += 1
         d.emit('message' => log_entry(emit_index),
                'timeNanos' => ts.tv_sec * 1_000_000_000 + ts.tv_nsec)
-        expected_ts.push(ts)
+        expected_ts.push(adjusted_ts)
         emit_index += 1
         d.emit('message' => log_entry(emit_index),
                'timestamp' => { 'seconds' => ts.tv_sec, 'nanos' => ts.tv_nsec })
-        expected_ts.push(ts)
+        expected_ts.push(adjusted_ts)
         emit_index += 1
         d.emit('message' => log_entry(emit_index),
                'timestampSeconds' => ts.tv_sec, 'timestampNanos' => ts.tv_nsec)
-        expected_ts.push(ts)
+        expected_ts.push(adjusted_ts)
         emit_index += 1
         d.emit('message' => log_entry(emit_index),
                'timestampSeconds' => ts.tv_sec.to_s,
                'timestampNanos' => ts.tv_nsec.to_s)
-        expected_ts.push(ts)
+        expected_ts.push(adjusted_ts)
         emit_index += 1
         d.run
         verify_index = 0

data/test/plugin/constants.rb

@@ -644,5 +644,5 @@ module Constants
         }
       ]
     }
-  }.to_json
+  }.freeze
 end

data/test/plugin/test_out_google_cloud.rb

@@ -53,24 +53,31 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
 
   def test_partial_success
     setup_gce_metadata_stubs
-    {
-      APPLICATION_DEFAULT_CONFIG => 4.0,
-      PARTIAL_SUCCESS_CONFIG => 3.0
-    }.each do |config, failed_entry_count|
-      setup_prometheus
-      # The API Client should not retry this and the plugin should consume
-      # the exception.
-      stub_request(:post, WRITE_LOG_ENTRIES_URI)
-        .to_return(status: 400, body: PARTIAL_SUCCESS_RESPONSE_BODY)
-      d = create_driver(PROMETHEUS_ENABLE_CONFIG + config)
-      4.times do |i|
-        d.emit('message' => log_entry(i.to_s))
-      end
-      d.run
-      assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
-                                     failed_entry_count)
+    setup_prometheus
+    # The API Client should not retry this and the plugin should consume
+    # the exception.
+    root_error_code = PARTIAL_SUCCESS_RESPONSE_BODY['error']['code']
+    stub_request(:post, WRITE_LOG_ENTRIES_URI)
+      .to_return(status: root_error_code,
+                 body: PARTIAL_SUCCESS_RESPONSE_BODY.to_json)
+    d = create_driver(PROMETHEUS_ENABLE_CONFIG + PARTIAL_SUCCESS_CONFIG)
+    4.times do |i|
+      d.emit('message' => log_entry(i.to_s))
     end
-    assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 2)
+    d.run
+    assert_prometheus_metric_value(
+      :stackdriver_successful_requests_count, 0, grpc: false, code: 200)
+    assert_prometheus_metric_value(
+      :stackdriver_failed_requests_count, 1, grpc: false, code: root_error_code)
+    assert_prometheus_metric_value(
+      :stackdriver_ingested_entries_count, 1, grpc: false, code: 200)
+    assert_prometheus_metric_value(
+      :stackdriver_dropped_entries_count, 2, grpc: false, code: 3)
+    assert_prometheus_metric_value(
+      :stackdriver_dropped_entries_count, 1, grpc: false, code: 7)
+    assert_prometheus_metric_value(
+      :stackdriver_retried_entries_count, 0, grpc: false)
+    assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
   end
 
   def test_server_error
@@ -131,16 +138,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
         ingested_entries_count, dropped_entries_count,
         retried_entries_count = metric_values
       assert_prometheus_metric_value(:stackdriver_successful_requests_count,
-                                     successful_requests_count, grpc: false)
+                                     successful_requests_count,
+                                     grpc: false, code: 200)
       assert_prometheus_metric_value(:stackdriver_failed_requests_count,
                                      failed_requests_count,
                                      grpc: false, code: code)
       assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
-                                     ingested_entries_count)
+                                     ingested_entries_count,
+                                     grpc: false, code: 200)
       assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
-                                     dropped_entries_count)
+                                     dropped_entries_count,
+                                     grpc: false, code: code)
       assert_prometheus_metric_value(:stackdriver_retried_entries_count,
-                                     retried_entries_count, code: code)
+                                     retried_entries_count,
+                                     grpc: false, code: code)
     end
   end
 
@@ -206,6 +217,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_equal(100, test_obj.parse_severity('     105'))
     assert_equal(100, test_obj.parse_severity('     105    '))
 
+    assert_equal(100, test_obj.parse_severity(100))
+
     assert_equal('DEFAULT', test_obj.parse_severity('-100'))
     assert_equal('DEFAULT', test_obj.parse_severity('105 100'))
 
@@ -243,6 +256,11 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_equal('DEFAULT', test_obj.parse_severity('ER ROR'))
 
     # anything else should translate to 'DEFAULT'
+    assert_equal('DEFAULT', test_obj.parse_severity(nil))
+    assert_equal('DEFAULT', test_obj.parse_severity(Object.new))
+    assert_equal('DEFAULT', test_obj.parse_severity({}))
+    assert_equal('DEFAULT', test_obj.parse_severity([]))
+    assert_equal('DEFAULT', test_obj.parse_severity(100.0))
     assert_equal('DEFAULT', test_obj.parse_severity(''))
     assert_equal('DEFAULT', test_obj.parse_severity('garbage'))
     assert_equal('DEFAULT', test_obj.parse_severity('er'))

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -28,12 +28,19 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
 
   def test_client_error
     setup_gce_metadata_stubs
-    { 8 => 'ResourceExhausted',
-      12 => 'Unimplemented',
-      16 => 'Unauthenticated' }.each_with_index do |(code, message), index|
+    {
+      GRPC::Core::StatusCodes::CANCELLED => 'Cancelled',
+      GRPC::Core::StatusCodes::UNKNOWN => 'Unknown',
+      GRPC::Core::StatusCodes::INVALID_ARGUMENT => 'InvalidArgument',
+      GRPC::Core::StatusCodes::NOT_FOUND => 'NotFound',
+      GRPC::Core::StatusCodes::PERMISSION_DENIED => 'PermissionDenied',
+      GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED => 'ResourceExhausted',
+      GRPC::Core::StatusCodes::FAILED_PRECONDITION => 'FailedPrecondition',
+      GRPC::Core::StatusCodes::ABORTED => 'Aborted',
+      GRPC::Core::StatusCodes::UNAUTHENTICATED => 'Unauthenticated'
+    }.each_with_index do |(code, message), index|
       setup_logging_stubs(true, code, message) do
-        d = create_driver(USE_GRPC_CONFIG, 'test',
-                          GRPCLoggingMockFailingService.rpc_stub_class)
+        d = create_driver(USE_GRPC_CONFIG, 'test')
         # The API Client should not retry this and the plugin should consume the
         # exception.
         d.emit('message' => log_entry(0))
@@ -45,23 +52,20 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
 
   def test_server_error
     setup_gce_metadata_stubs
-    { 1 => 'Cancelled',
-      2 => 'Unknown',
-      4 => 'DeadlineExceeded',
-      13 => 'Internal',
-      14 => 'Unavailable' }.each_with_index do |(code, message), index|
+    {
+      GRPC::Core::StatusCodes::DEADLINE_EXCEEDED => 'DeadlineExceeded',
+      GRPC::Core::StatusCodes::UNIMPLEMENTED => 'Unimplemented',
+      GRPC::Core::StatusCodes::INTERNAL => 'Internal',
+      GRPC::Core::StatusCodes::UNAVAILABLE => 'Unavailable'
+    }.each_with_index do |(code, message), index|
       exception_count = 0
       setup_logging_stubs(true, code, message) do
-        d = create_driver(USE_GRPC_CONFIG, 'test',
-                          GRPCLoggingMockFailingService.rpc_stub_class)
+        d = create_driver(USE_GRPC_CONFIG, 'test')
         # The API client should retry this once, then throw an exception which
         # gets propagated through the plugin
         d.emit('message' => log_entry(0))
         begin
           d.run
-        rescue GRPC::Cancelled
-          # No need to check the message -- we already know the code.
-          exception_count += 1
         rescue GRPC::BadStatus => error
           assert_equal "#{code}:#{message}", error.message
           exception_count += 1
@@ -93,8 +97,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
       setup_prometheus
       (1..request_count).each do
         setup_logging_stubs(should_fail, code, 'SomeMessage') do
-          d = create_driver(USE_GRPC_CONFIG + PROMETHEUS_ENABLE_CONFIG, 'test',
-                            GRPCLoggingMockFailingService.rpc_stub_class)
+          d = create_driver(USE_GRPC_CONFIG + PROMETHEUS_ENABLE_CONFIG, 'test')
           (1..entry_count).each do |i|
             d.emit('message' => log_entry(i.to_s))
           end
@@ -110,16 +113,20 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
         ingested_entries_count, dropped_entries_count,
         retried_entries_count = metric_values
       assert_prometheus_metric_value(:stackdriver_successful_requests_count,
-                                     successful_requests_count, grpc: true)
+                                     successful_requests_count,
+                                     grpc: true, code: 0)
       assert_prometheus_metric_value(:stackdriver_failed_requests_count,
                                      failed_requests_count,
                                      grpc: true, code: code)
       assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
-                                     ingested_entries_count)
+                                     ingested_entries_count,
+                                     grpc: true, code: 0)
       assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
-                                     dropped_entries_count)
+                                     dropped_entries_count,
+                                     grpc: true, code: code)
       assert_prometheus_metric_value(:stackdriver_retried_entries_count,
-                                     retried_entries_count, code: code)
+                                     retried_entries_count,
+                                     grpc: true, code: code)
     end
   end
 
@@ -212,82 +219,58 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     use_grpc true
   ).freeze
 
-  @@mock_port = 0 # rubocop:disable Style/ClassVars
-
-  def generate_mock_host
-    # rubocop:disable Style/ClassVars
-    @@mock_port = (@@mock_port + 1) % 10_000 + 50_000
-    "localhost:#{@@mock_port}"
-    # rubocop:enable Style/ClassVars
-  end
-
   # Create a Fluentd output test driver with the Google Cloud Output plugin with
   # grpc enabled. The signature of this method is different between the grpc
   # path and the non-grpc path. For grpc, an additional grpc stub class can be
   # passed in to construct the mock used by the test driver.
-  def create_driver(conf = APPLICATION_DEFAULT_CONFIG, tag = 'test',
-                    grpc_stub = GRPCLoggingMockService.rpc_stub_class)
+  def create_driver(conf = APPLICATION_DEFAULT_CONFIG, tag = 'test')
     conf += USE_GRPC_CONFIG
     Fluent::Test::BufferedOutputTestDriver.new(
-      GoogleCloudOutputWithGRPCMock.new(grpc_stub, @mock_host),
-      tag).configure(conf, true)
+      GoogleCloudOutputWithGRPCMock.new(@grpc_stub), tag).configure(conf, true)
   end
 
   # Google Cloud Fluent output stub with grpc mock.
   class GoogleCloudOutputWithGRPCMock < Fluent::GoogleCloudOutput
-    def initialize(grpc_stub, mock_host)
+    def initialize(grpc_stub)
       super()
       @grpc_stub = grpc_stub
-      @mock_host = mock_host
     end
 
     def api_client
       ssl_creds = GRPC::Core::ChannelCredentials.new
-      authentication = Google::Auth.get_application_default
-      creds = GRPC::Core::CallCredentials.new(authentication.updater_proc)
+      authentication = Google::Auth.get_application_default(LOGGING_SCOPE)
+      creds = GRPC::Core::CallCredentials.new(lambda do |a_hash|
+        authentication.apply(a_hash, use_configured_scope: true)
+      end)
       ssl_creds.compose(creds)
 
-      # Here we have obtained the creds, but for the mock, we will leave the
-      # channel insecure.
-      @grpc_stub.new(@mock_host, :this_channel_is_insecure)
+      @grpc_stub
     end
   end
 
   # GRPC logging mock that successfully logs the records.
-  class GRPCLoggingMockService < Google::Logging::V2::LoggingServiceV2::Service
+  class GRPCLoggingMockService <
+      Google::Cloud::Logging::V2::LoggingServiceV2Client
     def initialize(requests_received)
       super()
       @requests_received = requests_received
     end
 
-    def write_log_entries(request, _call)
+    def write_log_entries(entries, log_name: nil, resource: nil, labels: nil)
+      request = Google::Apis::LoggingV2::WriteLogEntriesRequest.new(
+        log_name: log_name,
+        resource: resource,
+        labels: labels,
+        entries: entries
+      )
       @requests_received << request
       WriteLogEntriesResponse.new
     end
-
-    # TODO(lingshi) Remove these dummy methods when grpc/9033 is fixed.
-    #
-    # These methods should never be called, so they will just fail the tests
-    # with "unimplemented" errors..
-    def _undefined
-      raise "Method #{__callee__} is unimplemented and needs to be overridden."
-    end
-
-    alias list_logs _undefined
-    alias list_log_entries _undefined
-    alias list_log_services _undefined
-    alias list_log_service_indexes _undefined
-    alias list_monitored_resource_descriptors _undefined
-    alias delete_log _undefined
-    undef_method :_undefined
   end
 
   # GRPC logging mock that fails and returns server side or client side errors.
   class GRPCLoggingMockFailingService <
-      Google::Logging::V2::LoggingServiceV2::Service
-    # 'code_sent' and 'message_sent' are references of external variables. We
-    #  will assert the values of them later. 'code_value' and 'message_value'
-    #  are actual error code and message we expect this mock to return.
+      Google::Cloud::Logging::V2::LoggingServiceV2Client
     def initialize(code, message, failed_attempts)
       @code = code
       @message = message
@@ -295,66 +278,42 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
       super()
     end
 
-    def write_log_entries(_request, _call)
+    # rubocop:disable Lint/UnusedMethodArgument
+    def write_log_entries(entries, log_name: nil, resource: nil, labels: nil)
       @failed_attempts << 1
-      raise GRPC::BadStatus.new(@code, @message)
-    end
-
-    # TODO(lingshi) Remove these dummy methods when grpc/9033 is fixed.
-    #
-    # These methods should never be called, so they will just fail the tests
-    # with "unimplemented" errors..
-    def _undefined
-      raise "Method #{__callee__} is unimplemented and needs to be overridden."
+      begin
+        raise GRPC::BadStatus.new_status_exception(@code, @message)
+      rescue
+        # Google::Gax::GaxError will wrap the latest thrown exception as @cause.
+        raise Google::Gax::GaxError, @message
+      end
     end
-
-    alias list_logs _undefined
-    alias list_log_entries _undefined
-    alias list_log_services _undefined
-    alias list_log_service_indexes _undefined
-    alias list_monitored_resource_descriptors _undefined
-    alias delete_log _undefined
-    undef_method :_undefined
+    # rubocop:enable Lint/UnusedMethodArgument
   end
 
   # Set up grpc stubs to mock the external calls.
-  # TODO(qingling128): Remove this comment after grpc/grpc#12506 is resolved.
-  # Due to a gRPC load balancing issue (grpc/grpc#12506), we have to use a
-  # different port each time we create a gRPC mock as a temporary workaround.
-  # Thus we can only create one driver in each setup_logging_stubs context.
-  def setup_logging_stubs(should_fail = false, code = 0, message = 'Ok')
-    # Save the mock host in an instance variable, so later on when creating the
-    # logging driver, we can refer to this host with exactly the same port.
-    @mock_host = generate_mock_host
-    srv = GRPC::RpcServer.new
-    @failed_attempts = []
-    @requests_sent = []
+  def setup_logging_stubs(should_fail = false, code = nil, message = nil)
     if should_fail
-      grpc = GRPCLoggingMockFailingService.new(code, message, @failed_attempts)
+      @failed_attempts = []
+      @grpc_stub = GRPCLoggingMockFailingService.new(
+        code, message, @failed_attempts)
     else
-      grpc = GRPCLoggingMockService.new(@requests_sent)
-    end
-    srv.handle(grpc)
-    srv.add_http2_port(@mock_host, :this_port_is_insecure)
-    t = Thread.new { srv.run }
-    srv.wait_till_running
-    begin
-      yield
-    rescue Test::Unit::Failure, StandardError => e
-      srv.stop
-      t.join
-      raise e
+      @requests_sent = []
+      @grpc_stub = GRPCLoggingMockService.new(@requests_sent)
     end
-    srv.stop
-    t.join
-    @mock_host = nil
+    yield
   end
 
   # Verify the number and the content of the log entries match the expectation.
   # The caller can optionally provide a block which is called for each entry.
   def verify_log_entries(n, params, payload_type = 'textPayload', &block)
     @requests_sent.each do |request|
-      @logs_sent << JSON.parse(request.to_json)
+      @logs_sent << {
+        'entries' => request.entries.map { |entry| JSON.parse(entry.to_json) },
+        'labels' => request.labels,
+        'resource' => request.resource,
+        'logName' => request.log_name
+      }
     end
     verify_json_log_entries(n, params, payload_type, &block)
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.10
+  version: 0.6.11.pre.memory.1
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-19 00:00:00.000000000 Z
+date: 2017-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -243,9 +243,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.8