fluent-plugin-google-cloud 0.6.8 → 0.6.9.pre.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e8ef1f49a71df9fe1f3d06422ab70861af8b236
-  data.tar.gz: 75944aae3e5ea94e1d90ddf17e4ebda82eb2a75a
+  metadata.gz: 88272d0a151ead26a57ac7b523602e566a2c6e79
+  data.tar.gz: 3161e4dd7387a67af1c725a5423165f42d85a448
 SHA512:
-  metadata.gz: 859c5b09cf8130aebdba203a2683d13e7f605eaebff7978b829d7f128e16999e4bc1272282f85ab79b8209eef175bae62e523941dcf3947b8cc9da030bd51f84
-  data.tar.gz: 38987eb08a7d895d830a0d586a5fcd7bbc2594ef3863977e9518db7bad085e241857e1852f485027dddcce039c48e880cc58ae565734efa5bc1b093c798a7df2
+  metadata.gz: b3e2e2acd6e6b7083bf806ad2cdda05f2bb0014cb751a2ee7777b05c21428e2c6256ff16764652b337700f8568df3e02b3e055e3ce8da8505f24e95220e50f6f
+  data.tar.gz: d9bed2291c45799ead9d6ef38d3f59bfb9e49f26c34b141e1f2f7db60e4bdc7623304d276b468d6816b02981b8439a39a5158d23b954bc319a93922db1b0e786

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.6.8)
+    fluent-plugin-google-cloud (0.6.9.pre.1)
       fluentd (~> 0.10)
-      google-api-client (~> 0.9.0)
-      google-cloud-logging (= 0.24.1)
+      google-api-client (~> 0.14)
+      google-cloud-logging (~> 1.2, >= 1.2.3)
       googleapis-common-protos (~> 1.3)
       googleauth (~> 0.4, < 0.5.2)
       grpc (~> 1.0, < 1.3)
@@ -21,6 +21,8 @@ GEM
     cool.io (1.5.1)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
+    declarative (0.0.10)
+    declarative-option (0.1.0)
     faraday (0.13.1)
       multipart-post (>= 1.2, < 3)
     fluentd (0.14.21)
@@ -34,22 +36,23 @@ GEM
       tzinfo (~> 1.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
-    google-api-client (0.9.28)
-      addressable (~> 2.3)
+    google-api-client (0.15.0)
+      addressable (~> 2.5, >= 2.5.1)
       googleauth (~> 0.5)
-      httpclient (~> 2.7)
-      hurley (~> 0.1)
-      memoist (~> 0.11)
-      mime-types (>= 1.6)
-      representable (~> 2.3.0)
-      retriable (~> 2.0)
-    google-cloud-core (0.21.1)
+      httpclient (>= 2.8.1, < 3.0)
+      mime-types (~> 3.0)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.0)
+    google-cloud-core (1.0.0)
+      google-cloud-env (~> 1.0)
       googleauth (~> 0.5.1)
-    google-cloud-logging (0.24.1)
-      google-cloud-core (~> 0.21.1)
+    google-cloud-env (1.0.1)
+      faraday (~> 0.11)
+    google-cloud-logging (1.2.3)
+      google-cloud-core (~> 1.0)
       google-gax (~> 0.8.0)
-      stackdriver-core (~> 0.21.0)
-    google-gax (0.8.7)
+      stackdriver-core (~> 1.2)
+    google-gax (0.8.10)
       google-protobuf (~> 3.2)
       googleapis-common-protos (~> 1.3.5)
       googleauth (~> 0.5.1)
@@ -73,10 +76,9 @@ GEM
     grpc (1.2.5)
       google-protobuf (~> 3.1)
       googleauth (~> 0.5.1)
-    hashdiff (0.3.6)
+    hashdiff (0.3.7)
     http_parser.rb (0.6.0)
     httpclient (2.8.3)
-    hurley (0.2)
     json (1.8.6)
     jwt (1.5.6)
     little-plugger (1.1.4)
@@ -105,9 +107,11 @@ GEM
     rainbow (2.2.2)
       rake
     rake (10.5.0)
-    representable (2.3.0)
-      uber (~> 0.0.7)
-    retriable (2.1.0)
+    representable (3.0.4)
+      declarative (< 0.1.0)
+      declarative-option (< 0.2.0)
+      uber (< 0.2.0)
+    retriable (3.1.1)
     rly (0.2.3)
     rubocop (0.35.1)
       astrolabe (~> 1.3)
@@ -116,18 +120,18 @@ GEM
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.7)
       tins (<= 1.6.0)
-    ruby-progressbar (1.8.3)
+    ruby-progressbar (1.9.0)
     ruby_dig (0.0.2)
     safe_yaml (1.0.4)
     serverengine (2.0.5)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    signet (0.7.3)
+    signet (0.8.1)
       addressable (~> 2.3)
       faraday (~> 0.9)
-      jwt (~> 1.5)
+      jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
-    stackdriver-core (0.21.0)
+    stackdriver-core (1.2.0)
     strptime (0.1.9)
     test-unit (3.2.6)
       power_assert
@@ -137,7 +141,7 @@ GEM
       thread_safe (~> 0.1)
     tzinfo-data (1.2017.2)
       tzinfo (>= 1.0.0)
-    uber (0.0.15)
+    uber (0.1.0)
     webmock (2.3.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.8'
+  gem.version       = '0.6.9.pre.1'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -21,8 +21,8 @@ eos
 
   gem.add_runtime_dependency 'fluentd', '~> 0.10'
   gem.add_runtime_dependency 'googleapis-common-protos', '~> 1.3'
-  gem.add_runtime_dependency 'google-api-client', '~> 0.9.0'
-  gem.add_runtime_dependency 'google-cloud-logging', '0.24.1'
+  gem.add_runtime_dependency 'google-api-client', '~> 0.14'
+  gem.add_runtime_dependency 'google-cloud-logging', '~> 1.2', '>= 1.2.3'
   gem.add_runtime_dependency 'googleauth', '~> 0.4', '< 0.5.2'
   gem.add_runtime_dependency 'grpc', '~> 1.0', '< 1.3'
   gem.add_runtime_dependency 'json', '~> 1.8'

data/lib/fluent/plugin/out_google_cloud.rb

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+require 'erb'
 require 'grpc'
 require 'json'
 require 'open-uri'
@@ -18,7 +19,7 @@ require 'socket'
 require 'time'
 require 'yaml'
 require 'google/apis'
-require 'google/apis/logging_v2beta1'
+require 'google/apis/logging_v2'
 require 'google/logging/v2/logging_pb'
 require 'google/logging/v2/logging_services_pb'
 require 'google/logging/v2/log_entry_pb'
@@ -67,7 +68,7 @@ module Fluent
       DATAFLOW_CONSTANTS = {
         service: 'dataflow.googleapis.com',
         resource_type: 'dataflow_step',
-        extra_common_labels: %w(region job_name job_id step_id)
+        extra_resource_labels: %w(region job_name job_id step_id)
       }
       DATAPROC_CONSTANTS = {
         service: 'cluster.dataproc.googleapis.com',
@@ -81,7 +82,7 @@ module Fluent
       ML_CONSTANTS = {
         service: 'ml.googleapis.com',
         resource_type: 'ml_job',
-        extra_common_labels: %w(job_id task_name)
+        extra_resource_labels: %w(job_id task_name)
       }
 
       # The map between a subservice name and a resource type.
@@ -112,7 +113,7 @@ module Fluent
         'http://local-metadata-agent.stackdriver.com:8000'
     end
 
-    # Constants for log entry field extraction.
+    # Internal constants.
     module InternalConstants
       # The label name of local_resource_id in the json payload. When a record
       # has this field in the payload, we will use the value to retrieve
@@ -144,7 +145,7 @@ module Fluent
           # The grpc version class name.
           'Google::Logging::Type::HttpRequest',
           # The non-grpc version class name.
-          'Google::Apis::LoggingV2beta1::HttpRequest'
+          'Google::Apis::LoggingV2::HttpRequest'
         ],
         'source_location' => [
           '@source_location_key',
@@ -154,7 +155,7 @@ module Fluent
             %w(line line parse_int)
           ],
           'Google::Logging::V2::LogEntrySourceLocation',
-          'Google::Apis::LoggingV2beta1::LogEntrySourceLocation'
+          'Google::Apis::LoggingV2::LogEntrySourceLocation'
         ],
         'operation' => [
           '@operation_key',
@@ -165,9 +166,13 @@ module Fluent
             %w(last last parse_bool)
           ],
           'Google::Logging::V2::LogEntryOperation',
-          'Google::Apis::LoggingV2beta1::LogEntryOperation'
+          'Google::Apis::LoggingV2::LogEntryOperation'
         ]
       }
+
+      # The name of the WriteLogEntriesPartialErrors field in the error details.
+      PARTIAL_ERROR_FIELD =
+        'type.googleapis.com/google.logging.v2.WriteLogEntriesPartialErrors'
     end
 
     include self::ServiceConstants
@@ -177,7 +182,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
-    PLUGIN_VERSION = '0.6.8'
+    PLUGIN_VERSION = '0.6.9.pre.1'
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
@@ -280,6 +285,12 @@ module Fluent
     # Cloud Logging API.
     config_param :use_grpc, :bool, :default => false
 
+    # Whether valid entries should be written even if some other entries fail
+    # due to INVALID_ARGUMENT or PERMISSION_DENIED errors when communicating to
+    # the Cloud Logging API. This is highly recommended. Right now this only
+    # works with the REST path (use_grpc = false).
+    config_param :partial_success, :bool, :default => false
+
     # Whether to allow non-UTF-8 characters in user logs. If set to true, any
     # non-UTF-8 character would be replaced by the string specified by
     # 'non_utf8_replacement_string'. If set to false, any non-UTF-8 character
@@ -338,6 +349,15 @@ module Fluent
     def configure(conf)
       super
 
+      # TODO(qingling128): Remove this warning after the support is added. Also
+      # remove the comment in the description of this configuration.
+      if @partial_success && @use_grpc
+        @log.warn 'Detected partial_success enabled while use_grpc is also' \
+                  ' enabled. The support for partial success in the gRPC path' \
+                  ' is to be added in the near future. For now the ' \
+                  ' partial_success flag will be ignored.'
+      end
+
       # If monitoring is enabled, register metrics in the default registry
       # and store metric objects for future use.
       if @enable_monitoring
@@ -519,7 +539,7 @@ module Fluent
             # Remove the labels if we didn't populate them with anything.
             entry_level_resource.labels = nil if
               entry_level_resource.labels.empty?
-            entry = Google::Apis::LoggingV2beta1::LogEntry.new(
+            entry = Google::Apis::LoggingV2::LogEntry.new(
               labels: entry_level_common_labels,
               resource: entry_level_resource,
               severity: severity,
@@ -622,20 +642,24 @@ module Fluent
         else
           begin
             write_request = \
-              Google::Apis::LoggingV2beta1::WriteLogEntriesRequest.new(
+              Google::Apis::LoggingV2::WriteLogEntriesRequest.new(
                 log_name: log_name,
                 resource: group_level_resource,
                 labels: group_level_common_labels,
+                partial_success: @partial_success,
                 entries: entries)
             entries_count = entries.length
 
-            # TODO: RequestOptions
             begin
-              client.write_entry_log_entries(write_request)
+              client.write_entry_log_entries(
+                write_request,
+                options: { api_format_version: '2' }
+              )
             rescue Google::Apis::Error => error
               increment_failed_requests_count(error.status_code)
               raise error
             end
+
             increment_successful_requests_count
             increment_ingested_entries_count(entries_count)
 
@@ -664,9 +688,20 @@ module Fluent
           rescue Google::Apis::ClientError => error
             # Most ClientErrors indicate a problem with the request itself and
             # should not be retried.
-            increment_dropped_entries_count(entries_count)
-            @log.warn "Dropping #{entries_count} log message(s)",
-                      error_class: error.class.to_s, error: error.to_s
+            error_details_map = construct_error_details_map(error)
+            if error_details_map.empty?
+              increment_dropped_entries_count(entries_count)
+              @log.warn "Dropping #{entries_count} log message(s)",
+                        error_class: error.class.to_s, error: error.to_s
+            else
+              error_details_map.each do |(error_code, error_message), indexes|
+                partial_error_count = indexes.length
+                increment_dropped_entries_count(partial_error_count)
+                @log.warn "Dropping #{partial_error_count} log message(s)",
+                          error_code: "google.rpc.Code[#{error_code}]",
+                          error: error_message
+              end
+            end
           end
         end
       end
@@ -836,7 +871,7 @@ module Fluent
     # Metadata Agent. Thus it should be equivalent to what Metadata Agent
     # returns.
     def determine_agent_level_monitored_resource_via_legacy
-      resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
+      resource = Google::Apis::LoggingV2::MonitoredResource.new(
         labels: {})
       resource.type = determine_agent_level_monitored_resource_type
       resource.labels = determine_agent_level_monitored_resource_labels(
@@ -1090,6 +1125,18 @@ module Fluent
         common_labels.delete("#{COMPUTE_CONSTANTS[:service]}/resource_name")
       end
 
+      # Cloud Dataflow and Cloud ML.
+      # These labels can be set via the 'labels' option.
+      # Report them as monitored resource labels instead of common labels.
+      # e.g. "dataflow.googleapis.com/job_id" => "job_id"
+      [DATAFLOW_CONSTANTS, ML_CONSTANTS].each do |service_constants|
+        next unless resource.type == service_constants[:resource_type]
+        resource.labels.merge!(
+          delete_and_extract_labels(
+            common_labels, service_constants[:extra_resource_labels]
+              .map { |l| ["#{service_constants[:service]}/#{l}", l] }.to_h))
+      end
+
       resource.freeze
       resource.labels.freeze
       common_labels.freeze
@@ -1157,14 +1204,14 @@ module Fluent
       common_labels.merge!(delete_and_extract_labels(record, @label_map))
 
       # Cloud Dataflow and Cloud ML.
-      # These labels can be set via configuring 'labels' or 'label_map'.
+      # These labels can be set via the 'labels' or 'label_map' options.
       # Report them as monitored resource labels instead of common labels.
       # e.g. "dataflow.googleapis.com/job_id" => "job_id"
       [DATAFLOW_CONSTANTS, ML_CONSTANTS].each do |service_constants|
         next unless resource.type == service_constants[:resource_type]
         resource.labels.merge!(
           delete_and_extract_labels(
-            common_labels, service_constants[:extra_common_labels]
+            common_labels, service_constants[:extra_resource_labels]
               .map { |l| ["#{service_constants[:service]}/#{l}", l] }.to_h))
       end
 
@@ -1187,7 +1234,7 @@ module Fluent
       # TODO(qingling128): Use Google::Api::MonitoredResource directly after we
       # upgrade gRPC version to include the fix for the protobuf map
       # corruption issue.
-      Google::Apis::LoggingV2beta1::MonitoredResource.new(
+      Google::Apis::LoggingV2::MonitoredResource.new(
         type: resource.type,
         labels: resource.labels.to_h
       )
@@ -1658,7 +1705,7 @@ module Fluent
       # TODO: Use a non-default ClientOptions object.
       Google::Apis::ClientOptions.default.application_name = PLUGIN_NAME
       Google::Apis::ClientOptions.default.application_version = PLUGIN_VERSION
-      @client = Google::Apis::LoggingV2beta1::LoggingService.new
+      @client = Google::Apis::LoggingV2::LoggingService.new
       @client.authorization = Google::Auth.get_application_default(
         LOGGING_SCOPE)
     end
@@ -1710,6 +1757,118 @@ module Fluent
       end
     end
 
+    # Extract a map of error details from an potentially partially successful
+    # request. Return an empty map if @partial_success is not enabled.
+    #
+    # The keys in this map are [error_code, error_message] pairs, and the values
+    # are a list of stringified indexes of log entries that failed due to this
+    # error.
+    #
+    # A sample error.body looks like:
+    # {
+    #   "error": {
+    #     "code": 403,
+    #     "message": "User not authorized.",
+    #     "status": "PERMISSION_DENIED",
+    #     "details": [
+    #       {
+    #         "@type": "type.googleapis.com/google.logging.v2.WriteLogEntriesPar
+    #           tialErrors",
+    #         "logEntryErrors": {
+    #           "0": {
+    #             "code": 7,
+    #             "message": "User not authorized."
+    #           },
+    #           "1": {
+    #             "code": 3,
+    #             "message": "Log name contains illegal character :"
+    #           },
+    #           "3": {
+    #             "code": 3,
+    #             "message": "Log name contains illegal character :"
+    #           }
+    #         }
+    #       },
+    #       {
+    #         "@type": "type.googleapis.com/google.rpc.DebugInfo",
+    #         "detail": ...
+    #       }
+    #     ]
+    #   }
+    # }
+    #
+    # The root level "code", "message", and "status" simply match the root
+    # cause of the first failed log entry. For example, if we switched the order
+    # of the log entries, then we would get:
+    # {
+    #    "error" : {
+    #       "code" : 400,
+    #       "message" : "Log name contains illegal character :",
+    #       "status" : "INVALID_ARGUMENT",
+    #       "details": ...
+    #    }
+    # }
+    # We will ignore it anyway and look at the details instead which includes
+    # info for all failed log entries.
+    #
+    # In this example, the logEntryErrors that we care are:
+    # {
+    #   "0": {
+    #     "code": 7,
+    #     "message": "User not authorized."
+    #   },
+    #   "1": {
+    #     "code": 3,
+    #     "message": "Log name contains illegal character :"
+    #   },
+    #   "3": {
+    #     "code": 3,
+    #     "message": "Log name contains illegal character :"
+    #   }
+    # }
+    #
+    # The ultimate map that is constructed is:
+    # {
+    #   [7, 'User not authorized.']: ['0'],
+    #   [3, 'Log name contains illegal character :']: ['1', '3']
+    # }
+    def construct_error_details_map(error)
+      return {} unless @partial_success
+      error_details_map = Hash.new { |h, k| h[k] = [] }
+
+      error_details = ensure_array(
+        ensure_hash(ensure_hash(JSON.parse(error.body))['error'])['details'])
+      partial_errors = error_details.detect(
+        -> { fail JSON::ParserError, "No type #{PARTIAL_ERROR_FIELD}." }
+      ) do |error_detail|
+        ensure_hash(error_detail)['@type'] == PARTIAL_ERROR_FIELD
+      end
+      log_entry_errors = ensure_hash(
+        ensure_hash(partial_errors)['logEntryErrors'])
+      log_entry_errors.each do |index, log_entry_error|
+        error_hash = ensure_hash(log_entry_error)
+        fail JSON::ParserError,
+             "Entry #{index} is missing 'code' or 'message'." unless
+          error_hash['code'] && error_hash['message']
+        error_key = [error_hash['code'], error_hash['message']].freeze
+        # TODO(qingling128): Convert indexes to integers.
+        error_details_map[error_key] << index
+      end
+      error_details_map
+    rescue JSON::ParserError => e
+      @log.warn 'Failed to extract log entry errors from the error details:' \
+                " #{error.body}.", error: e
+      {}
+    end
+
+    def ensure_array(value)
+      Array.try_convert(value) || (fail JSON::ParserError, "#{value.class}")
+    end
+
+    def ensure_hash(value)
+      Hash.try_convert(value) || (fail JSON::ParserError, "#{value.class}")
+    end
+
     # Increment the metric for the number of successful requests.
     def increment_successful_requests_count
       return unless @successful_requests_count
@@ -1748,7 +1907,7 @@ end
 
 module Google
   module Apis
-    module LoggingV2beta1
+    module LoggingV2
       # Override MonitoredResource::dup to make a deep copy.
       class MonitoredResource
         def dup

data/test/plugin/base_test.rb

@@ -93,6 +93,18 @@ module BaseTest
     end
   end
 
+  def test_configure_partial_success
+    setup_gce_metadata_stubs
+    {
+      APPLICATION_DEFAULT_CONFIG => false,
+      PARTIAL_SUCCESS_CONFIG => true
+    }.each do |(config, partial_success)|
+      d = create_driver(config)
+      assert_equal partial_success,
+                   d.instance.instance_variable_get(:@partial_success)
+    end
+  end
+
   def test_metadata_loading
     setup_gce_metadata_stubs
     d = create_driver
@@ -961,6 +973,26 @@ module BaseTest
     verify_log_entries(1, DATAPROC_PARAMS, 'jsonPayload')
   end
 
+  def test_cloud_ml_log
+    setup_gce_metadata_stubs
+    setup_logging_stubs do
+      d = create_driver(CONFIG_ML, ML_TAG)
+      d.emit(ml_log_entry(0))
+      d.run
+    end
+    verify_log_entries(1, ML_PARAMS)
+  end
+
+  def test_cloud_dataflow_log
+    setup_gce_metadata_stubs
+    setup_logging_stubs do
+      d = create_driver(CONFIG_DATAFLOW, DATAFLOW_TAG)
+      d.emit(dataflow_log_entry(0))
+      d.run
+    end
+    verify_log_entries(1, DATAFLOW_PARAMS)
+  end
+
   def test_log_entry_http_request_field_from_record
     verify_subfields_from_record(DEFAULT_HTTP_REQUEST_KEY)
   end
@@ -1023,11 +1055,11 @@ module BaseTest
       setup_logging_stubs do
         d = create_driver
         @logs_sent = []
-        d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge('latency' => input))
+        d.emit('httpRequest' => http_request_message.merge('latency' => input))
         d.run
       end
       verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
-        assert_equal HTTP_REQUEST_MESSAGE.merge('latency' => expected),
+        assert_equal http_request_message.merge('latency' => expected),
                      entry['httpRequest'], entry
         assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
       end
@@ -1044,11 +1076,11 @@ module BaseTest
       setup_logging_stubs do
         d = create_driver
         @logs_sent = []
-        d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge('latency' => input))
+        d.emit('httpRequest' => http_request_message.merge('latency' => input))
         d.run
       end
       verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
-        assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry
+        assert_equal http_request_message, entry['httpRequest'], entry
         assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
       end
     end
@@ -1597,8 +1629,22 @@ module BaseTest
     end
   end
 
+  def log_entry_subfields_params
+    {
+      # The keys are the names of fields in the payload that we are extracting
+      # LogEntry info from. The values are lists of two elements: the name of
+      # the subfield in LogEntry object and the expected value of that field.
+      DEFAULT_HTTP_REQUEST_KEY => [
+        'httpRequest', http_request_message],
+      DEFAULT_SOURCE_LOCATION_KEY => [
+        'sourceLocation', source_location_message],
+      DEFAULT_OPERATION_KEY => [
+        'operation', OPERATION_MESSAGE]
+    }
+  end
+
   def verify_subfields_from_record(payload_key)
-    destination_key, payload_value = LOG_ENTRY_SUBFIELDS_PARAMS[payload_key]
+    destination_key, payload_value = log_entry_subfields_params[payload_key]
     @logs_sent = []
     setup_gce_metadata_stubs
     setup_logging_stubs do
@@ -1614,7 +1660,7 @@ module BaseTest
   end
 
   def verify_subfields_partial_from_record(payload_key)
-    destination_key, payload_value = LOG_ENTRY_SUBFIELDS_PARAMS[payload_key]
+    destination_key, payload_value = log_entry_subfields_params[payload_key]
     @logs_sent = []
     setup_gce_metadata_stubs
     setup_logging_stubs do
@@ -1631,7 +1677,7 @@ module BaseTest
   end
 
   def verify_subfields_when_not_hash(payload_key)
-    destination_key = LOG_ENTRY_SUBFIELDS_PARAMS[payload_key][0]
+    destination_key = log_entry_subfields_params[payload_key][0]
     @logs_sent = []
     setup_gce_metadata_stubs
     setup_logging_stubs do
@@ -1646,14 +1692,22 @@ module BaseTest
     end
   end
 
+  def http_request_message
+    HTTP_REQUEST_MESSAGE
+  end
+
+  def source_location_message
+    SOURCE_LOCATION_MESSAGE
+  end
+
   # Replace the 'referer' field with nil.
   def http_request_message_with_nil_referer
-    HTTP_REQUEST_MESSAGE.merge('referer' => nil)
+    http_request_message.merge('referer' => nil)
   end
 
   # Unset the 'referer' field.
   def http_request_message_with_absent_referer
-    HTTP_REQUEST_MESSAGE.reject do |k, _|
+    http_request_message.reject do |k, _|
       k == 'referer'
     end
   end

data/test/plugin/constants.rb

@@ -97,7 +97,7 @@ module Constants
   DATAFLOW_JOB_NAME = 'job_name_1'
   DATAFLOW_JOB_ID = 'job_id_1'
   DATAFLOW_STEP_ID = 'step_1'
-  DATAFLOW_TAG = 'dataflow.googleapis.com/worker'
+  DATAFLOW_TAG = 'dataflow-worker'
 
   # Dataproc specific labels.
   DATAPROC_CLUSTER_NAME = 'test-cluster'
@@ -140,6 +140,10 @@ module Constants
     detect_json true
   )
 
+  PARTIAL_SUCCESS_CONFIG = %(
+    partial_success true
+  )
+
   # rubocop:disable Metrics/LineLength
   PRIVATE_KEY_CONFIG = %(
      auth_method private_key
@@ -533,13 +537,6 @@ module Constants
     'last' => true
   }
 
-  LOG_ENTRY_SUBFIELDS_PARAMS = {
-    # payload key, destination key, payload value
-    DEFAULT_HTTP_REQUEST_KEY => ['httpRequest', HTTP_REQUEST_MESSAGE],
-    DEFAULT_SOURCE_LOCATION_KEY => ['sourceLocation', SOURCE_LOCATION_MESSAGE],
-    DEFAULT_OPERATION_KEY => ['operation', OPERATION_MESSAGE]
-  }
-
   CUSTOM_LABELS_MESSAGE = {
     'customKey' => 'value'
   }
@@ -605,4 +602,43 @@ module Constants
         }
       }.to_json
   }
+
+  PARTIAL_SUCCESS_RESPONSE_BODY = {
+    'error' => {
+      'code' => 403,
+      'message' => 'User not authorized.',
+      'status' => 'PERMISSION_DENIED',
+      'details' => [
+        {
+          '@type' => 'type.googleapis.com/google.logging.v2.WriteLogEntriesPa' \
+            'rtialErrors',
+          'logEntryErrors' => {
+            '0' => {
+              'code' => 7,
+              'message' => 'User not authorized.'
+            },
+            '1' => {
+              'code' => 3,
+              'message' => 'Log name contains illegal character :'
+            },
+            '2' => {
+              'code' => 3,
+              'message' => 'Log name contains illegal character :'
+            }
+          }
+        },
+        {
+          '@type' => 'type.googleapis.com/google.rpc.DebugInfo',
+          'detail' => '[ORIGINAL ERROR] generic::permission_denied: User not ' \
+            'authorized. [google.rpc.error_details_ext] { message: \"User not' \
+            ' authorized.\" details { type_url: \"type.googleapis.com/google.' \
+            'logging.v2.WriteLogEntriesPartialErrors\" value: \"\\n\\034\\010' \
+            '\\000\\022\\030\\010\\007\\022\\024User not authorized.\\n-\\010' \
+            '\\001\\022)\\010\\003\\022%Log name contains illegal character :' \
+            '\\n-\\010\\002\\022)\\010\\003\\022%Log name contains illegal ch' \
+            'aracter :\" } }'
+        }
+      ]
+    }
+  }.to_json
 end

data/test/plugin/test_out_google_cloud.rb

@@ -51,6 +51,28 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 2)
   end
 
+  def test_partial_success
+    setup_gce_metadata_stubs
+    {
+      APPLICATION_DEFAULT_CONFIG => 4.0,
+      PARTIAL_SUCCESS_CONFIG => 3.0
+    }.each do |config, failed_entry_count|
+      setup_prometheus
+      # The API Client should not retry this and the plugin should consume
+      # the exception.
+      stub_request(:post, WRITE_LOG_ENTRIES_URI)
+        .to_return(status: 400, body: PARTIAL_SUCCESS_RESPONSE_BODY)
+      d = create_driver(PROMETHEUS_ENABLE_CONFIG + config)
+      4.times do |i|
+        d.emit('message' => log_entry(i.to_s))
+      end
+      d.run
+      assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
+                                     failed_entry_count)
+    end
+    assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 2)
+  end
+
   def test_server_error
     setup_gce_metadata_stubs
     # The API client should retry this once, then throw an exception which
@@ -251,7 +273,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
 
   private
 
-  WRITE_LOG_ENTRIES_URI = 'https://logging.googleapis.com/v2beta1/entries:write'
+  WRITE_LOG_ENTRIES_URI = 'https://logging.googleapis.com/v2/entries:write'
 
   def rename_key(hash, old_key, new_key)
     hash.merge(new_key => hash[old_key]).reject { |k, _| k == old_key }
@@ -316,4 +338,25 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   def null_value
     nil
   end
+
+  # 'responseSize' and 'requestSize' are Integers in the gRPC proto, yet Strings
+  # in REST API client.
+  # TODO(qingling128): Address this accordingly once the following question is
+  # answered: https://github.com/google/google-api-ruby-client/issues/619.
+  # If this discrepancy is legit, add some comments to explain the reason.
+  # Otherwise once the discrepancy is fixed, we need to upgrade to that version
+  # and change our tests accordingly.
+  def http_request_message
+    HTTP_REQUEST_MESSAGE.merge(
+      'responseSize' => HTTP_REQUEST_MESSAGE['responseSize'].to_s,
+      'requestSize' => HTTP_REQUEST_MESSAGE['requestSize'].to_s
+    )
+  end
+
+  # 'line' is an Integer in the gRPC proto, yet a String in the REST API client.
+  def source_location_message
+    SOURCE_LOCATION_MESSAGE.merge(
+      'line' => SOURCE_LOCATION_MESSAGE['line'].to_s
+    )
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.8
+  version: 0.6.9.pre.1
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-22 00:00:00.000000000 Z
+date: 2017-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -45,28 +45,34 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: '0.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: google-cloud-logging
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.24.1
+        version: 1.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.24.1
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.3
 - !ruby/object:Gem::Dependency
   name: googleauth
   requirement: !ruby/object:Gem::Requirement
@@ -249,9 +255,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.8