fluent-plugin-google-cloud 0.5.3.grpc.alpha.4 → 0.5.3.grpc.alpha.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 29718f443dea67bcf817ee44390e61373a307db7
-  data.tar.gz: 54b6472916c8657230db649ff339ab1d31b5362d
+  metadata.gz: af37e8c845023885ae3cec2dcef35368f30e7ee3
+  data.tar.gz: 57cd2c566bb2c89ee063dbb12fbd02a75e8744c9
 SHA512:
-  metadata.gz: 1d5695d8834eafc6f059c5a591ff9d3188739814fd7dcc4e2dc976324c8e45f3b5f0b7d914062c9581b2242f76dfd75d0356178c5a6b7f2cbadd3d49b064ec61
-  data.tar.gz: 46c8d492364314a4220f3ea61a81814fbd9db64c3af06331e025853582404d33806a5bcacc7be64379aa2a646f415d5e6507e6fcae6d95085fafe838b4f5e19a
+  metadata.gz: c8ca9cfcb3c564822cfd06870010e21358352612a401bdfda9c0d49f4715f9a0831e3daab338a69d9b584a2b568255aa2bae35e512771cfb837de74b64712be0
+  data.tar.gz: 43fa64e97398e3526a6153db587806b9a56291c330b6b066f5a7a106abf769c040cd4bb6a849a4d6bf7b8ba65427dc1a3b5cd5ff3a3ed156ee01e56f790005cf

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.5.3.grpc.alpha.4'
+  gem.version       = '0.5.3.grpc.alpha.5'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/lib/fluent/plugin/out_google_cloud.rb

@@ -39,7 +39,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
-    PLUGIN_VERSION = '0.5.3.grpc.alpha.4'
+    PLUGIN_VERSION = '0.5.3.grpc.alpha.5'
 
     # Constants for service names.
     APPENGINE_SERVICE = 'appengine.googleapis.com'
@@ -599,338 +599,6 @@ module Fluent
       end
     end
 
-    # def old_write(chunk)
-    #   # Group the entries since we have to make one call per tag.
-    #   grouped_entries = {}
-    #   chunk.msgpack_each do |tag, *arr|
-    #     grouped_entries[tag] = [] unless grouped_entries.key?(tag)
-    #     grouped_entries[tag].push(arr)
-    #   end
-    #
-    #   grouped_entries.each do |tag, arr|
-    #     entries = []
-    #     labels = @common_labels.clone
-    #
-    #     if @running_cloudfunctions
-    #       # If the current group of entries is coming from a Cloud Functions
-    #       # function, the function name can be extracted from the tag.
-    #       match_data = @cloudfunctions_tag_regexp.match(tag)
-    #       if match_data
-    #         # Service name is set to Cloud Functions only for logs actually
-    #         # coming from a function.
-    #         @service_name = CLOUDFUNCTIONS_SERVICE
-    #         labels["#{CLOUDFUNCTIONS_SERVICE}/region"] = @gcf_region
-    #         labels["#{CLOUDFUNCTIONS_SERVICE}/function_name"] =
-    #           decode_cloudfunctions_function_name(
-    #             match_data['encoded_function_name'])
-    #       else
-    #         # Other logs are considered as coming from the Container Engine
-    #         # service.
-    #         @service_name = CONTAINER_SERVICE
-    #       end
-    #     end
-    #     if @service_name == CONTAINER_SERVICE && \
-    #        @compiled_kubernetes_tag_regexp
-    #       # Container logs in Kubernetes are tagged based on where they came
-    #       # from, so we can extract useful metadata from the tag.
-    #       # Do this here to avoid having to repeat it for each record.
-    #       match_data = @compiled_kubernetes_tag_regexp.match(tag)
-    #       if match_data
-    #         %w(namespace_name pod_name container_name).each do |field|
-    #           labels["#{CONTAINER_SERVICE}/#{field}"] = match_data[field]
-    #         end
-    #       end
-    #     end
-    #
-    #     if @use_grpc
-    #       arr.each do |time, record|
-    #         next unless record.is_a?(Hash)
-    #
-    #         entry = Google::Logging::V1::LogEntry.new(
-    #           metadata: Google::Logging::V1::LogEntryMetadata.new(
-    #             service_name: @service_name.encode('utf-8'),
-    #             project_id: @project_id.encode('utf-8'),
-    #             zone: @zone.encode('utf-8'),
-    #             labels: {}
-    #           ))
-    #
-    #         if @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
-    #           @cloudfunctions_log_match =
-    #             @cloudfunctions_log_regexp.match(record['log'])
-    #         end
-    #         if @service_name == CONTAINER_SERVICE
-    #           # Move the stdout/stderr annotation from the record into a
-    #           # label.
-    #           field_to_label(record, 'stream', entry.metadata.labels,
-    #                          "#{CONTAINER_SERVICE}/stream")
-    #           # If the record has been annotated by the
-    #           # kubernetes_metadata_filter
-    #           # plugin, then use that metadata.  Otherwise, rely on
-    #           # commonLabels
-    #           # populated at the grouped_entries level from the group's tag.
-    #           if record.key?('kubernetes')
-    #             handle_container_metadata(record, entry)
-    #           end
-    #
-    #           # Save the timestamp if available, then clear it out to allow
-    #           # for determining whether we should parse the log or message
-    #           # field.
-    #           timestamp = record.key?('time') ? record['time'] : nil
-    #           record.delete('time')
-    #           # If the log is json, we want to export it as a structured log
-    #           # unless there is additional metadata that would be lost.
-    #           is_json = false
-    #           if record.length == 1 && record.key?('log')
-    #             record_json = parse_json_or_nil(record['log'])
-    #           end
-    #           if record.length == 1 && record.key?('message')
-    #             record_json = parse_json_or_nil(record['message'])
-    #           end
-    #           unless record_json.nil?
-    #             record = record_json
-    #             is_json = true
-    #           end
-    #           # Restore timestamp if necessary.
-    #           unless record.key?('time') || timestamp.nil?
-    #             record['time'] = timestamp
-    #           end
-    #         end
-    #
-    #         ts_secs, ts_nanos = compute_timestamp(record, time)
-    #         entry.metadata.timestamp = Google::Protobuf::Timestamp.new(
-    #           seconds: ts_secs,
-    #           nanos: ts_nanos
-    #         )
-    #
-    #         entry.metadata.severity =
-    #           grpc_severity(compute_severity(record, entry))
-    #
-    #         set_http_request_grpc(record, entry) # FIXME
-    #
-    #         # If a field is present in the label_map, send its value as a
-    #         # label
-    #         # (mapping the field name to label name as specified in the
-    #         # config)
-    #         # and do not send that field as part of the payload.
-    #         if @label_map
-    #           @label_map.each do |field, label|
-    #             field_to_label(record, field, entry.metadata.labels, label)
-    #           end
-    #         end
-    #
-    #         if @service_name == CLOUDFUNCTIONS_SERVICE &&
-    #            @cloudfunctions_log_match &&
-    #            @cloudfunctions_log_match['execution_id']
-    #           entry.metadata.labels['execution_id'] =
-    #             @cloudfunctions_log_match['execution_id']
-    #         end
-    #
-    #         set_payload_grpc(record, entry, is_json)
-    #
-    #         entries.push(entry)
-    #       end
-    #       # Don't send an empty request if we rejected all the entries.
-    #       next if entries.empty?
-    #
-    #       log_name = log_name(tag, labels)
-    #
-    #       begin
-    #         # Does the actual write to the cloud logging api.
-    #
-    #         client = api_client
-    #
-    #         labels_utf8_pairs = labels.map do |k, v|
-    #           [k.encode('utf-8'), v.encode('utf-8')]
-    #         end
-    #
-    #         write_request = Google::Logging::V1::WriteLogEntriesRequest.new(
-    #           log_name: log_name.encode('utf-8'),
-    #           common_labels: Hash[labels_utf8_pairs],
-    #           entries: entries
-    #         )
-    #
-    #         client.write_log_entries(write_request)
-    #
-    #         # Let the user explicitly know when the first call succeeded,
-    #         # to aid with verification and troubleshooting.
-    #         unless @successful_call
-    #           @successful_call = true
-    #           @log.info 'Successfully sent gRPC to Stackdriver Logging API.'
-    #         end
-    #
-    #       rescue GRPC::Cancelled => error
-    #         # RPC cancelled, so retry via re-raising the error.
-    #         raise error
-    #
-    #       rescue GRPC::BadStatus => error
-    #         case error.code
-    #         when GRPC::Core::StatusCodes::CANCELLED,
-    #              GRPC::Core::StatusCodes::UNAVAILABLE,
-    #              GRPC::Core::StatusCodes::DEADLINE_EXCEEDED,
-    #              GRPC::Core::StatusCodes::INTERNAL,
-    #              GRPC::Core::StatusCodes::UNKNOWN
-    #           # TODO
-    #           # Server error, so retry via re-raising the error.
-    #           raise error
-    #         when GRPC::Core::StatusCodes::UNIMPLEMENTED,
-    #              GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED
-    #           # Most client errors indicate a problem with the request itself
-    #           # and should not be retried.
-    #           dropped = entries.length
-    #           @log.warn "Dropping #{dropped} log message(s)",
-    #                     error: error.to_s, error_code: error.code.to_s
-    #         when GRPC::Core::StatusCodes::UNAUTHENTICATED
-    #           # Authorization error.
-    #           # These are usually solved via a `gcloud auth` call, or by
-    #           # modifying the permissions on the Google Cloud project.
-    #           dropped = entries.length
-    #           @log.warn "Dropping #{dropped} log message(s)",
-    #                     error: error.to_s, error_code: error.code.to_s
-    #         else
-    #           @log.error "Unknown response code #{error.code} from the " \
-    #                      "server",
-    #                      error: error.to_s, error_code: error.code.to_s
-    #         end
-    #       end
-    #     else
-    #       arr.each do |time, record|
-    #         next unless record.is_a?(Hash)
-    #
-    #         entry = Google::Apis::LoggingV1beta3::LogEntry.new(
-    #           metadata: Google::Apis::LoggingV1beta3::LogEntryMetadata.new(
-    #             service_name: @service_name,
-    #             project_id: @project_id,
-    #             zone: @zone,
-    #             labels: {}
-    #           ))
-    #
-    #         if @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
-    #           @cloudfunctions_log_match =
-    #             @cloudfunctions_log_regexp.match(record['log'])
-    #         end
-    #         if @service_name == CONTAINER_SERVICE
-    #           # Move the stdout/stderr annotation from the record into a label
-    #           field_to_label(record, 'stream', entry.metadata.labels,
-    #                          "#{CONTAINER_SERVICE}/stream")
-    #           # If the record has been annotated by the
-    #           # kubernetes_metadata_filter
-    #           # plugin, then use that metadata. Otherwise, rely on
-    #           # commonLabels
-    #           # populated at the grouped_entries level from the group's tag.
-    #           if record.key?('kubernetes')
-    #             handle_container_metadata(record, entry)
-    #           end
-    #
-    #           # Save the timestamp if available, then clear it out to allow
-    #           # for determining whether we should parse the log or message
-    #           # field.
-    #           timestamp = record.key?('time') ? record['time'] : nil
-    #           record.delete('time')
-    #           # If the log is json, we want to export it as a structured log
-    #           # unless there is additional metadata that would be lost.
-    #           is_json = false
-    #           if record.length == 1 && record.key?('log')
-    #             record_json = parse_json_or_nil(record['log'])
-    #           end
-    #           if record.length == 1 && record.key?('message')
-    #             record_json = parse_json_or_nil(record['message'])
-    #           end
-    #           unless record_json.nil?
-    #             record = record_json
-    #             is_json = true
-    #           end
-    #           # Restore timestamp if necessary.
-    #           unless record.key?('time') || timestamp.nil?
-    #             record['time'] = timestamp
-    #           end
-    #         end
-    #
-    #         ts_secs, ts_nanos = compute_timestamp(record, time)
-    #         entry.metadata.timestamp = {
-    #           seconds: ts_secs,
-    #           nanos: ts_nanos
-    #         }
-    #
-    #         entry.metadata.severity = compute_severity(record, entry)
-    #
-    #         set_http_request(record, entry)
-    #
-    #         # If a field is present in the label_map, send its value as a
-    #         # label
-    #         # (mapping the field name to label name as specified in the
-    #         # config)
-    #         # and do not send that field as part of the payload.
-    #         if @label_map
-    #           @label_map.each do |field, label|
-    #             field_to_label(record, field, entry.metadata.labels, label)
-    #           end
-    #         end
-    #
-    #         if @service_name == CLOUDFUNCTIONS_SERVICE &&
-    #            @cloudfunctions_log_match &&
-    #            @cloudfunctions_log_match['execution_id']
-    #           entry.metadata.labels['execution_id'] =
-    #             @cloudfunctions_log_match['execution_id']
-    #         end
-    #
-    #         set_payload(record, entry, is_json)
-    #         entry.metadata.labels = nil if entry.metadata.labels.empty?
-    #
-    #         entries.push(entry)
-    #       end
-    #       # Don't send an empty request if we rejected all the entries.
-    #       next if entries.empty?
-    #
-    #       log_name = log_name(tag, labels)
-    #
-    #       begin
-    #         # Does the actual write to the cloud logging api.
-    #         # The URI of the write is constructed by the Google::Api request;
-    #         # it is equivalent to this URL:
-    #         # 'https://logging.googleapis.com/v1beta3/projects/' \
-    #         #   "#{@project_id}/logs/#{log_name}/entries:write"
-    #
-    #         client = api_client
-    #
-    #         write_request = \
-    #           Google::Apis::LoggingV1beta3::WriteLogEntriesRequest.new(
-    #             common_labels: labels,
-    #             entries: entries)
-    #
-    #         # TODO: RequestOptions
-    #         client.write_log_entries(@project_id, log_name, write_request)
-    #
-    #         # Let the user explicitly know when the first call succeeded,
-    #         # to aid with verification and troubleshooting.
-    #         unless @successful_call
-    #           @successful_call = true
-    #           @log.info 'Successfully sent to Stackdriver Logging API.'
-    #         end
-    #
-    #       rescue Google::Apis::ServerError => error
-    #         # Server error, so retry via re-raising the error.
-    #         raise error
-    #
-    #       rescue Google::Apis::AuthorizationError => error
-    #         # Authorization error.
-    #         # These are usually solved via a `gcloud auth` call, or by
-    #         # modifying
-    #         # the permissions on the Google Cloud project.
-    #         dropped = entries.length
-    #         @log.warn "Dropping #{dropped} log message(s)",
-    #                   error_class: error.class.to_s, error: error.to_s
-    #
-    #       rescue Google::Apis::ClientError => error
-    #         # Most ClientErrors indicate a problem with the request itself and
-    #         # should not be retried.
-    #         dropped = entries.length
-    #         @log.warn "Dropping #{dropped} log message(s)",
-    #                   error_class: error.class.to_s, error: error.to_s
-    #       end
-    #     end
-    #   end
-    # end
-
     private
 
     def parse_json_or_nil(input)
@@ -1159,27 +827,32 @@ module Fluent
       return nil unless record['httpRequest'].is_a?(Hash)
       input = record['httpRequest']
       output = Google::Logging::Type::HttpRequest.new
-      output.request_method = input.delete('requestMethod') if
-        input.key?('requestMethod')
-      output.request_url = input.delete('requestUrl') if
-        input.key?('requestUrl')
-      output.request_size = input.delete('requestSize').to_i if
-        input.key?('requestSize')
-      output.status = input.delete('status').to_i if
-        input.key?('status')
-      output.response_size = input.delete('responseSize').to_i if
-        input.key?('responseSize')
-      output.user_agent = input.delete('userAgent') if
-        input.key?('userAgent')
-      output.remote_ip = input.delete('remoteIp') if
-        input.key?('remoteIp')
-      output.referer = input.delete('referer') if
-        input.key?('referer')
-      output.cache_hit = input.delete('cacheHit') if
-        input.key?('cacheHit')
+      # We need to delete each field from 'httpRequest' even if its value is
+      # nil. However we do not want to assign this nil value to proto fields
+      # defined as strings / integers.
+      request_method = input.delete('requestMethod')
+      output.request_method = request_method unless request_method.nil?
+      request_url = input.delete('requestUrl')
+      output.request_url = request_url unless request_url.nil?
+      request_size = input.delete('requestSize')
+      output.request_size = request_size.to_i unless request_size.nil?
+      status = input.delete('status')
+      output.status = status.to_i unless status.nil?
+      response_size = input.delete('responseSize')
+      output.response_size = response_size.to_i unless response_size.nil?
+      user_agent = input.delete('userAgent')
+      output.user_agent = user_agent unless user_agent.nil?
+      remote_ip = input.delete('remoteIp')
+      output.remote_ip = remote_ip unless remote_ip.nil?
+      referer = input.delete('referer')
+      output.referer = referer unless referer.nil?
+      cache_hit = input.delete('cacheHit')
+      output.cache_hit = cache_hit unless cache_hit.nil?
+      cache_validated_with_origin_server = \
+        input.delete('cacheValidatedWithOriginServer')
       output.cache_validated_with_origin_server = \
-        input.delete('cacheValidatedWithOriginServer') if
-        input.key?('cacheValidatedWithOriginServer')
+        cache_validated_with_origin_server \
+        unless cache_validated_with_origin_server.nil?
       record.delete('httpRequest') if input.empty?
       entry.http_request = output
     end

data/test/plugin/base_test.rb

@@ -307,6 +307,19 @@ module BaseTest
     }
   }
 
+  HTTP_REQUEST_MESSAGE = {
+    'requestMethod' => 'POST',
+    'requestUrl' => 'http://example/',
+    'requestSize' => 210,
+    'status' => 200,
+    'responseSize' => 65,
+    'userAgent' => 'USER AGENT 1.0',
+    'remoteIp' => '55.55.55.55',
+    'referer' => 'http://referer/',
+    'cacheHit' => true,
+    'cacheValidatedWithOriginServer' => true
+  }
+
   # Shared tests.
 
   def test_configure_service_account_application_default
@@ -517,15 +530,17 @@ module BaseTest
     setup_gce_metadata_stubs
     setup_logging_stubs do
       d = create_driver
-      d.emit('msg' => log_entry(0), 'tag2' => 'test', 'data' => 5000)
+      d.emit('msg' => log_entry(0), 'tag2' => 'test', 'data' => 5000,
+             'some_null_field' => nil)
       d.run
     end
     verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
       fields = get_fields(entry['structPayload'])
-      assert_equal 3, fields.size, entry
+      assert_equal 4, fields.size, entry
       assert_equal 'test log entry 0', get_string(fields['msg']), entry
       assert_equal 'test', get_string(fields['tag2']), entry
       assert_equal 5000, get_number(fields['data']), entry
+      assert_equal null_value, fields['some_null_field'], entry
     end
   end
 
@@ -550,7 +565,8 @@ module BaseTest
     setup_container_metadata_stubs
     setup_logging_stubs do
       d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
-      json_string = '{"msg": "test log entry 0", "tag2": "test", "data": 5000}'
+      json_string = '{"msg": "test log entry 0", "tag2": "test", ' \
+                    '"data": 5000, "some_null_field": null}'
       d.emit(container_log_entry_with_metadata('notJSON' + json_string))
       d.emit(container_log_entry_with_metadata(json_string))
       d.emit(container_log_entry_with_metadata("  \r\n \t" + json_string))
@@ -565,10 +581,11 @@ module BaseTest
       else
         assert entry.key?('structPayload'), 'Entry did not have structPayload'
         fields = get_fields(entry['structPayload'])
-        assert_equal 3, fields.size, entry
+        assert_equal 4, fields.size, entry
         assert_equal 'test log entry 0', get_string(fields['msg']), entry
         assert_equal 'test', get_string(fields['tag2']), entry
         assert_equal 5000, get_number(fields['data']), entry
+        assert_equal null_value, fields['some_null_field'], entry
       end
     end
   end
@@ -1025,20 +1042,6 @@ module BaseTest
     end
   end
 
-  def test_http_request_without_referer_from_record
-    setup_gce_metadata_stubs
-    setup_logging_stubs do
-      d = create_driver
-      d.emit('httpRequest' => http_request_message_without_referer)
-      d.run
-    end
-    verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
-      assert_equal http_request_message_without_referer, entry['httpRequest'],
-                   entry
-      assert_nil get_fields(entry['structPayload'])['httpRequest'], entry
-    end
-  end
-
   def test_http_request_when_not_hash
     setup_gce_metadata_stubs
     setup_logging_stubs do
@@ -1241,6 +1244,16 @@ module BaseTest
     assert i == n, "Number of entries #{i} does not match expected number #{n}"
   end
 
+  # The http request message to test against.
+  def http_request_message
+    HTTP_REQUEST_MESSAGE
+  end
+
+  # Replace the 'referer' field with nil.
+  def http_request_message_with_nil_referer
+    http_request_message.merge('referer' => nil)
+  end
+
   # This module expects the methods below to be overridden.
 
   # Create a Fluentd output test driver with the Google Cloud Output plugin.
@@ -1269,19 +1282,6 @@ module BaseTest
     _undefined
   end
 
-  # A wrapper around the constant HTTP_REQUEST_MESSAGE, so the definition can be
-  # skipped in the shared module here and defined in the test class later.
-  def http_request_message
-    _undefined
-  end
-
-  # A wrapper around the constant HTTP_REQUEST_MESSAGE_WITHOUT_REFERER, so the
-  # definition can be skipped in the shared module and defined in the test
-  # classes later.
-  def http_request_message_without_referer
-    _undefined
-  end
-
   # Get the fields of the struct payload.
   def get_fields(_struct_payload)
     _undefined
@@ -1302,6 +1302,11 @@ module BaseTest
     _undefined
   end
 
+  # The null value.
+  def null_value(_field)
+    _undefined
+  end
+
   def _undefined
     fail "Method #{__callee__} is unimplemented and needs to be overridden."
   end

data/test/plugin/test_out_google_cloud.rb

@@ -70,6 +70,24 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_equal 1, exception_count
   end
 
+  def test_http_request_from_record_with_referer_nil
+    setup_gce_metadata_stubs
+    setup_logging_stubs do
+      d = create_driver
+      d.emit('httpRequest' => http_request_message_with_nil_referer)
+      d.run
+    end
+    verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
+      # The request we send to Logging API has json like:
+      # "httpRequest": { "referer": null }, but eventually the stored LogEntry
+      # would be "httpRequest": {}, since 'referer' is defined as a string in
+      # the proto.
+      assert_equal http_request_message_with_nil_referer,
+                   entry['httpRequest'], entry
+      assert_nil get_fields(entry['structPayload'])['httpRequest'], entry
+    end
+  end
+
   # This test looks similar between the grpc and non-grpc paths except that when
   # parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
   # responds with "100".
@@ -175,25 +193,16 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
 
   private
 
-  # The non-grpc path has a unique field 'validatedWithOriginServer', while
-  # the grpc path has a unique field 'cacheValidatedWithOriginServer'.
-  HTTP_REQUEST_MESSAGE = {
-    'requestMethod' => 'POST',
-    'requestUrl' => 'http://example/',
-    'requestSize' => 210,
-    'status' => 200,
-    'responseSize' => 65,
-    'userAgent' => 'USER AGENT 1.0',
-    'remoteIp' => '55.55.55.55',
-    'referer' => 'http://referer/',
-    'cacheHit' => false,
-    'validatedWithOriginServer' => true
-  }
-
-  # In the non-grpc path 'referer' is nil, while in the grpc path 'referer' is
-  # absent.
-  HTTP_REQUEST_MESSAGE_WITHOUT_REFERER = HTTP_REQUEST_MESSAGE.merge(
-    'referer' => nil)
+  def rename_key(hash, old_key, new_key)
+    hash.merge(new_key => hash[old_key]).reject { |k, _| k == old_key }
+  end
+
+  # The REST path uses old bindings that were generated prior to the field
+  # rename, and has to use the old name, which is 'validatedWithOriginServer'.
+  def http_request_message
+    rename_key(super, 'cacheValidatedWithOriginServer',
+               'validatedWithOriginServer')
+  end
 
   # Set up http stubs to mock the external calls.
   def setup_logging_stubs
@@ -234,18 +243,6 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     end
   end
 
-  # A wrapper around the constant HTTP_REQUEST_MESSAGE, so the definition can be
-  # skipped in the shared module and defined here.
-  def http_request_message
-    HTTP_REQUEST_MESSAGE
-  end
-
-  # A wrapper around the constant HTTP_REQUEST_MESSAGE_WITHOUT_REFERER, so the
-  # definition can be skipped in the shared module and defined here.
-  def http_request_message_without_referer
-    HTTP_REQUEST_MESSAGE_WITHOUT_REFERER
-  end
-
   # Get the fields of the struct payload.
   def get_fields(struct_payload)
     struct_payload
@@ -265,4 +262,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   def get_number(field)
     field
   end
+
+  # The null value.
+  def null_value
+    nil
+  end
 end

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -72,6 +72,34 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
   end
 
+  def test_http_request_from_record_with_referer_nil
+    setup_gce_metadata_stubs
+    setup_logging_stubs do
+      d = create_driver
+      d.emit('httpRequest' => http_request_message_with_nil_referer)
+      d.run
+    end
+    verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
+      assert_equal http_request_message_with_absent_referer,
+                   entry['httpRequest'], entry
+      assert_nil get_fields(entry['structPayload'])['httpRequest'], entry
+    end
+  end
+
+  def test_http_request_from_record_with_referer_absent
+    setup_gce_metadata_stubs
+    setup_logging_stubs do
+      d = create_driver
+      d.emit('httpRequest' => http_request_message_with_absent_referer)
+      d.run
+    end
+    verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
+      assert_equal http_request_message_with_absent_referer,
+                   entry['httpRequest'], entry
+      assert_nil get_fields(entry['structPayload'])['httpRequest'], entry
+    end
+  end
+
   # This test looks similar between the grpc and non-grpc paths except that when
   # parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
   # responds with "100".
@@ -114,27 +142,6 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     use_grpc true
   )
 
-  # The non-grpc path has a unique field 'validatedWithOriginServer', while
-  # the grpc path has a unique field 'cacheValidatedWithOriginServer'.
-  HTTP_REQUEST_MESSAGE = {
-    'requestMethod' => 'POST',
-    'requestUrl' => 'http://example/',
-    'requestSize' => 210,
-    'status' => 200,
-    'responseSize' => 65,
-    'userAgent' => 'USER AGENT 1.0',
-    'remoteIp' => '55.55.55.55',
-    'referer' => 'http://referer/',
-    'cacheHit' => true,
-    'cacheValidatedWithOriginServer' => true
-  }
-
-  # In the non-grpc path 'referer' is nil, while in the grpc path 'referer' is
-  # absent.
-  HTTP_REQUEST_MESSAGE_WITHOUT_REFERER = HTTP_REQUEST_MESSAGE.reject do |k, _|
-    k == 'referer'
-  end
-
   # Create a Fluentd output test driver with the Google Cloud Output plugin with
   # grpc enabled. The signature of this method is different between the grpc
   # path and the non-grpc path. For grpc, an additional grpc stub class can be
@@ -275,16 +282,11 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
   end
 
-  # A wrapper around the constant HTTP_REQUEST_MESSAGE, so the definition can be
-  # skipped in the shared module and defined here.
-  def http_request_message
-    HTTP_REQUEST_MESSAGE
-  end
-
-  # A wrapper around the constant HTTP_REQUEST_MESSAGE_WITHOUT_REFERER, so the
-  # definition can be skipped in the shared module and defined here.
-  def http_request_message_without_referer
-    HTTP_REQUEST_MESSAGE_WITHOUT_REFERER
+  # Unset the 'referer' field.
+  def http_request_message_with_absent_referer
+    http_request_message.reject do |k, _|
+      k == 'referer'
+    end
   end
 
   # Get the fields of the struct payload.
@@ -306,4 +308,9 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
   def get_number(field)
     field['numberValue']
   end
+
+  # The null value.
+  def null_value
+    { 'nullValue' => 'NULL_VALUE' }
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.5.3.grpc.alpha.4
+  version: 0.5.3.grpc.alpha.5
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-12 00:00:00.000000000 Z
+date: 2016-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd