fluent-plugin-google-cloud 0.7.2 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d9571be109300197bc432ba7f561345addc0c2e9b477b676a05c0a56f661b46
-  data.tar.gz: b18d90aa70e5b25876fbe64912096225c21ad52e3bb81ad056fb0f5938d4efa4
+  metadata.gz: fbcd8e43170062fe787c54619c5ad959f0a8d1603fb7dfc2a1c2be5ac7997d4e
+  data.tar.gz: d22989b9d59f470ecc3e184bff6bd1bc049f66f0d0e3918a649f36e2cbbd0424
 SHA512:
-  metadata.gz: c64ce586b924a3900ea389ab424cf5027e2e5df3703d23afde7f3076c805acc9da6c382256f722ba463e5af64f2dd8f4a311457ceb6276c76c203d2d115b3b8f
-  data.tar.gz: 144524c87fb2d803591d57fa0f62543a49121a4028ac2cba0efb1cf45a1f7a19a5ae094b0dc3d56651609efc36f143fe0e9aadf02700b88c93d39f0c353780f5
+  metadata.gz: b5b5e63ddf4cc5bffcc92ae9a89dc47b25570c4108c5fde3a34a9c3a77b8b15dbeb2832e55295a5b44505fb2b7b3919a521ab89f847642427728f8c698537a1e
+  data.tar.gz: 19554fb0fc748dc0a246850053874dcc53dbc7264d4d75d321e0a64c812f2beb887616945d73f104090aae92656386dde464f87b52ffc9e6c745ad72c10d2bf3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.7.2)
+    fluent-plugin-google-cloud (0.7.3)
       fluentd (= 1.2.5)
       google-api-client (= 0.23.9)
       google-cloud-logging (= 1.5.4)
@@ -129,7 +129,7 @@ GEM
     stackdriver-core (1.3.3)
       google-cloud-core (~> 1.2)
     strptime (0.2.3)
-    test-unit (3.2.8)
+    test-unit (3.2.9)
       power_assert
     thread_safe (0.3.6)
     tzinfo (1.2.5)
@@ -137,7 +137,7 @@ GEM
     tzinfo-data (1.2018.7)
       tzinfo (>= 1.0.0)
     uber (0.1.0)
-    unicode-display_width (1.4.0)
+    unicode-display_width (1.4.1)
     webmock (2.3.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.7.2'
+  gem.version       = '0.7.3'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')

data/lib/fluent/plugin/out_google_cloud.rb

@@ -186,6 +186,7 @@ module Fluent
             %w(responseSize response_size parse_int),
             %w(userAgent user_agent parse_string),
             %w(remoteIp remote_ip parse_string),
+            %w(serverIp server_ip parse_string),
             %w(referer referer parse_string),
             %w(cacheHit cache_hit parse_bool),
             %w(cacheValidatedWithOriginServer
@@ -609,7 +610,14 @@ module Fluent
             # allow for determining whether we should parse the log or message
             # field.
             preserved_keys = [
-              'time', 'severity', @trace_key, @span_id_key, @insert_id_key
+              'time',
+              'severity',
+              @http_request_key,
+              @insert_id_key,
+              @operation_key,
+              @source_location_key,
+              @span_id_key,
+              @trace_key
             ]
 
             # If the log is json, we want to export it as a structured log

data/test/plugin/base_test.rb

@@ -612,6 +612,30 @@ module BaseTest
       end
   end
 
+  # Verify that when the log has only one effective field (named 'log',
+  # 'message', or 'msg') and the field is in JSON format, the field is parsed as
+  # JSON and sent as jsonPayload.
+  def test_detect_json_auto_triggered_with_one_field
+    setup_gce_metadata_stubs
+    json_string = '{"msg": "test log entry 0", "tag2": "test", ' \
+                  '"data": 5000, "some_null_field": null}'
+    setup_logging_stubs do
+      d = create_driver(DETECT_JSON_CONFIG)
+      %w(message log msg).each do |field|
+        d.emit(PRESERVED_KEYS_MAP.merge(field => json_string))
+      end
+      d.run
+    end
+    verify_log_entries(3, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+      fields = get_fields(entry['jsonPayload'])
+      assert_equal 4, fields.size, entry
+      assert_equal 'test log entry 0', get_string(fields['msg']), entry
+      assert_equal 'test', get_string(fields['tag2']), entry
+      assert_equal 5000, get_number(fields['data']), entry
+      assert_equal null_value, fields['some_null_field'], entry
+    end
+  end
+
   # Verify that we drop the log entries when 'require_valid_tags' is true and
   # any non-string tags or tags with non-utf8 characters are detected.
   def test_reject_invalid_tags_with_require_valid_tags_true
@@ -1219,6 +1243,8 @@ module BaseTest
     verify_log_entries(1, DATAFLOW_PARAMS)
   end
 
+  # Verify the subfields extraction of LogEntry fields.
+
   def test_log_entry_http_request_field_from_record
     verify_subfields_from_record(DEFAULT_HTTP_REQUEST_KEY)
   end
@@ -1231,6 +1257,9 @@ module BaseTest
     verify_subfields_from_record(DEFAULT_OPERATION_KEY)
   end
 
+  # Verify the subfields extraction of LogEntry fields when there are other
+  # fields.
+
   def test_log_entry_http_request_field_partial_from_record
     verify_subfields_partial_from_record(DEFAULT_HTTP_REQUEST_KEY)
   end
@@ -1243,6 +1272,8 @@ module BaseTest
     verify_subfields_partial_from_record(DEFAULT_OPERATION_KEY)
   end
 
+  # Verify the subfields extraction of LogEntry fields when they are not hashes.
+
   def test_log_entry_http_request_field_when_not_hash
     verify_subfields_when_not_hash(DEFAULT_HTTP_REQUEST_KEY)
   end
@@ -1255,6 +1286,8 @@ module BaseTest
     verify_subfields_when_not_hash(DEFAULT_OPERATION_KEY)
   end
 
+  # Verify the subfields extraction of LogEntry fields when they are nil.
+
   def test_log_entry_http_request_field_when_nil
     verify_subfields_when_nil(DEFAULT_HTTP_REQUEST_KEY)
   end
@@ -1324,9 +1357,23 @@ module BaseTest
     end
   end
 
-  def test_log_entry_trace_field
-    verify_field_key('trace', DEFAULT_TRACE_KEY, 'custom_trace_key',
-                     CONFIG_CUSTOM_TRACE_KEY_SPECIFIED, TRACE)
+  # Verify the default and customization of LogEntry field extraction key.
+
+  def test_log_entry_insert_id_field
+    verify_field_key('insertId', DEFAULT_INSERT_ID_KEY, 'custom_insert_id_key',
+                     CONFIG_CUSTOM_INSERT_ID_KEY_SPECIFIED, INSERT_ID)
+  end
+
+  def test_log_entry_operation_field
+    verify_field_key('operation', DEFAULT_OPERATION_KEY, 'custom_operation_key',
+                     CONFIG_CUSTOM_OPERATION_KEY_SPECIFIED, OPERATION_MESSAGE)
+  end
+
+  def test_log_entry_source_location_field
+    verify_field_key('sourceLocation', DEFAULT_SOURCE_LOCATION_KEY,
+                     'custom_source_location_key',
+                     CONFIG_CUSTOM_SOURCE_LOCATION_KEY_SPECIFIED,
+                     source_location_message)
   end
 
   def test_log_entry_span_id_field
@@ -1334,24 +1381,47 @@ module BaseTest
                      CONFIG_CUSTOM_SPAN_ID_KEY_SPECIFIED, SPAN_ID)
   end
 
-  def test_log_entry_insert_id_field
-    verify_field_key('insertId', DEFAULT_INSERT_ID_KEY, 'custom_insert_id_key',
-                     CONFIG_CUSTOM_INSERT_ID_KEY_SPECIFIED, INSERT_ID)
+  def test_log_entry_trace_field
+    verify_field_key('trace', DEFAULT_TRACE_KEY, 'custom_trace_key',
+                     CONFIG_CUSTOM_TRACE_KEY_SPECIFIED, TRACE)
   end
 
-  def test_cascading_json_detection_with_log_entry_trace_field
+  # Verify the cascading JSON detection of LogEntry fields.
+
+  def test_cascading_json_detection_with_log_entry_insert_id_field
     verify_cascading_json_detection_with_log_entry_fields(
-      'trace', DEFAULT_TRACE_KEY, TRACE, TRACE2)
+      'insertId', DEFAULT_INSERT_ID_KEY,
+      root_level_value: INSERT_ID,
+      nested_level_value: INSERT_ID2)
+  end
+
+  def test_cascading_json_detection_with_log_entry_operation_field
+    verify_cascading_json_detection_with_log_entry_fields(
+      'operation', DEFAULT_OPERATION_KEY,
+      root_level_value: OPERATION_MESSAGE,
+      nested_level_value: OPERATION_MESSAGE2,
+      expected_value_from_nested: expected_operation_message2)
+  end
+
+  def test_cascading_json_detection_with_log_entry_source_location_field
+    verify_cascading_json_detection_with_log_entry_fields(
+      'sourceLocation', DEFAULT_SOURCE_LOCATION_KEY,
+      root_level_value: source_location_message,
+      nested_level_value: source_location_message2)
   end
 
   def test_cascading_json_detection_with_log_entry_span_id_field
     verify_cascading_json_detection_with_log_entry_fields(
-      'spanId', DEFAULT_SPAN_ID_KEY, SPAN_ID, SPAN_ID2)
+      'spanId', DEFAULT_SPAN_ID_KEY,
+      root_level_value: SPAN_ID,
+      nested_level_value: SPAN_ID2)
   end
 
-  def test_cascading_json_detection_with_log_entry_insert_id_field
+  def test_cascading_json_detection_with_log_entry_trace_field
     verify_cascading_json_detection_with_log_entry_fields(
-      'insertId', DEFAULT_INSERT_ID_KEY, INSERT_ID, INSERT_ID2)
+      'trace', DEFAULT_TRACE_KEY,
+      root_level_value: TRACE,
+      nested_level_value: TRACE2)
   end
 
   # Metadata Agent related tests.
@@ -2177,7 +2247,14 @@ module BaseTest
   # left with name "log", "message" or "msg". This test verifies additional
   # LogEntry fields like spanId and traceId do not disable that by accident.
   def verify_cascading_json_detection_with_log_entry_fields(
-      log_entry_field, default_key, root_level_value, nested_level_value)
+      log_entry_field, default_key, expectation)
+    root_level_value = expectation[:root_level_value]
+    nested_level_value = expectation[:nested_level_value]
+    expected_value_from_root = expectation.fetch(
+      :expected_value_from_root, root_level_value)
+    expected_value_from_nested = expectation.fetch(
+      :expected_value_from_nested, nested_level_value)
+
     setup_gce_metadata_stubs
 
     # {
@@ -2215,9 +2292,9 @@ module BaseTest
       default_key => root_level_value)
 
     {
-      log_entry_with_root_level_field => root_level_value,
-      log_entry_with_nested_level_field => nested_level_value,
-      log_entry_with_both_level_fields => nested_level_value
+      log_entry_with_root_level_field => expected_value_from_root,
+      log_entry_with_nested_level_field => expected_value_from_nested,
+      log_entry_with_both_level_fields => expected_value_from_nested
     }.each_with_index do |(input_log_entry, expected_value), index|
       setup_logging_stubs do
         @logs_sent = []
@@ -2283,20 +2360,12 @@ module BaseTest
         payload_fields = get_fields(entry['jsonPayload'])
         assert_equal input[:expected_payload].size, payload_fields.size, input
         payload_fields.each do |key, value|
-          assert_equal input[:expected_payload][key], get_string(value), input
+          assert_hash_equal_json(input[:expected_payload][key], value)
         end
       end
     end
   end
 
-  def http_request_message
-    HTTP_REQUEST_MESSAGE
-  end
-
-  def source_location_message
-    SOURCE_LOCATION_MESSAGE
-  end
-
   # Replace the 'referer' field with nil.
   def http_request_message_with_nil_referer
     http_request_message.merge('referer' => nil)
@@ -2395,6 +2464,31 @@ module BaseTest
     _undefined
   end
 
+  # Defined in specific gRPC or REST files.
+  def http_request_message
+    _undefined
+  end
+
+  # Defined in specific gRPC or REST files.
+  def source_location_message
+    _undefined
+  end
+
+  # Defined in specific gRPC or REST files.
+  def source_location_message2
+    _undefined
+  end
+
+  # Defined in specific gRPC or REST files.
+  def expected_operation_message2
+    _undefined
+  end
+
+  # Defined in specific gRPC or REST files.
+  def assert_hash_equal_json(_expected, _actual)
+    _undefined
+  end
+
   def _undefined
     raise "Method #{__callee__} is unimplemented and needs to be overridden."
   end

data/test/plugin/constants.rb

@@ -371,16 +371,24 @@ module Constants
     label_map { "name": "#{ML_CONSTANTS[:service]}/job_id/log_area" }
   ).freeze
 
-  CONFIG_CUSTOM_TRACE_KEY_SPECIFIED = %(
-    trace_key custom_trace_key
+  CONFIG_CUSTOM_INSERT_ID_KEY_SPECIFIED = %(
+    insert_id_key custom_insert_id_key
+  ).freeze
+
+  CONFIG_CUSTOM_OPERATION_KEY_SPECIFIED = %(
+    operation_key custom_operation_key
+  ).freeze
+
+  CONFIG_CUSTOM_SOURCE_LOCATION_KEY_SPECIFIED = %(
+    source_location_key custom_source_location_key
   ).freeze
 
   CONFIG_CUSTOM_SPAN_ID_KEY_SPECIFIED = %(
     span_id_key custom_span_id_key
   ).freeze
 
-  CONFIG_CUSTOM_INSERT_ID_KEY_SPECIFIED = %(
-    insert_id_key custom_insert_id_key
+  CONFIG_CUSTOM_TRACE_KEY_SPECIFIED = %(
+    trace_key custom_trace_key
   ).freeze
 
   # Service configurations for various services.
@@ -740,6 +748,7 @@ module Constants
     'responseSize' => 65,
     'userAgent' => 'USER AGENT 1.0',
     'remoteIp' => '55.55.55.55',
+    'serverIp' => '66.66.66.66',
     'referer' => 'http://referer/',
     'cacheHit' => true,
     'cacheValidatedWithOriginServer' => true
@@ -751,12 +760,24 @@ module Constants
     'line' => 18
   }.freeze
 
+  SOURCE_LOCATION_MESSAGE2 = {
+    'file' => 'src/file',
+    'function' => 'my_func',
+    'line' => 8
+  }.freeze
+
   OPERATION_MESSAGE = {
     'id' => 'op_id',
     'producer' => 'my/app',
     'last' => true
   }.freeze
 
+  OPERATION_MESSAGE2 = {
+    'id' => 'op_id2',
+    'producer' => 'my/app2',
+    'last' => false
+  }.freeze
+
   CUSTOM_LABELS_MESSAGE = {
     'customKey' => 'value'
   }.freeze
@@ -950,4 +971,15 @@ module Constants
       'grpc-status-details-bin' => status_details.to_proto
     }.freeze
   end
+
+  PRESERVED_KEYS_MAP = {
+    'time' => K8S_TIMESTAMP,
+    'severity' => CONTAINER_SEVERITY,
+    DEFAULT_HTTP_REQUEST_KEY => HTTP_REQUEST_MESSAGE,
+    DEFAULT_INSERT_ID_KEY => INSERT_ID,
+    DEFAULT_OPERATION_KEY => OPERATION_MESSAGE,
+    DEFAULT_SOURCE_LOCATION_KEY => SOURCE_LOCATION_MESSAGE,
+    DEFAULT_SPAN_ID_KEY => SPAN_ID,
+    DEFAULT_TRACE_KEY => TRACE
+  }.freeze
 end

data/test/plugin/test_out_google_cloud.rb

@@ -407,24 +407,44 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     nil
   end
 
-  # 'responseSize' and 'requestSize' are Integers in the gRPC proto, yet Strings
-  # in REST API client.
-  # TODO(qingling128): Address this accordingly once the following question is
-  # answered: https://github.com/google/google-api-ruby-client/issues/619.
-  # If this discrepancy is legit, add some comments to explain the reason.
-  # Otherwise once the discrepancy is fixed, we need to upgrade to that version
-  # and change our tests accordingly.
+  # Convert certain fields to strings for compatibility between gRPC and REST.
+  # See more details in:
+  # https://github.com/google/google-api-ruby-client/issues/619.
+  def convert_subfields_to_strings(full_hash, fields_to_convert)
+    full_hash.merge(Hash[
+      fields_to_convert.collect do |field_name|
+        [field_name, full_hash[field_name].to_s]
+      end
+    ])
+  end
+
+  # 'responseSize' and 'requestSize' are Integers in the gRPC protos, yet
+  # Strings in REST API client libraries.
   def http_request_message
-    HTTP_REQUEST_MESSAGE.merge(
-      'responseSize' => HTTP_REQUEST_MESSAGE['responseSize'].to_s,
-      'requestSize' => HTTP_REQUEST_MESSAGE['requestSize'].to_s
-    )
+    convert_subfields_to_strings(
+      HTTP_REQUEST_MESSAGE, %w(responseSize requestSize))
   end
 
   # 'line' is an Integer in the gRPC proto, yet a String in the REST API client.
   def source_location_message
-    SOURCE_LOCATION_MESSAGE.merge(
-      'line' => SOURCE_LOCATION_MESSAGE['line'].to_s
-    )
+    convert_subfields_to_strings(
+      SOURCE_LOCATION_MESSAGE, ['line'])
+  end
+
+  # 'line' is an Integer in the gRPC proto, yet a String in the REST API client.
+  def source_location_message2
+    convert_subfields_to_strings(
+      SOURCE_LOCATION_MESSAGE2, ['line'])
+  end
+
+  def expected_operation_message2
+    OPERATION_MESSAGE2
+  end
+
+  # Both expected and actual are Ruby hashes that represent JSON
+  # objects.
+  # This method has a different implementation at the gRPC side.
+  def assert_hash_equal_json(expected, actual)
+    assert_equal expected, actual, "expected: #{expected}\nactual: #{actual}"
   end
 end

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -466,8 +466,79 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     field['numberValue']
   end
 
+  def get_bool(field)
+    field['boolValue']
+  end
+
   # The null value.
   def null_value
     { 'nullValue' => 'NULL_VALUE' }
   end
+
+  def http_request_message
+    HTTP_REQUEST_MESSAGE
+  end
+
+  def source_location_message
+    SOURCE_LOCATION_MESSAGE
+  end
+
+  def source_location_message2
+    SOURCE_LOCATION_MESSAGE2
+  end
+
+  def expected_operation_message2
+    # 'last' is a boolean field with false as default value. Protobuf omit
+    # fields with default values during deserialization.
+    OPERATION_MESSAGE2.reject { |k, _| k == 'last' }
+  end
+
+  # expected: A Ruby hash that represents a JSON object.
+  # e.g.:
+  #   {
+  #     "file" => "source/file",
+  #     "function" => "my_function",
+  #     "line" => 18
+  #   }
+  #
+  # actual: A Ruby hash that represents a Proto object.
+  # e.g.:
+  #   {
+  #     "structValue" => {
+  #       "fields" => {
+  #         "file" => {
+  #           "stringValue" => "source/file"
+  #         },
+  #         "function" => {
+  #           "stringValue" => "my_function"
+  #         },
+  #         "line" => {
+  #           "numberValue" => 18
+  #         }
+  #       }
+  #     }
+  #   }
+  # This method has a different implementation at the REST side.
+  def assert_hash_equal_json(expected, actual)
+    error_message = "expected: #{expected}\nactual: #{actual}"
+    assert_true actual.is_a?(Hash),
+                "Expect the actual value to be a hash. #{error_message}"
+    if actual.key?('stringValue')
+      assert_equal expected, get_string(actual), error_message
+    elsif actual.key?('numberValue')
+      assert_equal expected, get_number(actual), error_message
+    elsif actual.key?('boolValue')
+      assert_equal expected, get_bool(actual), error_message
+    elsif actual.key?('structValue')
+      expected_copy = expected.dup
+      get_fields(get_struct(actual)).each do |field_name, nested_actual|
+        assert_hash_equal_json expected_copy[field_name], nested_actual
+        expected_copy.reject! { |k, _| k == field_name }
+      end
+      # Make sure all fields are matched.
+      assert_true expected_copy.empty?
+    else
+      assert_true false, "Unsupported proto format. #{error_message}"
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.3
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-30 00:00:00.000000000 Z
+date: 2018-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd