fluent-plugin-google-cloud 0.7.15 → 0.7.16

data/test/plugin/constants.rb

@@ -464,6 +464,53 @@ module Constants
     #{CONFIG_LABEL_MAP_CONFLICTING}
   ).freeze
 
+  # For monitoring config.
+  CONFIG_UNKNOWN_MONITORING_TYPE = %(
+    enable_monitoring  true
+    monitoring_type    not_prometheus
+  ).freeze
+
+  # For statusz.
+  CONFIG_STATUSZ = %(
+    statusz_port                  5678
+
+    adjust_invalid_timestamps     false
+    autoformat_stackdriver_trace  false
+    coerce_to_utf8                false
+    detect_json                   true
+    detect_subservice             false
+    enable_metadata_agent         true
+    enable_monitoring             true
+    http_request_key              test_http_request_key
+    insert_id_key                 test_insert_id_key
+    k8s_cluster_location          test-k8s-cluster-location
+    k8s_cluster_name              test-k8s-cluster-name
+    kubernetes_tag_regexp         .*test-regexp.*
+    label_map                     { "label_map_key": "label_map_value" }
+    labels_key                    test_labels_key
+    labels                        { "labels_key": "labels_value" }
+    logging_api_url               http://localhost:52000
+    metadata_agent_url            http://localhost:12345
+    monitoring_type               not_prometheus
+    non_utf8_replacement_string   zzz
+    operation_key                 test_operation_key
+    partial_success               false
+    project_id                    test-project-id-123
+    require_valid_tags            true
+    source_location_key           test_source_location_key
+    span_id_key                   test_span_id_key
+    split_logs_by_tag             true
+    subservice_name               test_subservice_name
+    trace_key                     test_trace_key
+    trace_sampled_key             test_trace_sampled_key
+    use_aws_availability_zone     false
+    use_grpc                      true
+    use_metadata_service          false
+    vm_id                         12345
+    vm_name                       test.hostname.org
+    zone                          asia-east2
+  ).freeze
+
   # Service configurations for various services.
 
   # GCE.
@@ -798,16 +845,16 @@ module Constants
   ).freeze
 
   HTTP_REQUEST_MESSAGE = {
-    'cacheFillBytes' => 6653,
+    'cacheFillBytes' => '6653',
     'cacheHit' => true,
     'cacheLookup' => true,
     'cacheValidatedWithOriginServer' => true,
     'protocol' => 'HTTP/1.1',
     'referer' => 'http://referer/',
     'remoteIp' => '55.55.55.55',
-    'responseSize' => 65,
+    'responseSize' => '65',
     'requestMethod' => 'POST',
-    'requestSize' => 210,
+    'requestSize' => '210',
     'requestUrl' => 'http://example/',
     'serverIp' => '66.66.66.66',
     'status' => 200,
@@ -817,13 +864,13 @@ module Constants
   SOURCE_LOCATION_MESSAGE = {
     'file' => 'source/file',
     'function' => 'my_function',
-    'line' => 18
+    'line' => '18'
   }.freeze
 
   SOURCE_LOCATION_MESSAGE2 = {
     'file' => 'src/file',
     'function' => 'my_func',
-    'line' => 8
+    'line' => '8'
   }.freeze
 
   OPERATION_MESSAGE = {
@@ -1056,8 +1103,26 @@ module Constants
     }.freeze
   end
 
+  PRESERVED_KEYS_TIMESTAMP_FIELDS = [
+    {
+      'time' => K8S_TIMESTAMP
+    },
+    {
+      'timeNanos' => K8S_NANOS
+    },
+    {
+      'timestamp' => {
+        'nanos' => K8S_NANOS,
+        'seconds' => K8S_SECONDS_EPOCH
+      }
+    },
+    {
+      'timestampNanos' => K8S_NANOS,
+      'timestampSeconds' => K8S_SECONDS_EPOCH
+    }
+  ].freeze
+
   PRESERVED_KEYS_MAP = {
-    'time' => K8S_TIMESTAMP,
     'severity' => CONTAINER_SEVERITY,
     DEFAULT_HTTP_REQUEST_KEY => HTTP_REQUEST_MESSAGE,
     DEFAULT_INSERT_ID_KEY => INSERT_ID,

data/test/plugin/test_out_google_cloud.rb

@@ -339,10 +339,54 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     WebMock.disable_net_connect!(allow_localhost: true)
     # TODO(davidbtucker): Consider searching for an unused port
     # instead of hardcoding a constant here.
-    d = create_driver('statusz_port 5678')
+    d = create_driver(CONFIG_STATUSZ)
     d.run do
-      assert_match Regexp.new('.*<h1>Status</h1>.*'),
-                   Net::HTTP.get('127.0.0.1', '/statusz', 5678)
+      resp = Net::HTTP.get('127.0.0.1', '/statusz', 5678)
+      must_match = [
+        '<h1>Status for .*</h1>.*',
+
+        '\badjust_invalid_timestamps\b.*\bfalse\b',
+        '\bautoformat_stackdriver_trace\b.*\bfalse\b',
+        '\bcoerce_to_utf8\b.*\bfalse\b',
+        '\bdetect_json\b.*\btrue\b',
+        '\bdetect_subservice\b.*\bfalse\b',
+        '\benable_metadata_agent\b.*\btrue\b',
+        '\benable_monitoring\b.*\btrue\b',
+        '\bhttp_request_key\b.*\btest_http_request_key\b',
+        '\binsert_id_key\b.*\btest_insert_id_key\b',
+        '\bk8s_cluster_location\b.*\btest-k8s-cluster-location\b',
+        '\bk8s_cluster_name\b.*\btest-k8s-cluster-name\b',
+        '\bkubernetes_tag_regexp\b.*\b.*test-regexp.*\b',
+        '\blabel_map\b.*{"label_map_key"=>"label_map_value"}',
+        '\blabels_key\b.*\btest_labels_key\b',
+        '\blabels\b.*{"labels_key"=>"labels_value"}',
+        '\blogging_api_url\b.*\bhttp://localhost:52000\b',
+        '\bmetadata_agent_url\b.*\bhttp://localhost:12345\b',
+        '\bmonitoring_type\b.*\bnot_prometheus\b',
+        '\bnon_utf8_replacement_string\b.*\bzzz\b',
+        '\boperation_key\b.*\btest_operation_key\b',
+        '\bpartial_success\b.*\bfalse\b',
+        '\bproject_id\b.*\btest-project-id-123\b',
+        '\brequire_valid_tags\b.*\btrue\b',
+        '\bsource_location_key\b.*\btest_source_location_key\b',
+        '\bspan_id_key\b.*\btest_span_id_key\b',
+        '\bsplit_logs_by_tag\b.*\btrue\b',
+        '\bstatusz_port\b.*\b5678\b',
+        '\bsubservice_name\b.*\btest_subservice_name\b',
+        '\btrace_key\b.*\btest_trace_key\b',
+        '\btrace_sampled_key\b.*\btest_trace_sampled_key\b',
+        '\buse_aws_availability_zone\b.*\bfalse\b',
+        '\buse_grpc\b.*\btrue\b',
+        '\buse_metadata_service\b.*\bfalse\b',
+        '\bvm_id\b.*\b12345\b',
+        '\bvm_name\b.*\btest.hostname.org\b',
+        '\bzone\b.*\basia-east2\b',
+
+        '^</html>$'
+      ]
+      must_match.each do |re|
+        assert_match Regexp.new(re), resp
+      end
     end
   end
 
@@ -364,6 +408,27 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     yield
   end
 
+  # The conversions from user input to output.
+  def latency_conversion
+    {
+      '32 s' => { 'seconds' => 32 },
+      '32s' => { 'seconds' => 32 },
+      '0.32s' => { 'nanos' => 320_000_000 },
+      ' 123 s ' => { 'seconds' => 123 },
+      '1.3442 s' => { 'seconds' => 1, 'nanos' => 344_200_000 },
+
+      # Test whitespace.
+      # \t: tab. \r: carriage return. \n: line break.
+      # \v: vertical whitespace. \f: form feed.
+      "\t123.5\ts\t" => { 'seconds' => 123, 'nanos' => 500_000_000 },
+      "\r123.5\rs\r" => { 'seconds' => 123, 'nanos' => 500_000_000 },
+      "\n123.5\ns\n" => { 'seconds' => 123, 'nanos' => 500_000_000 },
+      "\v123.5\vs\v" => { 'seconds' => 123, 'nanos' => 500_000_000 },
+      "\f123.5\fs\f" => { 'seconds' => 123, 'nanos' => 500_000_000 },
+      "\r123.5\ts\f" => { 'seconds' => 123, 'nanos' => 500_000_000 }
+    }
+  end
+
   # Create a Fluentd output test driver with the Google Cloud Output plugin.
   def create_driver(conf = APPLICATION_DEFAULT_CONFIG,
                     tag = 'test',
@@ -400,69 +465,13 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     end
   end
 
-  # Get the fields of the payload.
-  def get_fields(payload)
-    payload
-  end
-
-  # Get the value of a struct field.
-  def get_struct(field)
-    field
-  end
-
-  # Get the value of a string field.
-  def get_string(field)
-    field
-  end
-
-  # Get the value of a number field.
-  def get_number(field)
-    field
-  end
-
-  # The null value.
-  def null_value
-    nil
-  end
-
-  # Convert certain fields to strings for compatibility between gRPC and REST.
-  # See more details in:
-  # https://github.com/google/google-api-ruby-client/issues/619.
-  def convert_subfields_to_strings(full_hash, fields_to_convert)
-    full_hash.merge(Hash[
-      fields_to_convert.collect do |field_name|
-        [field_name, full_hash[field_name].to_s]
-      end
-    ])
-  end
-
-  # 'responseSize', 'requestSize', and 'cacheFillBytes' are Integers in the gRPC
-  # protos, yet Strings in REST API client libraries.
-  def http_request_message
-    convert_subfields_to_strings(
-      HTTP_REQUEST_MESSAGE, %w(cacheFillBytes responseSize requestSize))
-  end
-
-  # 'line' is an Integer in the gRPC proto, yet a String in the REST API client.
-  def source_location_message
-    convert_subfields_to_strings(
-      SOURCE_LOCATION_MESSAGE, ['line'])
-  end
-
-  # 'line' is an Integer in the gRPC proto, yet a String in the REST API client.
-  def source_location_message2
-    convert_subfields_to_strings(
-      SOURCE_LOCATION_MESSAGE2, ['line'])
-  end
-
   def expected_operation_message2
     OPERATION_MESSAGE2
   end
 
-  # Both expected and actual are Ruby hashes that represent JSON
-  # objects.
-  # This method has a different implementation at the gRPC side.
-  def assert_hash_equal_json(expected, actual)
-    assert_equal expected, actual, "expected: #{expected}\nactual: #{actual}"
+  # Directly return the timestamp value, which should be a hash two keys:
+  # "seconds" and "nanos".
+  def timestamp_parse(timestamp)
+    timestamp
   end
 end

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -260,6 +260,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
   end
 
+  # TODO(qingling128): Verify if we need this on the REST side and add it if
+  # needed.
   def test_struct_payload_non_utf8_log
     setup_gce_metadata_stubs
     setup_logging_stubs do
@@ -273,14 +275,14 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
       d.run
     end
     verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
-      fields = get_fields(entry['jsonPayload'])
+      fields = entry['jsonPayload']
       assert_equal 5, fields.size, entry
-      assert_equal 'test log entry 0', get_string(fields['msg']), entry
-      assert_equal 'test non utf8', get_string(fields['normal_key']), entry
-      assert_equal 5000, get_number(fields['non_utf8 key']), entry
-      assert_equal 'test non utf8', get_string(get_fields(get_struct(fields \
-                   ['nested_struct']))['non_utf8 key']), entry
-      assert_equal null_value, fields['null_field'], entry
+      assert_equal 'test log entry 0', fields['msg'], entry
+      assert_equal 'test non utf8', fields['normal_key'], entry
+      assert_equal 5000, fields['non_utf8 key'], entry
+      assert_equal 'test non utf8', fields['nested_struct']['non_utf8 key'],
+                   entry
+      assert_nil fields['null_field'], entry
     end
   end
 
@@ -292,9 +294,9 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
       { 'seconds' => nil, 'nanos' => time.tv_nsec } => nil,
       { 'seconds' => 'seconds', 'nanos' => time.tv_nsec } => nil,
       { 'seconds' => time.tv_sec, 'nanos' => 'nanos' } => \
-        { 'seconds' => time.tv_sec },
+        time.utc.strftime('%Y-%m-%dT%H:%M:%SZ'),
       { 'seconds' => time.tv_sec, 'nanos' => nil } => \
-        { 'seconds' => time.tv_sec }
+        time.utc.strftime('%Y-%m-%dT%H:%M:%SZ')
     }.each do |input, expected|
       setup_logging_stubs do
         d = create_driver
@@ -318,6 +320,27 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     use_grpc true
   ).freeze
 
+  # The conversions from user input to output.
+  def latency_conversion
+    {
+      '32 s' => '32s',
+      '32s' => '32s',
+      '0.32s' => '0.320000000s',
+      ' 123 s ' => '123s',
+      '1.3442 s' => '1.344200000s',
+
+      # Test whitespace.
+      # \t: tab. \r: carriage return. \n: line break.
+      # \v: vertical whitespace. \f: form feed.
+      "\t123.5\ts\t" => '123.500000000s',
+      "\r123.5\rs\r" => '123.500000000s',
+      "\n123.5\ns\n" => '123.500000000s',
+      "\v123.5\vs\v" => '123.500000000s',
+      "\f123.5\fs\f" => '123.500000000s',
+      "\r123.5\ts\f" => '123.500000000s'
+    }
+  end
+
   # Create a Fluentd output test driver with the Google Cloud Output plugin with
   # grpc enabled. The signature of this method is different between the grpc
   # path and the non-grpc path. For grpc, an additional grpc stub class can be
@@ -393,7 +416,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
         raise @error
       rescue
         # Google::Gax::GaxError will wrap the latest thrown exception as @cause.
-        raise Google::Gax::GaxError, @message
+        raise Google::Gax::GaxError, 'This test message does not matter.'
       end
     end
     # rubocop:enable Lint/UnusedMethodArgument
@@ -448,99 +471,19 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
   end
 
-  # Get the fields of the payload.
-  def get_fields(payload)
-    payload['fields']
-  end
-
-  # Get the value of a struct field.
-  def get_struct(field)
-    field['structValue']
-  end
-
-  # Get the value of a string field.
-  def get_string(field)
-    field['stringValue']
-  end
-
-  # Get the value of a number field.
-  def get_number(field)
-    field['numberValue']
-  end
-
-  def get_bool(field)
-    field['boolValue']
-  end
-
-  # The null value.
-  def null_value
-    { 'nullValue' => 'NULL_VALUE' }
-  end
-
-  def http_request_message
-    HTTP_REQUEST_MESSAGE
-  end
-
-  def source_location_message
-    SOURCE_LOCATION_MESSAGE
-  end
-
-  def source_location_message2
-    SOURCE_LOCATION_MESSAGE2
-  end
-
   def expected_operation_message2
     # 'last' is a boolean field with false as default value. Protobuf omit
     # fields with default values during deserialization.
     OPERATION_MESSAGE2.reject { |k, _| k == 'last' }
   end
 
-  # expected: A Ruby hash that represents a JSON object.
-  # e.g.:
-  #   {
-  #     "file" => "source/file",
-  #     "function" => "my_function",
-  #     "line" => 18
-  #   }
-  #
-  # actual: A Ruby hash that represents a Proto object.
-  # e.g.:
-  #   {
-  #     "structValue" => {
-  #       "fields" => {
-  #         "file" => {
-  #           "stringValue" => "source/file"
-  #         },
-  #         "function" => {
-  #           "stringValue" => "my_function"
-  #         },
-  #         "line" => {
-  #           "numberValue" => 18
-  #         }
-  #       }
-  #     }
-  #   }
-  # This method has a different implementation at the REST side.
-  def assert_hash_equal_json(expected, actual)
-    error_message = "expected: #{expected}\nactual: #{actual}"
-    assert_true actual.is_a?(Hash),
-                "Expect the actual value to be a hash. #{error_message}"
-    if actual.key?('stringValue')
-      assert_equal expected, get_string(actual), error_message
-    elsif actual.key?('numberValue')
-      assert_equal expected, get_number(actual), error_message
-    elsif actual.key?('boolValue')
-      assert_equal expected, get_bool(actual), error_message
-    elsif actual.key?('structValue')
-      expected_copy = expected.dup
-      get_fields(get_struct(actual)).each do |field_name, nested_actual|
-        assert_hash_equal_json expected_copy[field_name], nested_actual
-        expected_copy.reject! { |k, _| k == field_name }
-      end
-      # Make sure all fields are matched.
-      assert_true expected_copy.empty?
-    else
-      assert_true false, "Unsupported proto format. #{error_message}"
-    end
+  # Parse timestamp and convert it to a hash with two keys:
+  # "seconds" and "nanos".
+  def timestamp_parse(timestamp)
+    parsed = Time.parse(timestamp)
+    {
+      'seconds' => parsed.tv_sec,
+      'nanos' => parsed.tv_nsec
+    }
   end
 end