vmik-fluent-plugin-google-cloud 0.5.5 → 0.6.4.pre.alpha

data/test/plugin/base_test.rb

@@ -16,9 +16,14 @@ require 'google/apis'
 require 'helper'
 require 'mocha/test_unit'
 require 'webmock/test_unit'
+require 'prometheus/client'
+
+require_relative 'constants'
 
 # Unit tests for Google Cloud Logging plugin
 module BaseTest
+  include Constants
+
   def setup
     Fluent::Test.setup
     # delete environment variables that googleauth uses to find credentials.
@@ -37,317 +42,6 @@ module BaseTest
     @logs_sent = []
   end
 
-  # generic attributes
-  HOSTNAME = Socket.gethostname
-
-  # attributes used for the GCE metadata service
-  PROJECT_ID = 'test-project-id'
-  ZONE = 'us-central1-b'
-  FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
-  VM_ID = '9876543210'
-
-  # attributes used for custom (overridden) configs
-  CUSTOM_PROJECT_ID = 'test-custom-project-id'
-  CUSTOM_ZONE = 'us-custom-central1-b'
-  CUSTOM_FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
-  CUSTOM_VM_ID = 'C9876543210'
-  CUSTOM_HOSTNAME = 'custom.hostname.org'
-
-  # attributes used for the EC2 metadata service
-  EC2_PROJECT_ID = 'test-ec2-project-id'
-  EC2_ZONE = 'us-west-2b'
-  EC2_PREFIXED_ZONE = 'aws:' + EC2_ZONE
-  EC2_VM_ID = 'i-81c16767'
-  EC2_ACCOUNT_ID = '123456789012'
-
-  # The formatting here matches the format used on the VM.
-  EC2_IDENTITY_DOCUMENT = %({
-    "accountId" : "#{EC2_ACCOUNT_ID}",
-    "availabilityZone" : "#{EC2_ZONE}",
-    "instanceId" : "#{EC2_VM_ID}"
-  })
-
-  # Managed VMs specific labels
-  MANAGED_VM_BACKEND_NAME = 'default'
-  MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
-
-  # Container Engine / Kubernetes specific labels
-  CONTAINER_CLUSTER_NAME = 'cluster-1'
-  CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
-  CONTAINER_NAMESPACE_NAME = 'kube-system'
-  CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
-  CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
-  CONTAINER_CONTAINER_NAME = 'redis'
-  CONTAINER_LABEL_KEY = 'component'
-  CONTAINER_LABEL_VALUE = 'redis-component'
-  CONTAINER_STREAM = 'stdout'
-  CONTAINER_SEVERITY = 'INFO'
-  # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch
-  CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
-  CONTAINER_SECONDS_EPOCH = 1_234_567_890
-  CONTAINER_NANOS = 987_654_321
-
-  # Cloud Functions specific labels
-  CLOUDFUNCTIONS_FUNCTION_NAME = '$My_Function.Name-@1'
-  CLOUDFUNCTIONS_REGION = 'us-central1'
-  CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
-  CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
-  CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
-  CLOUDFUNCTIONS_POD_NAME = 'd.dc.myu.uc.functionp.pc.name-a.a1.987-c0l82'
-  CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
-
-  # Parameters used for authentication
-  AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
-  FAKE_AUTH_TOKEN = 'abc123'
-
-  # Information about test credentials files.
-  # path: Path to the credentials file.
-  # project_id: ID of the project, which must correspond to the file contents.
-  IAM_CREDENTIALS = {
-    path: 'test/plugin/data/iam-credentials.json',
-    project_id: 'fluent-test-project'
-  }
-  LEGACY_CREDENTIALS = {
-    path: 'test/plugin/data/credentials.json',
-    project_id: '847859579879'
-  }
-  INVALID_CREDENTIALS = {
-    path: 'test/plugin/data/invalid_credentials.json',
-    project_id: ''
-  }
-
-  # Configuration files for various test scenarios
-  APPLICATION_DEFAULT_CONFIG = %(
-  )
-
-  # rubocop:disable Metrics/LineLength
-  PRIVATE_KEY_CONFIG = %(
-     auth_method private_key
-     private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
-     private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
-  )
-  # rubocop:enable Metrics/LineLength
-
-  REQUIRE_VALID_TAGS_CONFIG = %(
-    require_valid_tags true
-  )
-
-  NO_METADATA_SERVICE_CONFIG = %(
-    use_metadata_service false
-  )
-
-  NO_DETECT_SUBSERVICE_CONFIG = %(
-    detect_subservice false
-  )
-
-  CUSTOM_METADATA_CONFIG = %(
-    project_id #{CUSTOM_PROJECT_ID}
-    zone #{CUSTOM_ZONE}
-    vm_id #{CUSTOM_VM_ID}
-    vm_name #{CUSTOM_HOSTNAME}
-  )
-
-  CONFIG_MISSING_METADATA_PROJECT_ID = %(
-    zone #{CUSTOM_ZONE}
-    vm_id #{CUSTOM_VM_ID}
-  )
-  CONFIG_MISSING_METADATA_ZONE = %(
-    project_id #{CUSTOM_PROJECT_ID}
-    vm_id #{CUSTOM_VM_ID}
-  )
-  CONFIG_MISSING_METADATA_VM_ID = %(
-    project_id #{CUSTOM_PROJECT_ID}
-    zone #{CUSTOM_ZONE}
-  )
-  CONFIG_MISSING_METADATA_ALL = %(
-  )
-
-  CONFIG_EC2_PROJECT_ID = %(
-    project_id #{EC2_PROJECT_ID}
-  )
-
-  CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %(
-    project_id #{EC2_PROJECT_ID}
-    vm_id #{CUSTOM_VM_ID}
-  )
-
-  # Service configurations for various services
-  COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
-  APPENGINE_SERVICE_NAME = 'appengine.googleapis.com'
-  CONTAINER_SERVICE_NAME = 'container.googleapis.com'
-  CLOUDFUNCTIONS_SERVICE_NAME = 'cloudfunctions.googleapis.com'
-  EC2_SERVICE_NAME = 'ec2.amazonaws.com'
-
-  COMPUTE_PARAMS = {
-    service_name: COMPUTE_SERVICE_NAME,
-    log_name: 'test',
-    project_id: PROJECT_ID,
-    zone: ZONE,
-    labels: {
-      "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
-      "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
-      "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
-    }
-  }
-
-  VMENGINE_PARAMS = {
-    service_name: APPENGINE_SERVICE_NAME,
-    log_name: "#{APPENGINE_SERVICE_NAME}%2Ftest",
-    project_id: PROJECT_ID,
-    zone: ZONE,
-    labels: {
-      "#{APPENGINE_SERVICE_NAME}/module_id" => MANAGED_VM_BACKEND_NAME,
-      "#{APPENGINE_SERVICE_NAME}/version_id" => MANAGED_VM_BACKEND_VERSION,
-      "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
-      "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
-      "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
-    }
-  }
-
-  CONTAINER_TAG = "kubernetes.#{CONTAINER_POD_NAME}_" \
-                  "#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
-
-  CONTAINER_FROM_METADATA_PARAMS = {
-    service_name: CONTAINER_SERVICE_NAME,
-    log_name: CONTAINER_CONTAINER_NAME,
-    project_id: PROJECT_ID,
-    zone: ZONE,
-    labels: {
-      "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
-      "#{CONTAINER_SERVICE_NAME}/cluster_name" => CONTAINER_CLUSTER_NAME,
-      "#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
-      "#{CONTAINER_SERVICE_NAME}/namespace_id" => CONTAINER_NAMESPACE_ID,
-      "#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
-      "#{CONTAINER_SERVICE_NAME}/pod_id" => CONTAINER_POD_ID,
-      "#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
-      "#{CONTAINER_SERVICE_NAME}/stream" => CONTAINER_STREAM,
-      "label/#{CONTAINER_LABEL_KEY}" => CONTAINER_LABEL_VALUE,
-      "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
-      "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
-      "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
-    }
-  }
-
-  # Almost the same as from metadata, but missing namespace_id and pod_id.
-  CONTAINER_FROM_TAG_PARAMS = {
-    service_name: CONTAINER_SERVICE_NAME,
-    log_name: CONTAINER_CONTAINER_NAME,
-    project_id: PROJECT_ID,
-    zone: ZONE,
-    labels: {
-      "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
-      "#{CONTAINER_SERVICE_NAME}/cluster_name" => CONTAINER_CLUSTER_NAME,
-      "#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
-      "#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
-      "#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
-      "#{CONTAINER_SERVICE_NAME}/stream" => CONTAINER_STREAM,
-      "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
-      "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
-      "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
-    }
-  }
-
-  CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
-                        "#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
-                        "#{CLOUDFUNCTIONS_CONTAINER_NAME}"
-
-  CLOUDFUNCTIONS_PARAMS = {
-    service_name: CLOUDFUNCTIONS_SERVICE_NAME,
-    log_name: 'cloud-functions',
-    project_id: PROJECT_ID,
-    zone: ZONE,
-    labels: {
-      'execution_id' => CLOUDFUNCTIONS_EXECUTION_ID,
-      "#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
-        CLOUDFUNCTIONS_FUNCTION_NAME,
-      "#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
-      "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
-      "#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
-      "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
-      "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
-      "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
-    }
-  }
-
-  CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS = {
-    service_name: CLOUDFUNCTIONS_SERVICE_NAME,
-    log_name: 'cloud-functions',
-    project_id: PROJECT_ID,
-    zone: ZONE,
-    labels: {
-      "#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
-        CLOUDFUNCTIONS_FUNCTION_NAME,
-      "#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
-      "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
-      "#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
-      "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
-      "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
-      "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
-    }
-  }
-
-  CUSTOM_PARAMS = {
-    service_name: COMPUTE_SERVICE_NAME,
-    log_name: 'test',
-    project_id: CUSTOM_PROJECT_ID,
-    zone: CUSTOM_ZONE,
-    labels: {
-      "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
-      "#{COMPUTE_SERVICE_NAME}/resource_id" => CUSTOM_VM_ID,
-      "#{COMPUTE_SERVICE_NAME}/resource_name" => CUSTOM_HOSTNAME
-    }
-  }
-
-  EC2_PARAMS = {
-    service_name: EC2_SERVICE_NAME,
-    log_name: 'test',
-    project_id: EC2_PROJECT_ID,
-    zone: EC2_PREFIXED_ZONE,
-    labels: {
-      "#{EC2_SERVICE_NAME}/resource_type" => 'instance',
-      "#{EC2_SERVICE_NAME}/resource_id" => EC2_VM_ID,
-      "#{EC2_SERVICE_NAME}/account_id" => EC2_ACCOUNT_ID,
-      "#{EC2_SERVICE_NAME}/resource_name" => HOSTNAME
-    }
-  }
-
-  HTTP_REQUEST_MESSAGE = {
-    'requestMethod' => 'POST',
-    'requestUrl' => 'http://example/',
-    'requestSize' => 210,
-    'status' => 200,
-    'responseSize' => 65,
-    'userAgent' => 'USER AGENT 1.0',
-    'remoteIp' => '55.55.55.55',
-    'referer' => 'http://referer/',
-    'cacheHit' => true,
-    'cacheValidatedWithOriginServer' => true
-  }
-
-  # Tags and their sanitized and encoded version.
-  VALID_TAGS = {
-    'test' => 'test',
-    'germanß' => 'german%C3%9F',
-    'chinese中' => 'chinese%E4%B8%AD',
-    'specialCharacter/_-.' => 'specialCharacter%2F_-.',
-    'abc@&^$*' => 'abc%40%26%5E%24%2A',
-    '@&^$*' => '%40%26%5E%24%2A'
-  }
-  INVALID_TAGS = {
-    # Non-string tags.
-    123 => '123',
-    1.23 => '1.23',
-    [1, 2, 3] => '%5B1%2C%202%2C%203%5D',
-    { key: 'value' } => '%7B%22key%22%3D%3E%22value%22%7D',
-    # Non-utf8 string tags.
-    "nonutf8#{[0x92].pack('C*')}" => 'nonutf8%20',
-    "abc#{[0x92].pack('C*')}" => 'abc%20',
-    "#{[0x92].pack('C*')}" => '%20',
-    # Empty string tag.
-    '' => '_'
-  }
-  ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
-
   # Shared tests.
 
   def test_configure_service_account_application_default
@@ -434,12 +128,12 @@ module BaseTest
 
   def test_gce_used_when_detect_subservice_is_false
     setup_gce_metadata_stubs
-    # This would cause the service to be container.googleapis.com if not for the
-    # detect_subservice=false config.
+    # This would cause the resource type to be container.googleapis.com if not
+    # for the detect_subservice=false config.
     setup_container_metadata_stubs
     d = create_driver(NO_DETECT_SUBSERVICE_CONFIG)
     d.run
-    assert_equal COMPUTE_SERVICE_NAME, d.instance.service_name
+    assert_equal COMPUTE_CONSTANTS[:resource_type], d.instance.resource.type
   end
 
   def test_metadata_overrides
@@ -554,7 +248,7 @@ module BaseTest
     verify_log_entries(1, EC2_PARAMS)
   end
 
-  def test_struct_payload_log
+  def test_structured_payload_log
     setup_gce_metadata_stubs
     setup_logging_stubs do
       d = create_driver
@@ -562,8 +256,8 @@ module BaseTest
              'some_null_field' => nil)
       d.run
     end
-    verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
-      fields = get_fields(entry['structPayload'])
+    verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+      fields = get_fields(entry['jsonPayload'])
       assert_equal 4, fields.size, entry
       assert_equal 'test log entry 0', get_string(fields['msg']), entry
       assert_equal 'test', get_string(fields['tag2']), entry
@@ -572,7 +266,7 @@ module BaseTest
     end
   end
 
-  def test_struct_payload_malformatted_log
+  def test_structured_payload_malformatted_log
     setup_gce_metadata_stubs
     message = 'test message'
     setup_logging_stubs do
@@ -588,8 +282,8 @@ module BaseTest
       )
       d.run
     end
-    verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
-      fields = get_fields(entry['structPayload'])
+    verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+      fields = get_fields(entry['jsonPayload'])
       assert_equal 7, fields.size, entry
       assert_equal message, get_string(get_fields(get_struct(fields \
                    ['int_key']))['1']), entry
@@ -608,7 +302,7 @@ module BaseTest
     end
   end
 
-  def test_struct_payload_json_log
+  def test_structured_payload_json_log
     setup_gce_metadata_stubs
     setup_logging_stubs do
       d = create_driver
@@ -624,7 +318,7 @@ module BaseTest
     end
   end
 
-  def test_struct_payload_json_container_log
+  def test_structured_payload_json_container_log
     setup_gce_metadata_stubs
     setup_container_metadata_stubs
     setup_logging_stubs do
@@ -643,8 +337,8 @@ module BaseTest
       if log_index == 1
         assert entry.key?('textPayload'), 'Entry did not have textPayload'
       else
-        assert entry.key?('structPayload'), 'Entry did not have structPayload'
-        fields = get_fields(entry['structPayload'])
+        assert entry.key?('jsonPayload'), 'Entry did not have jsonPayload'
+        fields = get_fields(entry['jsonPayload'])
         assert_equal 4, fields.size, entry
         assert_equal 'test log entry 0', get_string(fields['msg']), entry
         assert_equal 'test', get_string(fields['tag2']), entry
@@ -665,7 +359,7 @@ module BaseTest
         d.emit('msg' => log_entry(0))
         d.run
       end
-      verify_log_entries(0, COMPUTE_PARAMS, 'structPayload')
+      verify_log_entries(0, COMPUTE_PARAMS, 'jsonPayload')
     end
   end
 
@@ -678,18 +372,17 @@ module BaseTest
     # This tag will not match the kubernetes regex because it requires a
     # non-empty container name.
     tag = container_tag_with_container_name(container_name)
-    params = CONTAINER_FROM_METADATA_PARAMS.merge(
-      labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
-        "#{CONTAINER_SERVICE_NAME}/container_name" => container_name),
-      log_name: tag)
-    setup_logging_stubs([params]) do
-      @logs_sent = []
+    setup_logging_stubs do
       d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
       d.emit(container_log_entry_with_metadata(log_entry(0), container_name))
       d.run
     end
+    params = CONTAINER_FROM_METADATA_PARAMS.merge(
+      resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
+        labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
+          'container_name' => container_name)),
+      log_name: tag)
     verify_log_entries(1, params, 'textPayload')
-    assert_equal "projects/#{PROJECT_ID}/logs/#{tag}", @logs_sent[0]['logName']
   end
 
   # Verify that container names with non-utf8 characters should be rejected when
@@ -701,18 +394,18 @@ module BaseTest
       tag.is_a?(String) && !tag.empty?
     end
     non_utf8_tags.each do |container_name, encoded_name|
-      params = CONTAINER_FROM_METADATA_PARAMS.merge(
-        labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
-          "#{CONTAINER_SERVICE_NAME}/container_name" =>
-            URI.decode(encoded_name)),
-        log_name: encoded_name)
-      setup_logging_stubs([params]) do
+      setup_logging_stubs do
         @logs_sent = []
         d = create_driver(REQUIRE_VALID_TAGS_CONFIG,
                           container_tag_with_container_name(container_name))
         d.emit(container_log_entry_with_metadata(log_entry(0), container_name))
         d.run
       end
+      params = CONTAINER_FROM_METADATA_PARAMS.merge(
+        labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
+          "#{CONTAINER_CONSTANTS[:service]}/container_name" =>
+            URI.decode(encoded_name)),
+        log_name: encoded_name)
       verify_log_entries(0, params, 'textPayload')
     end
   end
@@ -722,15 +415,14 @@ module BaseTest
   def test_encode_tags_with_require_valid_tags_true
     setup_gce_metadata_stubs
     VALID_TAGS.each do |tag, encoded_tag|
-      setup_logging_stubs([COMPUTE_PARAMS.merge(log_name: encoded_tag)]) do
+      setup_logging_stubs do
         @logs_sent = []
         d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
         d.emit('msg' => log_entry(0))
         d.run
       end
-      verify_log_entries(1, COMPUTE_PARAMS, 'structPayload')
-      assert_equal "projects/#{PROJECT_ID}/logs/#{encoded_tag}",
-                   @logs_sent[0]['logName']
+      verify_log_entries(1, COMPUTE_PARAMS.merge(log_name: encoded_tag),
+                         'jsonPayload')
     end
   end
 
@@ -739,20 +431,19 @@ module BaseTest
     setup_gce_metadata_stubs
     setup_container_metadata_stubs
     VALID_TAGS.each do |tag, encoded_tag|
-      params = CONTAINER_FROM_METADATA_PARAMS.merge(
-        labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
-          "#{CONTAINER_SERVICE_NAME}/container_name" => tag),
-        log_name: encoded_tag)
-      setup_logging_stubs([params]) do
+      setup_logging_stubs do
         @logs_sent = []
         d = create_driver(REQUIRE_VALID_TAGS_CONFIG,
                           container_tag_with_container_name(tag))
         d.emit(container_log_entry_with_metadata(log_entry(0), tag))
         d.run
       end
+      params = CONTAINER_FROM_METADATA_PARAMS.merge(
+        resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
+          labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
+            'container_name' => tag)),
+        log_name: encoded_tag)
       verify_log_entries(1, params, 'textPayload')
-      assert_equal "projects/#{PROJECT_ID}/logs/#{encoded_tag}",
-                   @logs_sent[0]['logName']
     end
   end
 
@@ -762,15 +453,14 @@ module BaseTest
   def test_sanitize_tags_with_require_valid_tags_false
     setup_gce_metadata_stubs
     ALL_TAGS.each do |tag, sanitized_tag|
-      setup_logging_stubs([COMPUTE_PARAMS.merge(log_name: sanitized_tag)]) do
+      setup_logging_stubs do
         @logs_sent = []
         d = create_driver(APPLICATION_DEFAULT_CONFIG, tag)
         d.emit('msg' => log_entry(0))
         d.run
       end
-      verify_log_entries(1, COMPUTE_PARAMS, 'structPayload')
-      assert_equal "projects/#{PROJECT_ID}/logs/#{sanitized_tag}",
-                   @logs_sent[0]['logName']
+      verify_log_entries(1, COMPUTE_PARAMS.merge(log_name: sanitized_tag),
+                         'jsonPayload')
     end
   end
 
@@ -787,21 +477,19 @@ module BaseTest
     string_tags.each do |container_name, encoded_container_name|
       # Container name in the label is sanitized but not encoded, while the log
       # name is encoded.
-      params = CONTAINER_FROM_METADATA_PARAMS.merge(
-        labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
-          "#{CONTAINER_SERVICE_NAME}/container_name" =>
-            URI.decode(encoded_container_name)),
-        log_name: encoded_container_name)
-      setup_logging_stubs([params]) do
+      setup_logging_stubs do
         @logs_sent = []
         d = create_driver(APPLICATION_DEFAULT_CONFIG,
                           container_tag_with_container_name(container_name))
         d.emit(container_log_entry_with_metadata(log_entry(0), container_name))
         d.run
       end
+      params = CONTAINER_FROM_METADATA_PARAMS.merge(
+        resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
+          labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
+            'container_name' => URI.decode(encoded_container_name))),
+        log_name: encoded_container_name)
       verify_log_entries(1, params, 'textPayload')
-      assert_equal "projects/#{PROJECT_ID}/logs/#{encoded_container_name}",
-                   @logs_sent[0]['logName']
     end
   end
 
@@ -833,9 +521,9 @@ module BaseTest
     end
     verify_index = 0
     verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
-      assert_equal_with_default entry['metadata']['timestamp']['seconds'],
+      assert_equal_with_default entry['timestamp']['seconds'],
                                 expected_ts[verify_index].tv_sec, 0, entry
-      assert_equal_with_default entry['metadata']['timestamp']['nanos'],
+      assert_equal_with_default entry['timestamp']['nanos'],
                                 expected_ts[verify_index].tv_nsec, 0, entry do
         # Fluentd v0.14 onwards supports nanosecond timestamp values.
         # Added in 600 ns delta to avoid flaky tests introduced
@@ -843,7 +531,7 @@ module BaseTest
         # (to account for the missing 9 bits of precision ~ 512 ns).
         # See http://wikipedia.org/wiki/Double-precision_floating-point_format
         assert_in_delta expected_ts[verify_index].tv_nsec,
-                        entry['metadata']['timestamp']['nanos'], 600, entry
+                        entry['timestamp']['nanos'], 600, entry
       end
       verify_index += 1
     end
@@ -853,12 +541,12 @@ module BaseTest
     setup_gce_metadata_stubs
     setup_logging_stubs do
       d = create_driver
-      # if timestamp is not a hash it is passed through to the struct payload.
+      # if timestamp is not a hash it is passed through to the json payload.
       d.emit('message' => log_entry(0), 'timestamp' => 'not-a-hash')
       d.run
     end
-    verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
-      fields = get_fields(entry['structPayload'])
+    verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+      fields = get_fields(entry['jsonPayload'])
       assert_equal 2, fields.size, entry
       assert_equal 'not-a-hash', get_string(fields['timestamp']), entry
     end
@@ -937,7 +625,7 @@ module BaseTest
         }
       )
       d = create_driver(config)
-      # not_a_label passes through to the struct payload
+      # not_a_label passes through to the json payload
       d.emit('message' => log_entry(0),
              'label1' => 'value1',
              'label_number_two' => 'value2',
@@ -950,8 +638,8 @@ module BaseTest
     params[:labels]['sent_label_1'] = 'value1'
     params[:labels]['foo.googleapis.com/bar'] = 'value2'
     params[:labels]['label3'] = 'value3'
-    verify_log_entries(1, params, 'structPayload') do |entry|
-      fields = get_fields(entry['structPayload'])
+    verify_log_entries(1, params, 'jsonPayload') do |entry|
+      fields = get_fields(entry['jsonPayload'])
       assert_equal 2, fields.size, entry
       assert_equal 'test log entry 0', get_string(fields['message']), entry
       assert_equal 'value4', get_string(fields['not_a_label']), entry
@@ -1023,11 +711,9 @@ module BaseTest
       d.run
     end
     verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS) do |entry|
-      assert_equal CONTAINER_SECONDS_EPOCH, \
-                   entry['metadata']['timestamp']['seconds'], entry
-      assert_equal CONTAINER_NANOS, \
-                   entry['metadata']['timestamp']['nanos'], entry
-      assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
+      assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+      assert_equal CONTAINER_SEVERITY, entry['severity'], entry
     end
   end
 
@@ -1045,11 +731,10 @@ module BaseTest
         d.run
       end
       verify_log_entries(n, CONTAINER_FROM_METADATA_PARAMS) do |entry|
-        assert_equal CONTAINER_SECONDS_EPOCH, \
-                     entry['metadata']['timestamp']['seconds'], entry
-        assert_equal CONTAINER_NANOS, \
-                     entry['metadata']['timestamp']['nanos'], entry
-        assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
+        assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'],
+                     entry
+        assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+        assert_equal CONTAINER_SEVERITY, entry['severity'], entry
       end
     end
   end
@@ -1068,11 +753,10 @@ module BaseTest
         d.run
       end
       verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS) do |entry|
-        assert_equal CONTAINER_SECONDS_EPOCH, \
-                     entry['metadata']['timestamp']['seconds'], entry
-        assert_equal CONTAINER_NANOS, \
-                     entry['metadata']['timestamp']['nanos'], entry
-        assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
+        assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'],
+                     entry
+        assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+        assert_equal CONTAINER_SEVERITY, entry['severity'], entry
       end
     end
   end
@@ -1086,11 +770,9 @@ module BaseTest
       d.run
     end
     verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS) do |entry|
-      assert_equal CONTAINER_SECONDS_EPOCH, \
-                   entry['metadata']['timestamp']['seconds'], entry
-      assert_equal CONTAINER_NANOS, \
-                   entry['metadata']['timestamp']['nanos'], entry
-      assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
+      assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+      assert_equal CONTAINER_SEVERITY, entry['severity'], entry
     end
   end
 
@@ -1103,18 +785,16 @@ module BaseTest
       d.run
     end
     expected_params = CONTAINER_FROM_TAG_PARAMS.merge(
-      labels: { "#{CONTAINER_SERVICE_NAME}/stream" => 'stderr' }
+      labels: { "#{CONTAINER_CONSTANTS[:service]}/stream" => 'stderr' }
     ) { |_, oldval, newval| oldval.merge(newval) }
     verify_log_entries(1, expected_params) do |entry|
-      assert_equal CONTAINER_SECONDS_EPOCH, \
-                   entry['metadata']['timestamp']['seconds'], entry
-      assert_equal CONTAINER_NANOS, \
-                   entry['metadata']['timestamp']['nanos'], entry
-      assert_equal 'ERROR', entry['metadata']['severity'], entry
+      assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+      assert_equal 'ERROR', entry['severity'], entry
     end
   end
 
-  def test_struct_container_log_metadata_from_plugin
+  def test_json_container_log_metadata_from_plugin
     setup_gce_metadata_stubs
     setup_container_metadata_stubs
     setup_logging_stubs do
@@ -1125,21 +805,19 @@ module BaseTest
       d.run
     end
     verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS,
-                       'structPayload') do |entry|
-      fields = get_fields(entry['structPayload'])
+                       'jsonPayload') do |entry|
+      fields = get_fields(entry['jsonPayload'])
       assert_equal 3, fields.size, entry
       assert_equal 'test log entry 0', get_string(fields['msg']), entry
       assert_equal 'test', get_string(fields['tag2']), entry
       assert_equal 5000, get_number(fields['data']), entry
-      assert_equal CONTAINER_SECONDS_EPOCH, \
-                   entry['metadata']['timestamp']['seconds'], entry
-      assert_equal CONTAINER_NANOS, \
-                   entry['metadata']['timestamp']['nanos'], entry
-      assert_equal 'WARNING', entry['metadata']['severity'], entry
+      assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+      assert_equal 'WARNING', entry['severity'], entry
     end
   end
 
-  def test_struct_container_log_metadata_from_tag
+  def test_json_container_log_metadata_from_tag
     setup_gce_metadata_stubs
     setup_container_metadata_stubs
     setup_logging_stubs do
@@ -1150,17 +828,15 @@ module BaseTest
       d.run
     end
     verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS,
-                       'structPayload') do |entry|
-      fields = get_fields(entry['structPayload'])
+                       'jsonPayload') do |entry|
+      fields = get_fields(entry['jsonPayload'])
       assert_equal 3, fields.size, entry
       assert_equal 'test log entry 0', get_string(fields['msg']), entry
       assert_equal 'test', get_string(fields['tag2']), entry
       assert_equal 5000, get_number(fields['data']), entry
-      assert_equal CONTAINER_SECONDS_EPOCH, \
-                   entry['metadata']['timestamp']['seconds'], entry
-      assert_equal CONTAINER_NANOS, \
-                   entry['metadata']['timestamp']['nanos'], entry
-      assert_equal 'WARNING', entry['metadata']['severity'], entry
+      assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+      assert_equal 'WARNING', entry['severity'], entry
     end
   end
 
@@ -1178,7 +854,7 @@ module BaseTest
         d.run
       end
       verify_log_entries(n, CLOUDFUNCTIONS_PARAMS) do |entry|
-        assert_equal 'DEBUG', entry['metadata']['severity'],
+        assert_equal 'DEBUG', entry['severity'],
                      "Test with #{n} logs failed. \n#{entry}"
       end
     end
@@ -1199,7 +875,7 @@ module BaseTest
       end
       verify_log_entries(
         n, CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS) do |entry|
-        assert_equal 'INFO', entry['metadata']['severity'],
+        assert_equal 'INFO', entry['severity'],
                      "Test with #{n} logs failed. \n#{entry}"
       end
     end
@@ -1228,16 +904,27 @@ module BaseTest
     end
   end
 
+  def test_dataproc_log
+    setup_gce_metadata_stubs
+    setup_dataproc_metadata_stubs
+    setup_logging_stubs do
+      d = create_driver
+      d.emit(dataproc_log_entry('test message'))
+      d.run
+    end
+    verify_log_entries(1, DATAPROC_PARAMS, 'jsonPayload')
+  end
+
   def test_http_request_from_record
     setup_gce_metadata_stubs
     setup_logging_stubs do
       d = create_driver
-      d.emit('httpRequest' => http_request_message)
+      d.emit('httpRequest' => HTTP_REQUEST_MESSAGE)
       d.run
     end
     verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
-      assert_equal http_request_message, entry['httpRequest'], entry
-      assert_nil get_fields(entry['structPayload'])['httpRequest'], entry
+      assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry
+      assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
     end
   end
 
@@ -1245,13 +932,13 @@ module BaseTest
     setup_gce_metadata_stubs
     setup_logging_stubs do
       d = create_driver
-      d.emit('httpRequest' => http_request_message.merge(
+      d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge(
         'otherKey' => 'value'))
       d.run
     end
     verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
-      assert_equal http_request_message, entry['httpRequest'], entry
-      fields = get_fields(entry['structPayload'])
+      assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry
+      fields = get_fields(entry['jsonPayload'])
       request = get_fields(get_struct(fields['httpRequest']))
       assert_equal 'value', get_string(request['otherKey']), entry
     end
@@ -1264,20 +951,72 @@ module BaseTest
       d.emit('httpRequest' => 'a_string')
       d.run
     end
-    verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
-      fields = get_fields(entry['structPayload'])
+    verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+      fields = get_fields(entry['jsonPayload'])
       assert_equal 'a_string', get_string(fields['httpRequest']), entry
       assert_nil entry['httpRequest'], entry
     end
   end
 
-  private
+  def test_http_request_from_record_with_referer_nil_or_absent
+    setup_gce_metadata_stubs
+    [
+      http_request_message_with_nil_referer,
+      http_request_message_with_absent_referer
+    ].each do |input|
+      setup_logging_stubs do
+        @logs_sent = []
+        d = create_driver
+        d.emit('httpRequest' => input)
+        d.run
+      end
+      verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
+        assert_equal http_request_message_with_absent_referer,
+                     entry['httpRequest'], entry
+        assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
+      end
+    end
+  end
 
-  def uri_for_log(params)
-    'https://logging.googleapis.com/v1beta3/projects/' + params[:project_id] +
-      '/logs/' + params[:log_name] + '/entries:write'
+  def test_http_request_with_latency
+    setup_gce_metadata_stubs
+    latency_conversion.each do |input, expected|
+      setup_logging_stubs do
+        d = create_driver
+        @logs_sent = []
+        d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge('latency' => input))
+        d.run
+      end
+      verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
+        assert_equal HTTP_REQUEST_MESSAGE.merge('latency' => expected),
+                     entry['httpRequest'], entry
+        assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
+      end
+    end
   end
 
+  # Skip setting latency when the field is null or has invalid format.
+  def test_http_request_skip_setting_latency
+    setup_gce_metadata_stubs
+    [
+      '', ' ', nil, 'null', '123', '1.23 seconds',
+      ' 123 s econds ', '1min', 'abc&^!$*('
+    ].each do |input|
+      setup_logging_stubs do
+        d = create_driver
+        @logs_sent = []
+        d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge('latency' => input))
+        d.run
+      end
+      verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
+        assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry
+        assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
+      end
+    end
+  end
+
+  private
+
   def stub_metadata_request(metadata_path, response_body)
     stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
                  metadata_path)
@@ -1375,6 +1114,22 @@ module BaseTest
                           CLOUDFUNCTIONS_REGION)
   end
 
+  def setup_dataproc_metadata_stubs
+    stub_metadata_request(
+      'instance/attributes/',
+      "attribute1\ndataproc-cluster-uuid\ndataproc-cluster-name")
+    stub_metadata_request('instance/attributes/dataproc-cluster-name',
+                          DATAPROC_CLUSTER_NAME)
+    stub_metadata_request('instance/attributes/dataproc-cluster-uuid',
+                          DATAPROC_CLUSTER_UUID)
+    stub_metadata_request('instance/attributes/dataproc-region',
+                          DATAPROC_REGION)
+  end
+
+  def setup_prometheus
+    Prometheus::Client.registry.instance_variable_set('@metrics', {})
+  end
+
   def container_tag_with_container_name(container_name)
     "kubernetes.#{CONTAINER_POD_NAME}_#{CONTAINER_NAMESPACE_NAME}_" \
       "#{container_name}"
@@ -1421,42 +1176,74 @@ module BaseTest
     }
   end
 
+  def dataflow_log_entry(i)
+    {
+      step: DATAFLOW_STEP_ID,
+      message: log_entry(i)
+    }
+  end
+
+  def dataproc_log_entry(message, source_class = 'com.example.Example',
+                         filename = 'test.log')
+    {
+      filename: filename,
+      class: source_class,
+      message: log_entry(message)
+    }
+  end
+
+  def ml_log_entry(i)
+    {
+      name: ML_LOG_AREA,
+      message: log_entry(i)
+    }
+  end
+
   def log_entry(i)
     'test log entry ' + i.to_s
   end
 
-  def check_labels(entry, common_labels, expected_labels)
-    # TODO(salty) test/handle overlap between common_labels and entry labels
-    all_labels ||= common_labels
-    all_labels.merge!(entry['metadata']['labels'] || {})
-    all_labels.each do |key, value|
+  def check_labels(labels, expected_labels)
+    labels.each do |key, value|
       assert value.is_a?(String), "Value #{value} for label #{key} " \
         'is not a string: ' + value.class.name
       assert expected_labels.key?(key), "Unexpected label #{key} => #{value}"
       assert_equal expected_labels[key], value, 'Value mismatch - expected ' \
         "#{expected_labels[key]} in #{key} => #{value}"
     end
-    assert_equal expected_labels.length, all_labels.length, 'Expected ' \
-      "#{expected_labels.length} labels, got #{all_labels.length}"
+    assert_equal expected_labels.length, labels.length, 'Expected ' \
+      "#{expected_labels.length} labels: #{expected_labels}, got " \
+      "#{labels.length} labels: #{labels}"
   end
 
   # The caller can optionally provide a block which is called for each entry.
   def verify_json_log_entries(n, params, payload_type = 'textPayload')
     i = 0
-    @logs_sent.each do |batch|
-      batch['entries'].each do |entry|
+    @logs_sent.each do |request|
+      request['entries'].each do |entry|
         unless payload_type.empty?
           assert entry.key?(payload_type), 'Entry did not contain expected ' \
             "#{payload_type} key: " + entry.to_s
           # Check the payload for textPayload, otherwise it's up to the caller.
           if payload_type == 'textPayload'
-            assert_equal "test log entry #{i}", entry['textPayload'], batch
+            assert_equal "test log entry #{i}", entry['textPayload'], request
           end
         end
 
-        assert_equal params[:zone], entry['metadata']['zone']
-        assert_equal params[:service_name], entry['metadata']['serviceName']
-        check_labels entry, batch['commonLabels'], params[:labels]
+        # per-entry resource or log_name overrides the corresponding field
+        # from the request.  Labels are merged, with the per-entry label
+        # taking precedence in case of overlap.
+        resource = entry['resource'] || request['resource']
+        log_name = entry['logName'] || request['logName']
+
+        labels ||= request['labels']
+        labels.merge!(entry['labels'] || {})
+
+        assert_equal \
+          "projects/#{params[:project_id]}/logs/#{params[:log_name]}", log_name
+        assert_equal params[:resource][:type], resource['type']
+        check_labels resource['labels'], params[:resource][:labels]
+        check_labels labels, params[:labels]
         yield(entry) if block_given?
         i += 1
         assert i <= n, "Number of entries #{i} exceeds expected number #{n}"
@@ -1465,14 +1252,37 @@ module BaseTest
     assert i == n, "Number of entries #{i} does not match expected number #{n}"
   end
 
-  # The http request message to test against.
-  def http_request_message
-    HTTP_REQUEST_MESSAGE
-  end
-
   # Replace the 'referer' field with nil.
   def http_request_message_with_nil_referer
-    http_request_message.merge('referer' => nil)
+    HTTP_REQUEST_MESSAGE.merge('referer' => nil)
+  end
+
+  # Unset the 'referer' field.
+  def http_request_message_with_absent_referer
+    HTTP_REQUEST_MESSAGE.reject do |k, _|
+      k == 'referer'
+    end
+  end
+
+  # The conversions from user input to output.
+  def latency_conversion
+    {
+      '32 s' => { 'seconds' => 32 },
+      '32s' => { 'seconds' => 32 },
+      '0.32s' => { 'nanos' => 320_000_000 },
+      ' 123 s ' => { 'seconds' => 123 },
+      '1.3442 s' => { 'seconds' => 1, 'nanos' => 344_200_000 },
+
+      # Test whitespace.
+      # \t: tab. \r: carriage return. \n: line break.
+      # \v: vertical whitespace. \f: form feed.
+      "\t123.5\ts\t" => { 'seconds' => 123, 'nanos' => 500_000_000 },
+      "\r123.5\rs\r" => { 'seconds' => 123, 'nanos' => 500_000_000 },
+      "\n123.5\ns\n" => { 'seconds' => 123, 'nanos' => 500_000_000 },
+      "\v123.5\vs\v" => { 'seconds' => 123, 'nanos' => 500_000_000 },
+      "\f123.5\fs\f" => { 'seconds' => 123, 'nanos' => 500_000_000 },
+      "\r123.5\ts\f" => { 'seconds' => 123, 'nanos' => 500_000_000 }
+    }
   end
 
   # This module expects the methods below to be overridden.
@@ -1503,8 +1313,14 @@ module BaseTest
     _undefined
   end
 
-  # Get the fields of the struct payload.
-  def get_fields(_struct_payload)
+  def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
+    metric = Prometheus::Client.registry.get(metric_name)
+    assert_not_nil(metric)
+    assert_equal(expected_value, metric.get(labels))
+  end
+
+  # Get the fields of the payload.
+  def get_fields(_payload)
     _undefined
   end