fluent-plugin-google-cloud 0.6.25.1 → 0.7.0.pre.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bb53a82f49086e32adb57a617f0ad53812493815
-  data.tar.gz: 6964a23f146726e01f71a74378c0fa4327c5b582
+  metadata.gz: 5fa418888cc895f5387d6d03b60154d4940a07d7
+  data.tar.gz: 06ecdf13e10c1262dabf80aedff34af14d132656
 SHA512:
-  metadata.gz: a9074434a1978f326a44fc4651d692a1e0f524c6c8ddeebda4efe89968059df9289749f21d85983a5308ee8629a208fb3963974ac234e8462a85a5c48aab94cb
-  data.tar.gz: 57e333f19ca5a7e5a9dbd2bd4b4bb65e461222bde2036b9a98a0e99ed2d95159d690c718b2b6bf95fe208dbbea2c7d5c1147a68b7d2dc39ff7f7dbfed1909664
+  metadata.gz: e8c1e36fdb141fc017340abcc27c39dd9989e1bda2a9dd16fdd77d8596f3b68203a022b4c221de35240546117f9ad78e67c8ed3dcf570804c62bb9459cd6676d
+  data.tar.gz: 1b4cc062cfbfe10db494c3ecb52499df5f5a52f204e863723f1212b72e61ac89d1a137039af4f7ea3d293a50c9f9eb14d043b22dfe1ef293fb888c7b4fc0da48

data/README.rdoc

@@ -1,10 +1,9 @@
 = Google Cloud Logging plugin for {fluentd}[http://github.com/fluent/fluentd]
 
-fluent-plugin-google-cloud gem includes two plugins:
-1. A {filter plugin for fluentd}[http://docs.fluentd.org/articles/filter-plugin-overview]
-that embeds insertIds into log entries to guarantee order and uniqueness.
-2. An {output plugin for fluentd}[http://docs.fluentd.org/articles/output-plugin-overview]
-which sends logs to the {Stackdriver Logging API}[https://cloud.google.com/logging/docs/api/].
+fluent-plugin-google-cloud is an
+{output plugin for fluentd}[http://docs.fluentd.org/articles/output-plugin-overview]
+which sends logs to the
+{Stackdriver Logging API}[https://cloud.google.com/logging/docs/api/].
 
 This is an official Google Ruby gem.
 
@@ -13,7 +12,8 @@ This is an official Google Ruby gem.
 
 == Installation
 
-This gem is hosted at {RubyGems.org}[https://rubygems.org/gems/fluent-plugin-google-cloud]
+This gem is hosted at
+{RubyGems.org}[https://rubygems.org/gems/fluent-plugin-google-cloud]
 and can be installed using:
 
     $ gem install fluent-plugin-google-cloud
@@ -23,31 +23,22 @@ will also install and configure the gem.
 
 == Configuration
 
-To embed insertIds into log entries, specify <code>@type add_insert_ids</code>
-in a {filter clause}[https://docs.fluentd.org/v1.0/articles/config-file#(3)-%E2%80%9Cfilter%E2%80%9D:-event-processing-pipeline]
-of your Fluentd configuration file, for example:
-
-    <filter **>
-      @type add_insert_ids
-      insert_id_key my_insert_id_field_name # Optional.
-    </filter>
-
-insert_id_key can be used to customize the insertId field name.
-
-To send logs to Google Cloud Logging, specify <code>@type google_cloud</code>
-in a {match clause}[http://docs.fluentd.org/articles/config-file#2-ldquomatchrdquo-tell-fluentd-what-to-do]
-of your Fluentd configuration file, for example:
+To send logs to Google Cloud Logging, specify <code>type google_cloud</code>
+in a
+{match clause}[http://docs.fluentd.org/articles/config-file#2-ldquomatchrdquo-tell-fluentd-what-to-do]
+of your fluentd configuration file, for example:
 
     <match **>
-      @type google_cloud
+      type google_cloud
     </match>
 
-See detailed instructions on how to configure this output plugin {here}[https://cloud.google.com/logging/docs/agent/configuration#cloud-fluentd-config].
-The plugin uses
+No further configuration is required. The plugin uses
 {Google Application Default Credentials}[https://developers.google.com/identity/protocols/application-default-credentials]
-for authorization - for additional information see
+for authorization - for additional information see 
 {here}[https://cloud.google.com/logging/docs/agent/authorization].
 
+<em>The previously documented parameters auth_method, private_key_email,
+and private_key_path are removed, and can no longer be used.</em>
 
 == Copyright
 

data/Rakefile

@@ -21,7 +21,8 @@ end
 desc 'Fix file permissions'
 task :fix_perms do
   files = [
-    'lib/fluent/plugin/*.rb',
+    'lib/fluent/plugin/out_google_cloud.rb',
+    'lib/fluent/plugin/monitoring.rb',
     'lib/google/**/*.rb'
   ].flat_map do |file|
     file.include?('*') ? Dir.glob(file) : [file]

data/fluent-plugin-google-cloud.gemspec

@@ -1,17 +1,17 @@
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-google-cloud'
   gem.description   = <<-eos
-   Fluentd plugins for the Stackdriver Logging API, which will make logs
-   viewable in the Stackdriver Logs Viewer and can optionally store them
-   in Google Cloud Storage and/or BigQuery.
+   Fluentd output plugin for the Stackdriver Logging API, which will make
+   logs viewable in the Developer Console's log viewer and can optionally
+   store them in Google Cloud Storage and/or BigQuery.
    This is an official Google Ruby gem.
 eos
-  gem.summary       = 'fluentd plugins for the Stackdriver Logging API'
+  gem.summary       = 'fluentd output plugin for the Stackdriver Logging API'
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.25.1'
-  gem.authors       = ['Stackdriver Agents Team']
+  gem.version       = '0.7.0.pre.1'
+  gem.authors       = ['Ling Huang', 'Igor Peshansky']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
 
@@ -19,13 +19,13 @@ eos
   gem.test_files    = gem.files.grep(/^(test)/)
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency 'fluentd', '~> 0.10'
+  gem.add_runtime_dependency 'fluentd', '~> 1.2.5'
   gem.add_runtime_dependency 'googleapis-common-protos', '~> 1.3'
   gem.add_runtime_dependency 'google-api-client', '~> 0.17'
   gem.add_runtime_dependency 'google-cloud-logging', '~> 1.3', '>= 1.3.2'
   gem.add_runtime_dependency 'googleauth', '~> 0.6'
   gem.add_runtime_dependency 'grpc', '~> 1.0'
-  gem.add_runtime_dependency 'json', '~> 2.1'
+  gem.add_runtime_dependency 'json', '~> 1.8'
   gem.add_runtime_dependency 'google-protobuf', '~> 3.6', '>= 3.6.1'
 
   gem.add_development_dependency 'mocha', '~> 1.1'

data/lib/fluent/plugin/out_google_cloud.rb

@@ -86,7 +86,7 @@ module Fluent
         resource_type: 'container',
         extra_resource_labels: %w(namespace_id pod_id container_name),
         extra_common_labels: %w(namespace_name pod_name),
-        metadata_attributes: %w(cluster-name cluster-location),
+        metadata_attributes: %w(kube-env),
         stream_severity_map: {
           'stdout' => 'INFO',
           'stderr' => 'ERROR'
@@ -228,16 +228,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
-    # Extract plugin version by finding the spec this file was loaded from.
-    PLUGIN_VERSION = begin
-      dependency = Gem::Dependency.new('fluent-plugin-google-cloud')
-      all_specs, = Gem::SpecFetcher.fetcher.spec_for_dependency(dependency)
-      matching_spec, = all_specs.grep(
-        proc { |spec,| __FILE__.include?(spec.full_gem_path) }) do |spec,|
-          spec.version.to_s
-        end
-      matching_spec
-    end.freeze
+    PLUGIN_VERSION = '0.7.0.pre.1'.freeze
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
@@ -550,12 +541,10 @@ module Fluent
         @write_request = method(:write_request_via_rest)
       end
 
-      if [Platform::GCE, Platform::EC2].include?(@platform)
-        # Log an informational message containing the Logs viewer URL
-        @log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
-                  "viewer?project=#{@project_id}&resource=#{@resource.type}/",
-                  "instance_id/#{@vm_id}"
-      end
+      # Log an informational message containing the Logs viewer URL
+      @log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
+                "viewer?project=#{@project_id}&resource=#{@resource.type}/",
+                "instance_id/#{@vm_id}"
     end
 
     def start
@@ -586,17 +575,16 @@ module Fluent
 
           is_json = false
           if @detect_json
-            # Save the following fields if available, then clear them out to
+            # Save the timestamp and severity if available, then clear it out to
             # allow for determining whether we should parse the log or message
             # field.
-            preserved_keys = [
-              'time', 'severity', @trace_key, @span_id_key, @insert_id_key
-            ]
+            timestamp = record.delete('time')
+            severity = record.delete('severity')
 
             # If the log is json, we want to export it as a structured log
             # unless there is additional metadata that would be lost.
             record_json = nil
-            if (record.keys - preserved_keys).length == 1
+            if record.length == 1
               %w(log message msg).each do |field|
                 if record.key?(field)
                   record_json = parse_json_or_nil(record[field])
@@ -604,15 +592,13 @@ module Fluent
               end
             end
             unless record_json.nil?
-              # Propagate these if necessary. Note that we don't want to
-              # override these keys in the JSON we've just parsed.
-              preserved_keys.each do |key|
-                record_json[key] ||= record[key] if record.key?(key)
-              end
-
               record = record_json
               is_json = true
             end
+            # Restore timestamp and severity if necessary. Note that we don't
+            # want to override these keys in the JSON we've just parsed.
+            record['time'] ||= timestamp if timestamp
+            record['severity'] ||= severity if severity
           end
 
           ts_secs, ts_nanos = compute_timestamp(
@@ -626,10 +612,13 @@ module Fluent
                                             ts_secs,
                                             ts_nanos)
 
-          trace = record.delete(@trace_key)
-          entry.trace = trace if trace
+          # Get fully-qualified trace id for LogEntry "trace" field.
+          fq_trace_id = record.delete(@trace_key)
+          entry.trace = fq_trace_id if fq_trace_id
+
           span_id = record.delete(@span_id_key)
           entry.span_id = span_id if span_id
+
           insert_id = record.delete(@insert_id_key)
           entry.insert_id = insert_id if insert_id
 
@@ -1029,10 +1018,8 @@ module Fluent
       # All metadata parameters must now be set.
       missing = []
       missing << 'project_id' unless @project_id
-      if @platform != Platform::OTHER
-        missing << 'zone' unless @zone
-        missing << 'vm_id' unless @vm_id
-      end
+      missing << 'zone' unless @zone
+      missing << 'vm_id' unless @vm_id
       return if missing.empty?
       raise Fluent::ConfigError,
             "Unable to obtain metadata parameters: #{missing.join(' ')}"
@@ -1148,11 +1135,12 @@ module Fluent
 
       # GKE container.
       when GKE_CONSTANTS[:resource_type]
+        raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
+        kube_env = YAML.load(raw_kube_env)
         return {
           'instance_id' => @vm_id,
           'zone' => @zone,
-          'cluster_name' =>
-            fetch_gce_metadata('instance/attributes/cluster-name')
+          'cluster_name' => cluster_name_from_kube_env(kube_env)
         }
 
       # Cloud Dataproc.
@@ -1534,6 +1522,15 @@ module Fluent
       end
     end
 
+    def cluster_name_from_kube_env(kube_env)
+      return kube_env['CLUSTER_NAME'] if kube_env.key?('CLUSTER_NAME')
+      instance_prefix = kube_env['INSTANCE_PREFIX']
+      gke_name_match = /^gke-(.+)-[0-9a-f]{8}$/.match(instance_prefix)
+      return gke_name_match.captures[0] if gke_name_match &&
+                                           !gke_name_match.captures.empty?
+      instance_prefix
+    end
+
     def time_or_nil(ts_secs, ts_nanos)
       Time.at((Integer ts_secs), (Integer ts_nanos) / 1_000.0)
     rescue ArgumentError, TypeError

data/test/plugin/base_test.rb

@@ -81,27 +81,26 @@ module BaseTest
     assert_equal CUSTOM_VM_ID, d.instance.vm_id
   end
 
-  def test_configure_metadata_missing_parts_on_other_platforms
+  def test_configure_invalid_metadata_missing_parts
     setup_no_metadata_service_stubs
     Fluent::GoogleCloudOutput::CredentialsInfo.stubs(:project_id).returns(nil)
-    [[CONFIG_MISSING_METADATA_PROJECT_ID, ['project_id'], false],
-     [CONFIG_MISSING_METADATA_ZONE, [], true],
-     [CONFIG_MISSING_METADATA_VM_ID, [], true],
-     [CONFIG_MISSING_METADATA_ALL, ['project_id'], false]
-    ].each_with_index do |(config, missing_parts, is_valid_config), index|
+    { CONFIG_MISSING_METADATA_PROJECT_ID => ['project_id'],
+      CONFIG_MISSING_METADATA_ZONE => ['zone'],
+      CONFIG_MISSING_METADATA_VM_ID => ['vm_id'],
+      CONFIG_MISSING_METADATA_ALL => %w(project_id zone vm_id)
+    }.each_with_index do |(config, parts), index|
+      exception_count = 0
       begin
         create_driver(config)
-        assert_true is_valid_config, "Invalid config at index #{index} should "\
-          'have raised an error.'
       rescue Fluent::ConfigError => error
-        assert_false is_valid_config, "Valid config at index #{index} should "\
-          "not have raised an error #{error}."
         assert error.message.include?('Unable to obtain metadata parameters:'),
                "Index #{index} failed."
-        missing_parts.each do |part|
+        parts.each do |part|
           assert error.message.include?(part), "Index #{index} failed."
         end
+        exception_count += 1
       end
+      assert_equal 1, exception_count, "Index #{index} failed."
     end
   end
 
@@ -180,7 +179,7 @@ module BaseTest
     setup_gce_metadata_stubs
     # This would cause the resource type to be container.googleapis.com if not
     # for the detect_subservice=false config.
-    setup_k8s_metadata_stubs
+    setup_container_metadata_stubs
     d = create_driver(NO_DETECT_SUBSERVICE_CONFIG)
     d.run
     assert_equal COMPUTE_CONSTANTS[:resource_type], d.instance.resource.type
@@ -468,7 +467,7 @@ module BaseTest
 
   def test_structured_payload_json_log_default_container_not_parsed
     setup_gce_metadata_stubs
-    setup_k8s_metadata_stubs
+    setup_container_metadata_stubs
     json_string = '{"msg": "test log entry 0", "tag2": "test", ' \
                   '"data": 5000, "some_null_field": null}'
     setup_logging_stubs do
@@ -485,7 +484,7 @@ module BaseTest
 
   def test_structured_payload_json_log_detect_json_container_not_parsed
     setup_gce_metadata_stubs
-    setup_k8s_metadata_stubs
+    setup_container_metadata_stubs
     json_string = '{"msg": "test log entry 0", "tag2": "test", ' \
                   '"data": 5000, "some_null_field": null}'
     setup_logging_stubs do
@@ -500,7 +499,7 @@ module BaseTest
 
   def test_structured_payload_json_log_detect_json_container_parsed
     setup_gce_metadata_stubs
-    setup_k8s_metadata_stubs
+    setup_container_metadata_stubs
     json_string = '{"msg": "test log entry 0", "tag2": "test", ' \
                   '"data": 5000, "some_null_field": null}'
     setup_logging_stubs do
@@ -539,7 +538,7 @@ module BaseTest
   # match, thus the original tag is used as the log name.
   def test_handle_empty_container_name
     setup_gce_metadata_stubs
-    setup_k8s_metadata_stubs
+    setup_container_metadata_stubs
     container_name = ''
     # This tag will not match the kubernetes regex because it requires a
     # non-empty container name.
@@ -561,7 +560,7 @@ module BaseTest
   # 'require_valid_tags' is true.
   def test_reject_non_utf8_container_name_with_require_valid_tags_true
     setup_gce_metadata_stubs
-    setup_k8s_metadata_stubs
+    setup_container_metadata_stubs
     non_utf8_tags = INVALID_TAGS.select do |tag, _|
       tag.is_a?(String) && !tag.empty?
     end
@@ -601,7 +600,7 @@ module BaseTest
   # Verify that tags extracted from container names are properly encoded.
   def test_encode_tags_from_container_name_with_require_valid_tags_true
     setup_gce_metadata_stubs
-    setup_k8s_metadata_stubs
+    setup_container_metadata_stubs
     VALID_TAGS.each do |tag, encoded_tag|
       setup_logging_stubs do
         @logs_sent = []
@@ -640,7 +639,7 @@ module BaseTest
   # sanitized.
   def test_sanitize_tags_from_container_name_with_require_valid_tags_false
     setup_gce_metadata_stubs
-    setup_k8s_metadata_stubs
+    setup_container_metadata_stubs
     # Log names are derived from container names for containers. And container
     # names are extracted from the tag based on a regex match pattern. As a
     # prerequisite, the tag should already be a string, thus we only test
@@ -969,7 +968,7 @@ module BaseTest
 
   def test_one_container_log_from_tag_stderr
     setup_gce_metadata_stubs
-    setup_k8s_metadata_stubs
+    setup_container_metadata_stubs
     setup_logging_stubs do
       d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
       d.emit(container_log_entry(log_entry(0), 'stderr'))
@@ -980,15 +979,15 @@ module BaseTest
     ) { |_, oldval, newval| oldval.merge(newval) }
     verify_log_entries(1, expected_params) do |entry, i|
       verify_default_log_entry_text(entry['textPayload'], i, entry)
-      assert_equal K8S_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
-      assert_equal K8S_NANOS, entry['timestamp']['nanos'], entry
+      assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
       assert_equal 'ERROR', entry['severity'], entry
     end
   end
 
   def test_json_container_log_metadata_from_plugin
     setup_gce_metadata_stubs
-    setup_k8s_metadata_stubs
+    setup_container_metadata_stubs
     setup_logging_stubs do
       d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
       d.emit(container_log_entry_with_metadata('{"msg": "test log entry 0", ' \
@@ -1003,15 +1002,15 @@ module BaseTest
       assert_equal 'test log entry 0', get_string(fields['msg']), entry
       assert_equal 'test', get_string(fields['tag2']), entry
       assert_equal 5000, get_number(fields['data']), entry
-      assert_equal K8S_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
-      assert_equal K8S_NANOS, entry['timestamp']['nanos'], entry
+      assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
       assert_equal 'WARNING', entry['severity'], entry
     end
   end
 
   def test_json_container_log_metadata_from_tag
     setup_gce_metadata_stubs
-    setup_k8s_metadata_stubs
+    setup_container_metadata_stubs
     setup_logging_stubs do
       d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
       d.emit(container_log_entry('{"msg": "test log entry 0", ' \
@@ -1026,8 +1025,8 @@ module BaseTest
       assert_equal 'test log entry 0', get_string(fields['msg']), entry
       assert_equal 'test', get_string(fields['tag2']), entry
       assert_equal 5000, get_number(fields['data']), entry
-      assert_equal K8S_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
-      assert_equal K8S_NANOS, entry['timestamp']['nanos'], entry
+      assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
       assert_equal 'WARNING', entry['severity'], entry
     end
   end
@@ -1222,32 +1221,18 @@ module BaseTest
 
   def test_log_entry_trace_field
     verify_field_key('trace', DEFAULT_TRACE_KEY, 'custom_trace_key',
-                     CONFIG_CUSTOM_TRACE_KEY_SPECIFIED, TRACE)
+                     CONFIG_CUSTOM_TRACE_KEY_SPECIFIED,
+                     'projects/proj1/traces/1234567890abcdef1234567890abcdef')
   end
 
   def test_log_entry_span_id_field
     verify_field_key('spanId', DEFAULT_SPAN_ID_KEY, 'custom_span_id_key',
-                     CONFIG_CUSTOM_SPAN_ID_KEY_SPECIFIED, SPAN_ID)
+                     CONFIG_CUSTOM_SPAN_ID_KEY_SPECIFIED, '000000000000004a')
   end
 
   def test_log_entry_insert_id_field
     verify_field_key('insertId', DEFAULT_INSERT_ID_KEY, 'custom_insert_id_key',
-                     CONFIG_CUSTOM_INSERT_ID_KEY_SPECIFIED, INSERT_ID)
-  end
-
-  def test_cascading_json_detection_with_log_entry_trace_field
-    verify_cascading_json_detection_with_log_entry_fields(
-      'trace', DEFAULT_TRACE_KEY, TRACE, TRACE2)
-  end
-
-  def test_cascading_json_detection_with_log_entry_span_id_field
-    verify_cascading_json_detection_with_log_entry_fields(
-      'spanId', DEFAULT_SPAN_ID_KEY, SPAN_ID, SPAN_ID2)
-  end
-
-  def test_cascading_json_detection_with_log_entry_insert_id_field
-    verify_cascading_json_detection_with_log_entry_fields(
-      'insertId', DEFAULT_INSERT_ID_KEY, INSERT_ID, INSERT_ID2)
+                     CONFIG_CUSTOM_INSERT_ID_KEY_SPECIFIED, 'fah7yr7iw64tg857y')
   end
 
   # Metadata Agent related tests.
@@ -1340,24 +1325,25 @@ module BaseTest
     end
   end
 
-  # Test k8s_container monitored resource including the fallback when Metadata
-  # Agent restarts.
-  def test_k8s_container_monitored_resource_fallback
+  # Test k8s monitored resource including the fallback when Metadata Agent
+  # restarts.
+  def test_k8s_monitored_resource_fallback
     [
+      # k8s_container.
       # When enable_metadata_agent is false.
       {
         config: APPLICATION_DEFAULT_CONFIG,
         setup_metadata_agent_stub: false,
         setup_k8s_stub: false,
         log_entry: k8s_container_log_entry(log_entry(0)),
-        expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
+        expected_params: COMPUTE_PARAMS
       },
       {
         config: APPLICATION_DEFAULT_CONFIG,
         setup_metadata_agent_stub: true,
         setup_k8s_stub: false,
         log_entry: k8s_container_log_entry(log_entry(0)),
-        expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
+        expected_params: COMPUTE_PARAMS
       },
       {
         config: APPLICATION_DEFAULT_CONFIG,
@@ -1379,7 +1365,7 @@ module BaseTest
         setup_metadata_agent_stub: false,
         setup_k8s_stub: false,
         log_entry: k8s_container_log_entry(log_entry(0)),
-        expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
+        expected_params: COMPUTE_PARAMS
       },
       {
         config: ENABLE_METADATA_AGENT_CONFIG,
@@ -1415,30 +1401,7 @@ module BaseTest
         setup_k8s_stub: true,
         log_entry: k8s_container_log_entry(log_entry(0)),
         expected_params: K8S_CONTAINER_PARAMS
-      }
-    ].each do |test_params|
-      new_stub_context do
-        setup_gce_metadata_stubs
-        setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
-        setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
-        setup_logging_stubs do
-          d = create_driver(test_params[:config], CONTAINER_TAG)
-          d.emit(test_params[:log_entry])
-          d.run
-        end
-        verify_log_entries(1, test_params[:expected_params],
-                           'jsonPayload') do |entry|
-          fields = get_fields(entry['jsonPayload'])
-          assert_equal 2, fields.size, entry
-          assert_equal 'test log entry 0', get_string(fields['log']), entry
-          assert_equal K8S_STREAM, get_string(fields['stream']), entry
-        end
-      end
-    end
-  end
-
-  def test_k8s_container_monitored_resource_invalid_local_resource_id
-    [
+      },
       # When local_resource_id is not present or does not match k8s regexes.
       {
         config: ENABLE_METADATA_AGENT_CONFIG,
@@ -1446,7 +1409,7 @@ module BaseTest
         setup_k8s_stub: true,
         log_entry: k8s_container_log_entry(
           log_entry(0)).reject { |k, _| k == LOCAL_RESOURCE_ID_KEY },
-        expected_params: CONTAINER_FROM_TAG_PARAMS
+        expected_params: COMPUTE_PARAMS
       },
       {
         config: ENABLE_METADATA_AGENT_CONFIG,
@@ -1455,29 +1418,11 @@ module BaseTest
         log_entry: k8s_container_log_entry(
           log_entry(0),
           local_resource_id: RANDOM_LOCAL_RESOURCE_ID),
-        expected_params: CONTAINER_FROM_TAG_PARAMS
-      }
-    ].each do |test_params|
-      new_stub_context do
-        setup_gce_metadata_stubs
-        setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
-        setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
-        setup_logging_stubs do
-          d = create_driver(test_params[:config], CONTAINER_TAG)
-          d.emit(test_params[:log_entry])
-          d.run
-        end
-        verify_log_entries(1, test_params[:expected_params]) do |entry|
-          assert_equal 'test log entry 0', entry['textPayload'], entry
-        end
-      end
-    end
-  end
-
-  # Test k8s_node monitored resource including the fallback when Metadata Agent
-  # restarts.
-  def test_k8s_node_monitored_resource_fallback
-    [
+        # When 'kube-env' is present, "compute.googleapis.com/resource_name" is
+        # not added.
+        expected_params: COMPUTE_PARAMS
+      },
+      # Specific cases for k8s_node.
       {
         config: APPLICATION_DEFAULT_CONFIG,
         setup_metadata_agent_stub: true,
@@ -1516,8 +1461,16 @@ module BaseTest
     ].each do |test_params|
       new_stub_context do
         setup_gce_metadata_stubs
-        setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
-        setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
+        if test_params[:setup_metadata_agent_stub]
+          setup_metadata_agent_stubs
+        else
+          setup_no_metadata_agent_stubs
+        end
+        if test_params[:setup_k8s_stub]
+          setup_k8s_metadata_stubs
+        else
+          setup_no_k8s_metadata_stubs
+        end
         setup_logging_stubs do
           d = create_driver(test_params[:config])
           d.emit(test_params[:log_entry])
@@ -1594,7 +1547,7 @@ module BaseTest
     [1, 2, 3, 5, 11, 50].each do |n|
       new_stub_context do
         setup_gce_metadata_stubs
-        setup_k8s_metadata_stubs
+        setup_container_metadata_stubs
         setup_metadata_agent_stubs
         setup_logging_stubs do
           d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
@@ -1606,7 +1559,7 @@ module BaseTest
         verify_log_entries(n, CONTAINER_FROM_APPLICATION_PARAMS)
         assert_requested_metadata_agent_stub(
           "#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
-          ".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}")
+          ".#{CONTAINER_POD_NAME}.#{CONTAINER_CONTAINER_NAME}")
       end
     end
   end
@@ -1686,34 +1639,44 @@ module BaseTest
                           MANAGED_VM_BACKEND_VERSION)
   end
 
-  def setup_k8s_metadata_stubs(should_respond = true)
-    if should_respond
-      stub_metadata_request(
-        'instance/attributes/',
-        "attribute1\ncluster-location\ncluster-name\nlast_attribute")
-      stub_metadata_request('instance/attributes/cluster-location',
-                            K8S_LOCATION2)
-      stub_metadata_request('instance/attributes/cluster-name',
-                            K8S_CLUSTER_NAME)
-    else
-      ['cluster-location', 'cluster-name'].each do |metadata_name|
-        stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
-          .to_return(status: 404,
-                     body: 'The requested URL /computeMetadata/v1/instance/' \
-                           "attributes/#{metadata_name} was not found on this" \
-                           ' server.')
-      end
+  def setup_container_metadata_stubs
+    stub_metadata_request(
+      'instance/attributes/',
+      "attribute1\nkube-env\nlast_attribute")
+    stub_metadata_request('instance/attributes/kube-env',
+                          "ENABLE_NODE_LOGGING: \"true\"\n"\
+                          'INSTANCE_PREFIX: '\
+                          "gke-#{CONTAINER_CLUSTER_NAME}-740fdafa\n"\
+                          'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
+  end
+
+  def setup_k8s_metadata_stubs
+    stub_metadata_request(
+      'instance/attributes/',
+      "attribute1\ncluster-name\ncluster-location\nlast_attribute")
+    stub_metadata_request('instance/attributes/cluster-location', K8S_LOCATION2)
+    stub_metadata_request('instance/attributes/cluster-name', K8S_CLUSTER_NAME)
+  end
+
+  def setup_no_k8s_metadata_stubs
+    ['cluster-location', 'cluster-name'].each do |metadata_name|
+      stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
+        .to_return(status: 404,
+                   body: 'The requested URL /computeMetadata/v1/instance/' \
+                         "attributes/#{metadata_name} was not found on this" \
+                         ' server.')
     end
   end
 
   def setup_cloudfunctions_metadata_stubs
     stub_metadata_request(
       'instance/attributes/',
-      "attribute1\ncluster-location\ncluster-name\ngcf_region\nlast_attribute")
-    stub_metadata_request('instance/attributes/cluster-location',
-                          K8S_LOCATION2)
-    stub_metadata_request('instance/attributes/cluster-name',
-                          K8S_CLUSTER_NAME)
+      "attribute1\nkube-env\ngcf_region\nlast_attribute")
+    stub_metadata_request('instance/attributes/kube-env',
+                          "ENABLE_NODE_LOGGING: \"true\"\n"\
+                          'INSTANCE_PREFIX: '\
+                          "gke-#{CLOUDFUNCTIONS_CLUSTER_NAME}-740fdafa\n"\
+                          'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
     stub_metadata_request('instance/attributes/gcf_region',
                           CLOUDFUNCTIONS_REGION)
   end
@@ -1748,20 +1711,19 @@ module BaseTest
     WebMock.reset!
   end
 
-  def setup_metadata_agent_stubs(should_respond = true)
-    if should_respond
-      MONITORED_RESOURCE_STUBS.each do |local_resource_id, resource|
-        stub_request(:get, metadata_request_url(local_resource_id))
-          .to_return(status: 200, body: resource)
-      end
-      stub_request(:get, metadata_request_url(RANDOM_LOCAL_RESOURCE_ID))
-        .to_return(status: 404, body: '')
-    else
-      # Simulate an environment with no metadata agent endpoint present.
-      stub_request(:get,
-                   %r{#{DEFAULT_METADATA_AGENT_URL}\/monitoredResource/.*})
-        .to_raise(Errno::EHOSTUNREACH)
+  def setup_metadata_agent_stubs
+    MONITORED_RESOURCE_STUBS.each do |local_resource_id, resource|
+      stub_request(:get, metadata_request_url(local_resource_id))
+        .to_return(status: 200, body: resource)
     end
+    stub_request(:get, metadata_request_url(RANDOM_LOCAL_RESOURCE_ID))
+      .to_return(status: 404, body: '')
+  end
+
+  def setup_no_metadata_agent_stubs
+    # Simulate an environment with no metadata agent endpoint present.
+    stub_request(:get, %r{#{DEFAULT_METADATA_AGENT_URL}\/monitoredResource/.*})
+      .to_raise(Errno::EHOSTUNREACH)
   end
 
   def assert_requested_metadata_agent_stub(local_resource_id)
@@ -1771,20 +1733,21 @@ module BaseTest
   # GKE Container.
 
   def container_tag_with_container_name(container_name)
-    "kubernetes.#{K8S_POD_NAME}_#{K8S_NAMESPACE_NAME}_#{container_name}"
+    "kubernetes.#{CONTAINER_POD_NAME}_#{CONTAINER_NAMESPACE_NAME}_" \
+      "#{container_name}"
   end
 
   def container_log_entry_with_metadata(
-      log, container_name = K8S_CONTAINER_NAME)
+      log, container_name = CONTAINER_CONTAINER_NAME)
     {
       log: log,
-      stream: K8S_STREAM,
-      time: K8S_TIMESTAMP,
+      stream: CONTAINER_STREAM,
+      time: CONTAINER_TIMESTAMP,
       kubernetes: {
         namespace_id: CONTAINER_NAMESPACE_ID,
-        namespace_name: K8S_NAMESPACE_NAME,
+        namespace_name: CONTAINER_NAMESPACE_NAME,
         pod_id: CONTAINER_POD_ID,
-        pod_name: K8S_POD_NAME,
+        pod_name: CONTAINER_POD_NAME,
         container_name: container_name,
         labels: {
           CONTAINER_LABEL_KEY => CONTAINER_LABEL_VALUE
@@ -1793,11 +1756,11 @@ module BaseTest
     }
   end
 
-  def container_log_entry(log, stream = K8S_STREAM)
+  def container_log_entry(log, stream = CONTAINER_STREAM)
     {
       log: log,
       stream: stream,
-      time: K8S_TIMESTAMP
+      time: CONTAINER_TIMESTAMP
     }
   end
 
@@ -1806,7 +1769,7 @@ module BaseTest
       log: log,
       LOCAL_RESOURCE_ID_KEY =>
         "#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
-        ".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}"
+        ".#{CONTAINER_POD_NAME}.#{CONTAINER_CONTAINER_NAME}"
     }
   end
 
@@ -1897,13 +1860,6 @@ module BaseTest
     }
   end
 
-  def structured_log_entry
-    {
-      'name' => 'test name',
-      'code' => 'test code'
-    }
-  end
-
   def log_entry(i)
     "test log entry #{i}"
   end
@@ -1972,7 +1928,7 @@ module BaseTest
 
   def verify_container_logs(log_entry_factory, expected_params)
     setup_gce_metadata_stubs
-    setup_k8s_metadata_stubs
+    setup_container_metadata_stubs
     [1, 2, 3, 5, 11, 50].each do |n|
       @logs_sent = []
       setup_logging_stubs do
@@ -1982,8 +1938,9 @@ module BaseTest
       end
       verify_log_entries(n, expected_params) do |entry, i|
         verify_default_log_entry_text(entry['textPayload'], i, entry)
-        assert_equal K8S_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
-        assert_equal K8S_NANOS, entry['timestamp']['nanos'], entry
+        assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'],
+                     entry
+        assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
         assert_equal CONTAINER_SEVERITY, entry['severity'], entry
       end
     end
@@ -2052,71 +2009,6 @@ module BaseTest
     end
   end
 
-  # Cascading JSON detection is only triggered when the record has one field
-  # left with name "log", "message" or "msg". This test verifies additional
-  # LogEntry fields like spanId and traceId do not disable that by accident.
-  def verify_cascading_json_detection_with_log_entry_fields(
-      log_entry_field, default_key, root_level_value, nested_level_value)
-    setup_gce_metadata_stubs
-
-    # {
-    #   "logging.googleapis.com/XXX' => 'sample value'
-    #   "msg": {
-    #     "name": "test name",
-    #     "code": "test code"
-    #   }
-    # }
-    log_entry_with_root_level_field = {
-      default_key => root_level_value,
-      'msg' => structured_log_entry.to_json
-    }
-    # {
-    #   "msg": {
-    #     "logging.googleapis.com/XXX' => 'another value',
-    #     "name": "test name",
-    #     "code": "test code"
-    #   }
-    # }
-    log_entry_with_nested_level_field = {
-      'msg' => {
-        default_key => nested_level_value
-      }.merge(structured_log_entry).to_json
-    }
-    # {
-    #   "logging.googleapis.com/XXX' => 'sample value'
-    #   "msg": {
-    #     "logging.googleapis.com/XXX' => 'another value',
-    #     "name": "test name",
-    #     "code": "test code"
-    #   }
-    # }
-    log_entry_with_both_level_fields = log_entry_with_nested_level_field.merge(
-      default_key => root_level_value)
-
-    {
-      log_entry_with_root_level_field => root_level_value,
-      log_entry_with_nested_level_field => nested_level_value,
-      log_entry_with_both_level_fields => nested_level_value
-    }.each_with_index do |(input_log_entry, expected_value), index|
-      setup_logging_stubs do
-        @logs_sent = []
-        d = create_driver(DETECT_JSON_CONFIG)
-        d.emit(input_log_entry)
-        d.run
-      end
-      verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
-        assert_equal expected_value, entry[log_entry_field],
-                     "Index #{index} failed. #{expected_value} is expected" \
-                     " for #{log_entry_field} field."
-        payload_fields = get_fields(entry['jsonPayload'])
-        assert_equal structured_log_entry.size, payload_fields.size
-        payload_fields.each do |key, value|
-          assert_equal structured_log_entry[key], get_string(value)
-        end
-      end
-    end
-  end
-
   def verify_field_key(log_entry_field, default_key, custom_key,
                        custom_key_config, sample_value)
     setup_gce_metadata_stubs