fluent-plugin-google-cloud 0.6.24 → 0.6.25.pre.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e9d4e0c7112b19f2fdf43ac7bad7c2d030932f63
-  data.tar.gz: ea83fcd3888666a51aae9c4a03ea0ffb0bdba527
+  metadata.gz: ae6de5c365ece78d5541faf3c03e1ebbd7dd218a
+  data.tar.gz: 8236d705f453819f152eb08130c0c1bdfb452080
 SHA512:
-  metadata.gz: 2b179f5f31c11216c3e04b5d071cb244700f8352ed11fe1fa00fcf3282e5016e52a02e1880859d92a445d641e9bc1280a861a19896e5166ee493ad9b81569b2a
-  data.tar.gz: 3e9e799eb94b3e35156ce44cb8511f942082ac2ebc6bde9d21590543b421148f029f24f499c5373c7ca60a57054e066bbce3c9b19e8c312fa562f902420edf1d
+  metadata.gz: '0695b58ed5d1b52cde93dad05374fb761e56ec0163594d6987ccd6d67c47a517009a4363032c634f36a1c946cbbf0534c1a662b019d24d81777ff95f2fe7a730'
+  data.tar.gz: f63241976bed1b8bcda026de9e20fff9d91347879a9fded573f5b24e58676fe8d2e84c8d1f823b9d40b8a412fdb421dde184b322145e27fafadbc5e5ab89c765

data/Gemfile.lock

@@ -0,0 +1,159 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-google-cloud (0.6.25.pre.1)
+      fluentd (~> 0.10)
+      google-api-client (~> 0.17)
+      google-cloud-logging (~> 1.3, >= 1.3.2)
+      google-protobuf (~> 3.6, >= 3.6.1)
+      googleapis-common-protos (~> 1.3)
+      googleauth (~> 0.6)
+      grpc (~> 1.0)
+      json (~> 2.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    ast (2.4.0)
+    cool.io (1.5.3)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    declarative (0.0.10)
+    declarative-option (0.1.0)
+    faraday (0.15.2)
+      multipart-post (>= 1.2, < 3)
+    fluentd (0.14.25)
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 0.7.0, < 2.0.0)
+      ruby_dig (~> 0.0.2)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (~> 0.1.7)
+      tzinfo (~> 1.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    google-api-client (0.23.9)
+      addressable (~> 2.5, >= 2.5.1)
+      googleauth (>= 0.5, < 0.7.0)
+      httpclient (>= 2.8.1, < 3.0)
+      mime-types (~> 3.0)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.0)
+      signet (~> 0.9)
+    google-cloud-core (1.2.6)
+      google-cloud-env (~> 1.0)
+    google-cloud-env (1.0.4)
+      faraday (~> 0.11)
+    google-cloud-logging (1.5.4)
+      google-cloud-core (~> 1.2)
+      google-gax (~> 1.3)
+      googleapis-common-protos-types (>= 1.0.2)
+      stackdriver-core (~> 1.3)
+    google-gax (1.3.0)
+      google-protobuf (~> 3.2)
+      googleapis-common-protos (>= 1.3.5, < 2.0)
+      googleauth (~> 0.6.2)
+      grpc (>= 1.7.2, < 2.0)
+      rly (~> 0.2.3)
+    google-protobuf (3.6.1)
+    googleapis-common-protos (1.3.7)
+      google-protobuf (~> 3.0)
+      googleapis-common-protos-types (~> 1.0)
+      grpc (~> 1.0)
+    googleapis-common-protos-types (1.0.2)
+      google-protobuf (~> 3.0)
+    googleauth (0.6.6)
+      faraday (~> 0.12)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.12)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (~> 0.7)
+    grpc (1.15.0)
+      google-protobuf (~> 3.1)
+      googleapis-common-protos-types (~> 1.0.0)
+    hashdiff (0.3.7)
+    http_parser.rb (0.6.0)
+    httpclient (2.8.3)
+    json (2.1.0)
+    jwt (2.1.0)
+    memoist (0.16.0)
+    metaclass (0.0.4)
+    mime-types (3.2.2)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2018.0812)
+    mocha (1.7.0)
+      metaclass (~> 0.0.1)
+    msgpack (1.2.4)
+    multi_json (1.13.1)
+    multipart-post (2.0.0)
+    os (1.0.0)
+    parser (2.5.1.2)
+      ast (~> 2.4.0)
+    power_assert (1.1.3)
+    powerpack (0.1.2)
+    prometheus-client (0.7.1)
+      quantile (~> 0.2.0)
+    public_suffix (3.0.3)
+    quantile (0.2.1)
+    rainbow (2.2.2)
+      rake
+    rake (10.5.0)
+    representable (3.0.4)
+      declarative (< 0.1.0)
+      declarative-option (< 0.2.0)
+      uber (< 0.2.0)
+    retriable (3.1.2)
+    rly (0.2.3)
+    rubocop (0.39.0)
+      parser (>= 2.3.0.7, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.10.0)
+    ruby_dig (0.0.2)
+    safe_yaml (1.0.4)
+    serverengine (2.0.7)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    signet (0.9.1)
+      addressable (~> 2.3)
+      faraday (~> 0.9)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
+    stackdriver-core (1.3.2)
+      google-cloud-core (~> 1.2)
+    strptime (0.1.9)
+    test-unit (3.2.8)
+      power_assert
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2018.5)
+      tzinfo (>= 1.0.0)
+    uber (0.1.0)
+    unicode-display_width (1.4.0)
+    webmock (2.3.2)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-google-cloud!
+  mocha (~> 1.1)
+  prometheus-client (~> 0.7.1)
+  rake (~> 10.3)
+  rubocop (~> 0.39.0)
+  test-unit (~> 3.0)
+  webmock (~> 2.3.1)
+
+BUNDLED WITH
+   1.16.4

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.24'
+  gem.version       = '0.6.25.pre.1'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
@@ -25,7 +25,7 @@ eos
   gem.add_runtime_dependency 'google-cloud-logging', '~> 1.3', '>= 1.3.2'
   gem.add_runtime_dependency 'googleauth', '~> 0.6'
   gem.add_runtime_dependency 'grpc', '~> 1.0'
-  gem.add_runtime_dependency 'json', '~> 1.8'
+  gem.add_runtime_dependency 'json', '~> 2.1'
   gem.add_runtime_dependency 'google-protobuf', '~> 3.6', '>= 3.6.1'
 
   gem.add_development_dependency 'mocha', '~> 1.1'

data/lib/fluent/plugin/filter_add_insert_ids.rb

@@ -100,4 +100,7 @@ module Fluent
       end
     end
   end
+
+  # For Fluentd v0.12 compatibility.
+  AddInsertIdsFilter = Fluent::Plugin::AddInsertIdsFilter
 end

data/lib/fluent/plugin/out_google_cloud.rb

@@ -86,7 +86,7 @@ module Fluent
         resource_type: 'container',
         extra_resource_labels: %w(namespace_id pod_id container_name),
         extra_common_labels: %w(namespace_name pod_name),
-        metadata_attributes: %w(kube-env),
+        metadata_attributes: %w(cluster-name cluster-location),
         stream_severity_map: {
           'stdout' => 'INFO',
           'stderr' => 'ERROR'
@@ -550,10 +550,12 @@ module Fluent
         @write_request = method(:write_request_via_rest)
       end
 
-      # Log an informational message containing the Logs viewer URL
-      @log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
-                "viewer?project=#{@project_id}&resource=#{@resource.type}/",
-                "instance_id/#{@vm_id}"
+      if [Platform::GCE, Platform::EC2].include?(@platform)
+        # Log an informational message containing the Logs viewer URL
+        @log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
+                  "viewer?project=#{@project_id}&resource=#{@resource.type}/",
+                  "instance_id/#{@vm_id}"
+      end
     end
 
     def start
@@ -1027,8 +1029,10 @@ module Fluent
       # All metadata parameters must now be set.
       missing = []
       missing << 'project_id' unless @project_id
-      missing << 'zone' unless @zone
-      missing << 'vm_id' unless @vm_id
+      if @platform != Platform::OTHER
+        missing << 'zone' unless @zone
+        missing << 'vm_id' unless @vm_id
+      end
       return if missing.empty?
       raise Fluent::ConfigError,
             "Unable to obtain metadata parameters: #{missing.join(' ')}"
@@ -1144,12 +1148,11 @@ module Fluent
 
       # GKE container.
       when GKE_CONSTANTS[:resource_type]
-        raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
-        kube_env = YAML.load(raw_kube_env)
         return {
           'instance_id' => @vm_id,
           'zone' => @zone,
-          'cluster_name' => cluster_name_from_kube_env(kube_env)
+          'cluster_name' =>
+            fetch_gce_metadata('instance/attributes/cluster-name')
         }
 
       # Cloud Dataproc.
@@ -1531,15 +1534,6 @@ module Fluent
       end
     end
 
-    def cluster_name_from_kube_env(kube_env)
-      return kube_env['CLUSTER_NAME'] if kube_env.key?('CLUSTER_NAME')
-      instance_prefix = kube_env['INSTANCE_PREFIX']
-      gke_name_match = /^gke-(.+)-[0-9a-f]{8}$/.match(instance_prefix)
-      return gke_name_match.captures[0] if gke_name_match &&
-                                           !gke_name_match.captures.empty?
-      instance_prefix
-    end
-
     def time_or_nil(ts_secs, ts_nanos)
       Time.at((Integer ts_secs), (Integer ts_nanos) / 1_000.0)
     rescue ArgumentError, TypeError

data/test/plugin/base_test.rb

@@ -81,26 +81,27 @@ module BaseTest
     assert_equal CUSTOM_VM_ID, d.instance.vm_id
   end
 
-  def test_configure_invalid_metadata_missing_parts
+  def test_configure_metadata_missing_parts_on_other_platforms
     setup_no_metadata_service_stubs
     Fluent::GoogleCloudOutput::CredentialsInfo.stubs(:project_id).returns(nil)
-    { CONFIG_MISSING_METADATA_PROJECT_ID => ['project_id'],
-      CONFIG_MISSING_METADATA_ZONE => ['zone'],
-      CONFIG_MISSING_METADATA_VM_ID => ['vm_id'],
-      CONFIG_MISSING_METADATA_ALL => %w(project_id zone vm_id)
-    }.each_with_index do |(config, parts), index|
-      exception_count = 0
+    [[CONFIG_MISSING_METADATA_PROJECT_ID, ['project_id'], false],
+     [CONFIG_MISSING_METADATA_ZONE, [], true],
+     [CONFIG_MISSING_METADATA_VM_ID, [], true],
+     [CONFIG_MISSING_METADATA_ALL, ['project_id'], false]
+    ].each_with_index do |(config, missing_parts, is_valid_config), index|
       begin
         create_driver(config)
+        assert_true is_valid_config, "Invalid config at index #{index} should "\
+          'have raised an error.'
       rescue Fluent::ConfigError => error
+        assert_false is_valid_config, "Valid config at index #{index} should "\
+          "not have raised an error #{error}."
         assert error.message.include?('Unable to obtain metadata parameters:'),
                "Index #{index} failed."
-        parts.each do |part|
+        missing_parts.each do |part|
           assert error.message.include?(part), "Index #{index} failed."
         end
-        exception_count += 1
       end
-      assert_equal 1, exception_count, "Index #{index} failed."
     end
   end
 
@@ -179,7 +180,7 @@ module BaseTest
     setup_gce_metadata_stubs
     # This would cause the resource type to be container.googleapis.com if not
     # for the detect_subservice=false config.
-    setup_container_metadata_stubs
+    setup_k8s_metadata_stubs
     d = create_driver(NO_DETECT_SUBSERVICE_CONFIG)
     d.run
     assert_equal COMPUTE_CONSTANTS[:resource_type], d.instance.resource.type
@@ -467,7 +468,7 @@ module BaseTest
 
   def test_structured_payload_json_log_default_container_not_parsed
     setup_gce_metadata_stubs
-    setup_container_metadata_stubs
+    setup_k8s_metadata_stubs
     json_string = '{"msg": "test log entry 0", "tag2": "test", ' \
                   '"data": 5000, "some_null_field": null}'
     setup_logging_stubs do
@@ -484,7 +485,7 @@ module BaseTest
 
   def test_structured_payload_json_log_detect_json_container_not_parsed
     setup_gce_metadata_stubs
-    setup_container_metadata_stubs
+    setup_k8s_metadata_stubs
     json_string = '{"msg": "test log entry 0", "tag2": "test", ' \
                   '"data": 5000, "some_null_field": null}'
     setup_logging_stubs do
@@ -499,7 +500,7 @@ module BaseTest
 
   def test_structured_payload_json_log_detect_json_container_parsed
     setup_gce_metadata_stubs
-    setup_container_metadata_stubs
+    setup_k8s_metadata_stubs
     json_string = '{"msg": "test log entry 0", "tag2": "test", ' \
                   '"data": 5000, "some_null_field": null}'
     setup_logging_stubs do
@@ -538,7 +539,7 @@ module BaseTest
   # match, thus the original tag is used as the log name.
   def test_handle_empty_container_name
     setup_gce_metadata_stubs
-    setup_container_metadata_stubs
+    setup_k8s_metadata_stubs
     container_name = ''
     # This tag will not match the kubernetes regex because it requires a
     # non-empty container name.
@@ -560,7 +561,7 @@ module BaseTest
   # 'require_valid_tags' is true.
   def test_reject_non_utf8_container_name_with_require_valid_tags_true
     setup_gce_metadata_stubs
-    setup_container_metadata_stubs
+    setup_k8s_metadata_stubs
     non_utf8_tags = INVALID_TAGS.select do |tag, _|
       tag.is_a?(String) && !tag.empty?
     end
@@ -600,7 +601,7 @@ module BaseTest
   # Verify that tags extracted from container names are properly encoded.
   def test_encode_tags_from_container_name_with_require_valid_tags_true
     setup_gce_metadata_stubs
-    setup_container_metadata_stubs
+    setup_k8s_metadata_stubs
     VALID_TAGS.each do |tag, encoded_tag|
       setup_logging_stubs do
         @logs_sent = []
@@ -639,7 +640,7 @@ module BaseTest
   # sanitized.
   def test_sanitize_tags_from_container_name_with_require_valid_tags_false
     setup_gce_metadata_stubs
-    setup_container_metadata_stubs
+    setup_k8s_metadata_stubs
     # Log names are derived from container names for containers. And container
     # names are extracted from the tag based on a regex match pattern. As a
     # prerequisite, the tag should already be a string, thus we only test
@@ -968,7 +969,7 @@ module BaseTest
 
   def test_one_container_log_from_tag_stderr
     setup_gce_metadata_stubs
-    setup_container_metadata_stubs
+    setup_k8s_metadata_stubs
     setup_logging_stubs do
       d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
       d.emit(container_log_entry(log_entry(0), 'stderr'))
@@ -979,15 +980,15 @@ module BaseTest
     ) { |_, oldval, newval| oldval.merge(newval) }
     verify_log_entries(1, expected_params) do |entry, i|
       verify_default_log_entry_text(entry['textPayload'], i, entry)
-      assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
-      assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+      assert_equal K8S_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+      assert_equal K8S_NANOS, entry['timestamp']['nanos'], entry
       assert_equal 'ERROR', entry['severity'], entry
     end
   end
 
   def test_json_container_log_metadata_from_plugin
     setup_gce_metadata_stubs
-    setup_container_metadata_stubs
+    setup_k8s_metadata_stubs
     setup_logging_stubs do
       d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
       d.emit(container_log_entry_with_metadata('{"msg": "test log entry 0", ' \
@@ -1002,15 +1003,15 @@ module BaseTest
       assert_equal 'test log entry 0', get_string(fields['msg']), entry
       assert_equal 'test', get_string(fields['tag2']), entry
       assert_equal 5000, get_number(fields['data']), entry
-      assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
-      assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+      assert_equal K8S_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+      assert_equal K8S_NANOS, entry['timestamp']['nanos'], entry
       assert_equal 'WARNING', entry['severity'], entry
     end
   end
 
   def test_json_container_log_metadata_from_tag
     setup_gce_metadata_stubs
-    setup_container_metadata_stubs
+    setup_k8s_metadata_stubs
     setup_logging_stubs do
       d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
       d.emit(container_log_entry('{"msg": "test log entry 0", ' \
@@ -1025,8 +1026,8 @@ module BaseTest
       assert_equal 'test log entry 0', get_string(fields['msg']), entry
       assert_equal 'test', get_string(fields['tag2']), entry
       assert_equal 5000, get_number(fields['data']), entry
-      assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
-      assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+      assert_equal K8S_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+      assert_equal K8S_NANOS, entry['timestamp']['nanos'], entry
       assert_equal 'WARNING', entry['severity'], entry
     end
   end
@@ -1339,25 +1340,24 @@ module BaseTest
     end
   end
 
-  # Test k8s monitored resource including the fallback when Metadata Agent
-  # restarts.
-  def test_k8s_monitored_resource_fallback
+  # Test k8s_container monitored resource including the fallback when Metadata
+  # Agent restarts.
+  def test_k8s_container_monitored_resource_fallback
     [
-      # k8s_container.
       # When enable_metadata_agent is false.
       {
         config: APPLICATION_DEFAULT_CONFIG,
         setup_metadata_agent_stub: false,
         setup_k8s_stub: false,
         log_entry: k8s_container_log_entry(log_entry(0)),
-        expected_params: COMPUTE_PARAMS
+        expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
       },
       {
         config: APPLICATION_DEFAULT_CONFIG,
         setup_metadata_agent_stub: true,
         setup_k8s_stub: false,
         log_entry: k8s_container_log_entry(log_entry(0)),
-        expected_params: COMPUTE_PARAMS
+        expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
       },
       {
         config: APPLICATION_DEFAULT_CONFIG,
@@ -1379,7 +1379,7 @@ module BaseTest
         setup_metadata_agent_stub: false,
         setup_k8s_stub: false,
         log_entry: k8s_container_log_entry(log_entry(0)),
-        expected_params: COMPUTE_PARAMS
+        expected_params: K8S_CONTAINER_PARAMS_FROM_FALLBACK
       },
       {
         config: ENABLE_METADATA_AGENT_CONFIG,
@@ -1415,7 +1415,30 @@ module BaseTest
         setup_k8s_stub: true,
         log_entry: k8s_container_log_entry(log_entry(0)),
         expected_params: K8S_CONTAINER_PARAMS
-      },
+      }
+    ].each do |test_params|
+      new_stub_context do
+        setup_gce_metadata_stubs
+        setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
+        setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
+        setup_logging_stubs do
+          d = create_driver(test_params[:config], CONTAINER_TAG)
+          d.emit(test_params[:log_entry])
+          d.run
+        end
+        verify_log_entries(1, test_params[:expected_params],
+                           'jsonPayload') do |entry|
+          fields = get_fields(entry['jsonPayload'])
+          assert_equal 2, fields.size, entry
+          assert_equal 'test log entry 0', get_string(fields['log']), entry
+          assert_equal K8S_STREAM, get_string(fields['stream']), entry
+        end
+      end
+    end
+  end
+
+  def test_k8s_container_monitored_resource_invalid_local_resource_id
+    [
       # When local_resource_id is not present or does not match k8s regexes.
       {
         config: ENABLE_METADATA_AGENT_CONFIG,
@@ -1423,7 +1446,7 @@ module BaseTest
         setup_k8s_stub: true,
         log_entry: k8s_container_log_entry(
           log_entry(0)).reject { |k, _| k == LOCAL_RESOURCE_ID_KEY },
-        expected_params: COMPUTE_PARAMS
+        expected_params: CONTAINER_FROM_TAG_PARAMS
       },
       {
         config: ENABLE_METADATA_AGENT_CONFIG,
@@ -1432,11 +1455,29 @@ module BaseTest
         log_entry: k8s_container_log_entry(
           log_entry(0),
           local_resource_id: RANDOM_LOCAL_RESOURCE_ID),
-        # When 'kube-env' is present, "compute.googleapis.com/resource_name" is
-        # not added.
-        expected_params: COMPUTE_PARAMS
-      },
-      # Specific cases for k8s_node.
+        expected_params: CONTAINER_FROM_TAG_PARAMS
+      }
+    ].each do |test_params|
+      new_stub_context do
+        setup_gce_metadata_stubs
+        setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
+        setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
+        setup_logging_stubs do
+          d = create_driver(test_params[:config], CONTAINER_TAG)
+          d.emit(test_params[:log_entry])
+          d.run
+        end
+        verify_log_entries(1, test_params[:expected_params]) do |entry|
+          assert_equal 'test log entry 0', entry['textPayload'], entry
+        end
+      end
+    end
+  end
+
+  # Test k8s_node monitored resource including the fallback when Metadata Agent
+  # restarts.
+  def test_k8s_node_monitored_resource_fallback
+    [
       {
         config: APPLICATION_DEFAULT_CONFIG,
         setup_metadata_agent_stub: true,
@@ -1475,16 +1516,8 @@ module BaseTest
     ].each do |test_params|
       new_stub_context do
         setup_gce_metadata_stubs
-        if test_params[:setup_metadata_agent_stub]
-          setup_metadata_agent_stubs
-        else
-          setup_no_metadata_agent_stubs
-        end
-        if test_params[:setup_k8s_stub]
-          setup_k8s_metadata_stubs
-        else
-          setup_no_k8s_metadata_stubs
-        end
+        setup_metadata_agent_stubs(test_params[:setup_metadata_agent_stub])
+        setup_k8s_metadata_stubs(test_params[:setup_k8s_stub])
         setup_logging_stubs do
           d = create_driver(test_params[:config])
           d.emit(test_params[:log_entry])
@@ -1561,7 +1594,7 @@ module BaseTest
     [1, 2, 3, 5, 11, 50].each do |n|
       new_stub_context do
         setup_gce_metadata_stubs
-        setup_container_metadata_stubs
+        setup_k8s_metadata_stubs
         setup_metadata_agent_stubs
         setup_logging_stubs do
           d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
@@ -1573,7 +1606,7 @@ module BaseTest
         verify_log_entries(n, CONTAINER_FROM_APPLICATION_PARAMS)
         assert_requested_metadata_agent_stub(
           "#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
-          ".#{CONTAINER_POD_NAME}.#{CONTAINER_CONTAINER_NAME}")
+          ".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}")
       end
     end
   end
@@ -1653,44 +1686,34 @@ module BaseTest
                           MANAGED_VM_BACKEND_VERSION)
   end
 
-  def setup_container_metadata_stubs
-    stub_metadata_request(
-      'instance/attributes/',
-      "attribute1\nkube-env\nlast_attribute")
-    stub_metadata_request('instance/attributes/kube-env',
-                          "ENABLE_NODE_LOGGING: \"true\"\n"\
-                          'INSTANCE_PREFIX: '\
-                          "gke-#{CONTAINER_CLUSTER_NAME}-740fdafa\n"\
-                          'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
-  end
-
-  def setup_k8s_metadata_stubs
-    stub_metadata_request(
-      'instance/attributes/',
-      "attribute1\ncluster-name\ncluster-location\nlast_attribute")
-    stub_metadata_request('instance/attributes/cluster-location', K8S_LOCATION2)
-    stub_metadata_request('instance/attributes/cluster-name', K8S_CLUSTER_NAME)
-  end
-
-  def setup_no_k8s_metadata_stubs
-    ['cluster-location', 'cluster-name'].each do |metadata_name|
-      stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
-        .to_return(status: 404,
-                   body: 'The requested URL /computeMetadata/v1/instance/' \
-                         "attributes/#{metadata_name} was not found on this" \
-                         ' server.')
+  def setup_k8s_metadata_stubs(should_respond = true)
+    if should_respond
+      stub_metadata_request(
+        'instance/attributes/',
+        "attribute1\ncluster-location\ncluster-name\nlast_attribute")
+      stub_metadata_request('instance/attributes/cluster-location',
+                            K8S_LOCATION2)
+      stub_metadata_request('instance/attributes/cluster-name',
+                            K8S_CLUSTER_NAME)
+    else
+      ['cluster-location', 'cluster-name'].each do |metadata_name|
+        stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
+          .to_return(status: 404,
+                     body: 'The requested URL /computeMetadata/v1/instance/' \
+                           "attributes/#{metadata_name} was not found on this" \
+                           ' server.')
+      end
     end
   end
 
   def setup_cloudfunctions_metadata_stubs
     stub_metadata_request(
       'instance/attributes/',
-      "attribute1\nkube-env\ngcf_region\nlast_attribute")
-    stub_metadata_request('instance/attributes/kube-env',
-                          "ENABLE_NODE_LOGGING: \"true\"\n"\
-                          'INSTANCE_PREFIX: '\
-                          "gke-#{CLOUDFUNCTIONS_CLUSTER_NAME}-740fdafa\n"\
-                          'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
+      "attribute1\ncluster-location\ncluster-name\ngcf_region\nlast_attribute")
+    stub_metadata_request('instance/attributes/cluster-location',
+                          K8S_LOCATION2)
+    stub_metadata_request('instance/attributes/cluster-name',
+                          K8S_CLUSTER_NAME)
     stub_metadata_request('instance/attributes/gcf_region',
                           CLOUDFUNCTIONS_REGION)
   end
@@ -1725,19 +1748,20 @@ module BaseTest
     WebMock.reset!
   end
 
-  def setup_metadata_agent_stubs
-    MONITORED_RESOURCE_STUBS.each do |local_resource_id, resource|
-      stub_request(:get, metadata_request_url(local_resource_id))
-        .to_return(status: 200, body: resource)
+  def setup_metadata_agent_stubs(should_respond = true)
+    if should_respond
+      MONITORED_RESOURCE_STUBS.each do |local_resource_id, resource|
+        stub_request(:get, metadata_request_url(local_resource_id))
+          .to_return(status: 200, body: resource)
+      end
+      stub_request(:get, metadata_request_url(RANDOM_LOCAL_RESOURCE_ID))
+        .to_return(status: 404, body: '')
+    else
+      # Simulate an environment with no metadata agent endpoint present.
+      stub_request(:get,
+                   %r{#{DEFAULT_METADATA_AGENT_URL}\/monitoredResource/.*})
+        .to_raise(Errno::EHOSTUNREACH)
     end
-    stub_request(:get, metadata_request_url(RANDOM_LOCAL_RESOURCE_ID))
-      .to_return(status: 404, body: '')
-  end
-
-  def setup_no_metadata_agent_stubs
-    # Simulate an environment with no metadata agent endpoint present.
-    stub_request(:get, %r{#{DEFAULT_METADATA_AGENT_URL}\/monitoredResource/.*})
-      .to_raise(Errno::EHOSTUNREACH)
   end
 
   def assert_requested_metadata_agent_stub(local_resource_id)
@@ -1747,21 +1771,20 @@ module BaseTest
   # GKE Container.
 
   def container_tag_with_container_name(container_name)
-    "kubernetes.#{CONTAINER_POD_NAME}_#{CONTAINER_NAMESPACE_NAME}_" \
-      "#{container_name}"
+    "kubernetes.#{K8S_POD_NAME}_#{K8S_NAMESPACE_NAME}_#{container_name}"
   end
 
   def container_log_entry_with_metadata(
-      log, container_name = CONTAINER_CONTAINER_NAME)
+      log, container_name = K8S_CONTAINER_NAME)
     {
       log: log,
-      stream: CONTAINER_STREAM,
-      time: CONTAINER_TIMESTAMP,
+      stream: K8S_STREAM,
+      time: K8S_TIMESTAMP,
       kubernetes: {
         namespace_id: CONTAINER_NAMESPACE_ID,
-        namespace_name: CONTAINER_NAMESPACE_NAME,
+        namespace_name: K8S_NAMESPACE_NAME,
         pod_id: CONTAINER_POD_ID,
-        pod_name: CONTAINER_POD_NAME,
+        pod_name: K8S_POD_NAME,
         container_name: container_name,
         labels: {
           CONTAINER_LABEL_KEY => CONTAINER_LABEL_VALUE
@@ -1770,11 +1793,11 @@ module BaseTest
     }
   end
 
-  def container_log_entry(log, stream = CONTAINER_STREAM)
+  def container_log_entry(log, stream = K8S_STREAM)
     {
       log: log,
       stream: stream,
-      time: CONTAINER_TIMESTAMP
+      time: K8S_TIMESTAMP
     }
   end
 
@@ -1783,7 +1806,7 @@ module BaseTest
       log: log,
       LOCAL_RESOURCE_ID_KEY =>
         "#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
-        ".#{CONTAINER_POD_NAME}.#{CONTAINER_CONTAINER_NAME}"
+        ".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}"
     }
   end
 
@@ -1949,7 +1972,7 @@ module BaseTest
 
   def verify_container_logs(log_entry_factory, expected_params)
     setup_gce_metadata_stubs
-    setup_container_metadata_stubs
+    setup_k8s_metadata_stubs
     [1, 2, 3, 5, 11, 50].each do |n|
       @logs_sent = []
       setup_logging_stubs do
@@ -1959,9 +1982,8 @@ module BaseTest
       end
       verify_log_entries(n, expected_params) do |entry, i|
         verify_default_log_entry_text(entry['textPayload'], i, entry)
-        assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'],
-                     entry
-        assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
+        assert_equal K8S_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
+        assert_equal K8S_NANOS, entry['timestamp']['nanos'], entry
         assert_equal CONTAINER_SEVERITY, entry['severity'], entry
       end
     end

data/test/plugin/constants.rb

@@ -143,20 +143,11 @@ module Constants
     ".#{K8S_CONTAINER_NAME}".freeze
 
   # Container Engine / Kubernetes specific labels.
-  CONTAINER_CLUSTER_NAME = 'cluster-1'.freeze
   CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'.freeze
-  CONTAINER_NAMESPACE_NAME = 'kube-system'.freeze
   CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'.freeze
-  CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'.freeze
-  CONTAINER_CONTAINER_NAME = 'redis'.freeze
   CONTAINER_LABEL_KEY = 'component'.freeze
   CONTAINER_LABEL_VALUE = 'redis-component'.freeze
-  CONTAINER_STREAM = 'stdout'.freeze
   CONTAINER_SEVERITY = 'INFO'.freeze
-  # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
-  CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'.freeze
-  CONTAINER_SECONDS_EPOCH = 1_234_567_890
-  CONTAINER_NANOS = 987_654_321
   CONTAINER_LOCAL_RESOURCE_ID_PREFIX = 'gke_container'.freeze
 
   # Cloud Functions specific labels.
@@ -408,28 +399,27 @@ module Constants
 
   # GKE Container.
   CONTAINER_TAG =
-    "kubernetes.#{CONTAINER_POD_NAME}_#{CONTAINER_NAMESPACE_NAME}_" \
-    "#{CONTAINER_CONTAINER_NAME}".freeze
+    "kubernetes.#{K8S_POD_NAME}_#{K8S_NAMESPACE_NAME}_" \
+    "#{K8S_CONTAINER_NAME}".freeze
 
   CONTAINER_FROM_METADATA_PARAMS = {
     resource: {
       type: GKE_CONSTANTS[:resource_type],
       labels: {
-        'cluster_name' => CONTAINER_CLUSTER_NAME,
+        'cluster_name' => K8S_CLUSTER_NAME,
         'namespace_id' => CONTAINER_NAMESPACE_ID,
         'instance_id' => VM_ID,
         'pod_id' => CONTAINER_POD_ID,
-        'container_name' => CONTAINER_CONTAINER_NAME,
+        'container_name' => K8S_CONTAINER_NAME,
         'zone' => ZONE
       }
     },
-    log_name: CONTAINER_CONTAINER_NAME,
+    log_name: K8S_CONTAINER_NAME,
     project_id: PROJECT_ID,
     labels: {
-      "#{GKE_CONSTANTS[:service]}/namespace_name" =>
-        CONTAINER_NAMESPACE_NAME,
-      "#{GKE_CONSTANTS[:service]}/pod_name" => CONTAINER_POD_NAME,
-      "#{GKE_CONSTANTS[:service]}/stream" => CONTAINER_STREAM,
+      "#{GKE_CONSTANTS[:service]}/namespace_name" => K8S_NAMESPACE_NAME,
+      "#{GKE_CONSTANTS[:service]}/pod_name" => K8S_POD_NAME,
+      "#{GKE_CONSTANTS[:service]}/stream" => K8S_STREAM,
       "label/#{CONTAINER_LABEL_KEY}" => CONTAINER_LABEL_VALUE,
       "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
     }
@@ -441,21 +431,20 @@ module Constants
     resource: {
       type: GKE_CONSTANTS[:resource_type],
       labels: {
-        'cluster_name' => CONTAINER_CLUSTER_NAME,
-        'namespace_id' => CONTAINER_NAMESPACE_NAME,
+        'cluster_name' => K8S_CLUSTER_NAME,
+        'namespace_id' => K8S_NAMESPACE_NAME,
         'instance_id' => VM_ID,
-        'pod_id' => CONTAINER_POD_NAME,
-        'container_name' => CONTAINER_CONTAINER_NAME,
+        'pod_id' => K8S_POD_NAME,
+        'container_name' => K8S_CONTAINER_NAME,
         'zone' => ZONE
       }
     },
-    log_name: CONTAINER_CONTAINER_NAME,
+    log_name: K8S_CONTAINER_NAME,
     project_id: PROJECT_ID,
     labels: {
-      "#{GKE_CONSTANTS[:service]}/namespace_name" =>
-        CONTAINER_NAMESPACE_NAME,
-      "#{GKE_CONSTANTS[:service]}/pod_name" => CONTAINER_POD_NAME,
-      "#{GKE_CONSTANTS[:service]}/stream" => CONTAINER_STREAM,
+      "#{GKE_CONSTANTS[:service]}/namespace_name" => K8S_NAMESPACE_NAME,
+      "#{GKE_CONSTANTS[:service]}/pod_name" => K8S_POD_NAME,
+      "#{GKE_CONSTANTS[:service]}/stream" => K8S_STREAM,
       "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
     }
   }.freeze
@@ -464,11 +453,11 @@ module Constants
     resource: {
       type: GKE_CONSTANTS[:resource_type],
       labels: {
-        'cluster_name' => CONTAINER_CLUSTER_NAME,
+        'cluster_name' => K8S_CLUSTER_NAME,
         'namespace_id' => CONTAINER_NAMESPACE_ID,
         'instance_id' => VM_ID,
         'pod_id' => CONTAINER_POD_ID,
-        'container_name' => CONTAINER_CONTAINER_NAME,
+        'container_name' => K8S_CONTAINER_NAME,
         'zone' => ZONE
       }
     },
@@ -491,7 +480,6 @@ module Constants
         'location' => K8S_LOCATION
       }
     },
-    log_name: 'test',
     project_id: PROJECT_ID,
     labels: {}
   }.freeze
@@ -510,6 +498,10 @@ module Constants
       )
     )
   ).freeze
+  # Used in k8s fallback tests.
+  K8S_CONTAINER_PARAMS_FROM_FALLBACK = COMPUTE_PARAMS_NO_LOG_NAME.merge(
+    log_name: CONTAINER_TAG
+  ).freeze
 
   # K8s Node.
   K8S_NODE_PARAMS = {
@@ -785,12 +777,12 @@ module Constants
       }.to_json,
     # GKE container logs.
     "#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
-    ".#{CONTAINER_POD_NAME}.#{CONTAINER_CONTAINER_NAME}" =>
+    ".#{K8S_POD_NAME}.#{K8S_CONTAINER_NAME}" =>
       {
         'type' => GKE_CONSTANTS[:resource_type],
         'labels' => {
-          'cluster_name' => CONTAINER_CLUSTER_NAME,
-          'container_name' => CONTAINER_CONTAINER_NAME,
+          'cluster_name' => K8S_CLUSTER_NAME,
+          'container_name' => K8S_CONTAINER_NAME,
           'instance_id' => VM_ID,
           'namespace_id' => CONTAINER_NAMESPACE_ID,
           'pod_id' => CONTAINER_POD_ID,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.24
+  version: 0.6.25.pre.1
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-05 00:00:00.000000000 Z
+date: 2018-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -106,14 +106,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
@@ -231,6 +231,7 @@ extra_rdoc_files: []
 files:
 - CONTRIBUTING
 - Gemfile
+- Gemfile.lock
 - LICENSE
 - README.rdoc
 - Rakefile
@@ -265,9 +266,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.14