fluent-plugin-google-cloud 0.6.15 → 0.6.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5a893e6f22835b55a791d1c530286733d22d49a0
-  data.tar.gz: ecd9cf870e72ff2c7b2961cfc5f135df8911c941
+  metadata.gz: dc441330446e0c25bc08501530b66b7d6564dc37
+  data.tar.gz: 55a4592b58913b49b23cab5ddfc469a97afd5c4f
 SHA512:
-  metadata.gz: 9156624b572bbe99385777b01a6f331920f19387aa2631dedbf14e9b2c1395519932f4ae7bcfde2d8098403e302dbebe0b99ee135fc5dcd30bea67af198a3342
-  data.tar.gz: dc2d3c73e3917020de5da358f5bf731068b1b9bc6e7f20e50130e0e2d6ccb2fe506b561ab17f27581ed72032519aec2d5f1223e2be69907f401e45db2525546e
+  metadata.gz: 55d494d4e51e821a800b06a3cf936fdbb0fa01305d4e04531f5c0f34001188be0da9b90ea8b84111a6d5f77abd9745f91cf3ec4465939c7af5d2ff11edd55b9f
+  data.tar.gz: 55566646fd0f2656df5fc9d8373d17fc6ddb364b527b74ae6a1be24173e7d0aaec141ea8e90c2bf0a4f7c66d7f471e8a85b9ecc4ef65d1985d164fb86b51126f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.6.15)
+    fluent-plugin-google-cloud (0.6.16)
       fluentd (~> 0.10)
       google-api-client (~> 0.17)
       google-cloud-logging (~> 1.3, >= 1.3.2)
@@ -34,21 +34,21 @@ GEM
       tzinfo (~> 1.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
-    google-api-client (0.19.5)
+    google-api-client (0.19.8)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (>= 0.5, < 0.7.0)
       httpclient (>= 2.8.1, < 3.0)
       mime-types (~> 3.0)
       representable (~> 3.0)
       retriable (>= 2.0, < 4.0)
-    google-cloud-core (1.1.0)
+    google-cloud-core (1.2.0)
       google-cloud-env (~> 1.0)
     google-cloud-env (1.0.1)
       faraday (~> 0.11)
-    google-cloud-logging (1.4.0)
-      google-cloud-core (~> 1.1)
+    google-cloud-logging (1.5.0)
+      google-cloud-core (~> 1.2)
       google-gax (~> 1.0)
-      stackdriver-core (~> 1.2)
+      stackdriver-core (~> 1.3)
     google-gax (1.0.1)
       google-protobuf (~> 3.2)
       googleapis-common-protos (>= 1.3.5, < 2.0)
@@ -70,7 +70,7 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
-    grpc (1.8.3)
+    grpc (1.9.1)
       google-protobuf (~> 3.1)
       googleapis-common-protos-types (~> 1.0.0)
       googleauth (>= 0.5.1, < 0.7)
@@ -94,13 +94,13 @@ GEM
     multi_json (1.13.1)
     multipart-post (2.0.0)
     os (0.9.6)
-    parser (2.4.0.2)
-      ast (~> 2.3)
+    parser (2.5.0.2)
+      ast (~> 2.4.0)
     power_assert (1.1.1)
     powerpack (0.1.1)
     prometheus-client (0.7.1)
       quantile (~> 0.2.0)
-    public_suffix (3.0.1)
+    public_suffix (3.0.2)
     quantile (0.2.0)
     rainbow (2.2.2)
       rake
@@ -128,12 +128,13 @@ GEM
       faraday (~> 0.9)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
-    stackdriver-core (1.2.0)
+    stackdriver-core (1.3.0)
+      google-cloud-core (~> 1.2)
     strptime (0.1.9)
     test-unit (3.2.7)
       power_assert
     thread_safe (0.3.6)
-    tzinfo (1.2.4)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
     tzinfo-data (1.2018.3)
       tzinfo (>= 1.0.0)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.15'
+  gem.version       = '0.6.16'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/lib/fluent/plugin/out_google_cloud.rb

@@ -124,6 +124,7 @@ module Fluent
 
       DEFAULT_METADATA_AGENT_URL =
         'http://local-metadata-agent.stackdriver.com:8000'.freeze
+      METADATA_AGENT_URL_ENV_VAR = 'STACKDRIVER_METADATA_AGENT_URL'.freeze
     end
 
     # Internal constants.
@@ -198,7 +199,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
-    PLUGIN_VERSION = '0.6.15'.freeze
+    PLUGIN_VERSION = '0.6.16'.freeze
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
@@ -350,8 +351,13 @@ module Fluent
 
     # Whether to call metadata agent to retrieve monitored resource.
     config_param :enable_metadata_agent, :bool, :default => false
-    config_param :metadata_agent_url, :string,
-                 :default => DEFAULT_METADATA_AGENT_URL
+
+    # The URL of the Metadata Agent.
+    # If this option is set, its value is used to contact the Metadata Agent.
+    # Otherwise, the value of the STACKDRIVER_METADATA_AGENT_URL environment
+    # variable is used. If that is also unset, this defaults to
+    # 'http://local-metadata-agent.stackdriver.com:8000'.
+    config_param :metadata_agent_url, :string, :default => nil
 
     # Whether to split log entries with different log tags into different
     # requests when talking to Stackdriver Logging API.
@@ -387,6 +393,20 @@ module Fluent
                   ' ignored.'
       end
 
+      # 1. If @metadata_agent_url is customized (aka not nil), use that.
+      # 2. Otherwise check the presence of the environment variable
+      #    STACKDRIVER_METADATA_AGENT_URL and use that if set.
+      # 3. Fall back to the default if neither is set.
+      if @enable_metadata_agent
+        # Convert to string to capture empty string.
+        @metadata_agent_url ||=
+          if ENV[METADATA_AGENT_URL_ENV_VAR].to_s.empty?
+            DEFAULT_METADATA_AGENT_URL
+          else
+            ENV[METADATA_AGENT_URL_ENV_VAR]
+          end
+      end
+
       # If monitoring is enabled, register metrics in the default registry
       # and store metric objects for future use.
       if @enable_monitoring
@@ -1181,11 +1201,9 @@ module Fluent
       # inferred from the tag) if failed to get a monitored resource from
       # Metadata Agent with this key.
       #
-      # Docker container:
-      #   "container.<container_id>"
-      #   "containerName.<container_name>"
-      # GKE container:
-      #   "gke_containerName.<namespace_id>.<pod_name>.<container_name>"
+      # Examples:
+      # "container.<container_id>" // Docker container.
+      # "k8s_pod.<namespace_name>.<pod_name>" // GKE pod.
       if @enable_metadata_agent && local_resource_id
         @log.debug 'Calling metadata agent with local_resource_id: ' \
                   "#{local_resource_id}."
@@ -1384,7 +1402,7 @@ module Fluent
         return parsed_hash
       end
     rescue StandardError => e
-      @log.error 'Error calling Metadata Agent.', error: e
+      @log.error "Error calling Metadata Agent at #{url}.", error: e
     end
 
     # TODO: This functionality should eventually be available in another

data/test/plugin/base_test.rb

@@ -116,6 +116,30 @@ module BaseTest
     end
   end
 
+  def test_metadata_agent_url_customization
+    [
+      # If @metadata_agent_url is set, use that even if the environment
+      # variable is set.
+      [CUSTOM_METADATA_AGENT_URL_CONFIG, true, CUSTOM_METADATA_AGENT_URL],
+      # If @metadata_agent_url is set and the environment variable is
+      # not set, use @metadata_agent_url.
+      [CUSTOM_METADATA_AGENT_URL_CONFIG, false, CUSTOM_METADATA_AGENT_URL],
+      # If @metadata_agent_url is not set and the environment variable is set,
+      # use the env.
+      [APPLICATION_DEFAULT_CONFIG, true, METADATA_AGENT_URL_FROM_ENV],
+      # If @metadata_agent_url is not set and the environment variable is
+      # not set, fall back to the default.
+      [APPLICATION_DEFAULT_CONFIG, false, DEFAULT_METADATA_AGENT_URL]
+    ].each do |(config, url_from_env, expected_url)|
+      ENV[METADATA_AGENT_URL_ENV_VAR] = METADATA_AGENT_URL_FROM_ENV if
+        url_from_env
+      setup_gce_metadata_stubs
+      d = create_driver(ENABLE_METADATA_AGENT_CONFIG + config)
+      assert_equal expected_url, d.instance.metadata_agent_url
+      ENV.delete(METADATA_AGENT_URL_ENV_VAR)
+    end
+  end
+
   def test_metadata_loading
     setup_gce_metadata_stubs
     d = create_driver
@@ -1260,7 +1284,7 @@ module BaseTest
 
   # Test logs from applications running in Docker containers. These logs have
   # the label "logging.googleapis.com/local_resource_id" set in the format of
-  # "containerName.<container_name>".
+  # "container.<container_name>".
   def test_docker_container_application_logs
     new_stub_context do
       setup_gce_metadata_stubs
@@ -1274,7 +1298,7 @@ module BaseTest
       end
       verify_log_entries(1, DOCKER_CONTAINER_PARAMS_NO_STREAM)
       assert_requested_metadata_agent_stub(
-        "containerName.#{DOCKER_CONTAINER_NAME}")
+        "#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_NAME}")
     end
   end
 
@@ -1298,7 +1322,7 @@ module BaseTest
                      entry['timestamp']['nanos'], entry
       end
       assert_requested_metadata_agent_stub(
-        "containerName.#{DOCKER_CONTAINER_NAME}")
+        "#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_NAME}")
     end
   end
 
@@ -1333,7 +1357,7 @@ module BaseTest
 
   # Test GKE container logs. These logs have the label
   # "logging.googleapis.com/local_resource_id" set in the format of
-  # "gke_containerName.<namespace_id>.<pod_name>.<container_name>".
+  # "gke_container.<namespace_id>.<pod_name>.<container_name>".
   def test_gke_container_logs
     [1, 2, 3, 5, 11, 50].each do |n|
       new_stub_context do
@@ -1349,8 +1373,8 @@ module BaseTest
         end
         verify_log_entries(n, CONTAINER_FROM_APPLICATION_PARAMS)
         assert_requested_metadata_agent_stub(
-          "gke_containerName.#{CONTAINER_NAMESPACE_ID}.#{CONTAINER_POD_NAME}." \
-          "#{CONTAINER_CONTAINER_NAME}")
+          "#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
+          ".#{CONTAINER_POD_NAME}.#{CONTAINER_CONTAINER_NAME}")
       end
     end
   end
@@ -1533,7 +1557,7 @@ module BaseTest
     {
       log: log,
       LOCAL_RESOURCE_ID_KEY =>
-        "gke_containerName.#{CONTAINER_NAMESPACE_ID}" \
+        "#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
         ".#{CONTAINER_POD_NAME}.#{CONTAINER_CONTAINER_NAME}"
     }
   end
@@ -1559,7 +1583,8 @@ module BaseTest
     {
       log: log,
       time: DOCKER_CONTAINER_TIMESTAMP,
-      LOCAL_RESOURCE_ID_KEY => "containerName.#{DOCKER_CONTAINER_NAME}"
+      LOCAL_RESOURCE_ID_KEY => "#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}." \
+                               "#{DOCKER_CONTAINER_NAME}"
     }
   end
 

data/test/plugin/constants.rb

@@ -21,6 +21,8 @@ module Constants
   # Generic attributes.
   HOSTNAME = Socket.gethostname
   CUSTOM_LOGGING_API_URL = 'http://localhost:52000'.freeze
+  CUSTOM_METADATA_AGENT_URL = 'http://localhost:12345'.freeze
+  METADATA_AGENT_URL_FROM_ENV = 'http://localhost:54321'.freeze
 
   # TODO(qingling128) Separate constants into different submodules.
   # Attributes used for the GCE metadata service.
@@ -71,6 +73,7 @@ module Constants
   DOCKER_CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'.freeze
   DOCKER_CONTAINER_SECONDS_EPOCH = 1_234_567_890
   DOCKER_CONTAINER_NANOS = 987_654_321
+  DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX = 'container'.freeze
 
   # Container Engine / Kubernetes specific labels.
   CONTAINER_CLUSTER_NAME = 'cluster-1'.freeze
@@ -87,6 +90,7 @@ module Constants
   CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'.freeze
   CONTAINER_SECONDS_EPOCH = 1_234_567_890
   CONTAINER_NANOS = 987_654_321
+  CONTAINER_LOCAL_RESOURCE_ID_PREFIX = 'gke_container'.freeze
 
   # Cloud Functions specific labels.
   CLOUDFUNCTIONS_FUNCTION_NAME = '$My_Function.Name-@1'.freeze
@@ -146,6 +150,10 @@ module Constants
     logging_api_url #{CUSTOM_LOGGING_API_URL}
   ).freeze
 
+  CUSTOM_METADATA_AGENT_URL_CONFIG = %(
+    metadata_agent_url #{CUSTOM_METADATA_AGENT_URL}
+  ).freeze
+
   DETECT_JSON_CONFIG = %(
     detect_json true
   ).freeze
@@ -605,7 +613,7 @@ module Constants
   # Map from the local_resource_id to the retrieved monitored resource.
   MONITORED_RESOURCE_STUBS = {
     # Docker container stderr / stdout logs.
-    "container.#{DOCKER_CONTAINER_ID}" =>
+    "#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_ID}" =>
       {
         'type' => DOCKER_CONSTANTS[:resource_type],
         'labels' => {
@@ -614,7 +622,7 @@ module Constants
         }
       }.to_json,
     # Docker container application logs.
-    "containerName.#{DOCKER_CONTAINER_NAME}" =>
+    "#{DOCKER_CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{DOCKER_CONTAINER_NAME}" =>
       {
         'type' => DOCKER_CONSTANTS[:resource_type],
         'labels' => {
@@ -623,8 +631,8 @@ module Constants
         }
       }.to_json,
     # GKE container logs.
-    "gke_containerName.#{CONTAINER_NAMESPACE_ID}.#{CONTAINER_POD_NAME}." \
-    "#{CONTAINER_CONTAINER_NAME}" =>
+    "#{CONTAINER_LOCAL_RESOURCE_ID_PREFIX}.#{CONTAINER_NAMESPACE_ID}" \
+    ".#{CONTAINER_POD_NAME}.#{CONTAINER_CONTAINER_NAME}" =>
       {
         'type' => GKE_CONSTANTS[:resource_type],
         'labels' => {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.15
+  version: 0.6.16
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-31 00:00:00.000000000 Z
+date: 2018-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -249,7 +249,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: fluentd output plugin for the Stackdriver Logging API