fluent-plugin-google-cloud 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 357b0cf2cba57aafc6011c911419b746cc7103b2
-  data.tar.gz: 91941e153f05ff927e8f83d342b651219075ebc8
+  metadata.gz: a0d7c7c98a69f3fa2f2c40ad70020260bf55e8be
+  data.tar.gz: 539deff09173b3b13fe9c226eaee60f8bd3695c7
 SHA512:
-  metadata.gz: 10cfc826b705a84b017571b34b759ee9013f416436e6fcd79c632b8e6daa7a016e87c1630a6d5e03a23b7627abb2e46bd32709957a7fdb7589c91af8b2b4f3bc
-  data.tar.gz: 915d378471e6b055f8eef48e73c7d27300d9fcadf420b2029efdf5a73e46eb83f674e3b631e1257986cae43e9e2603148d61707363ad53e4d3cdb81fe575f1ba
+  metadata.gz: 9bd48380bc63a1863ddc846beaf10d47053b3c17013e5d4044bc6474383d5958cd331ed9195092b8ffbf27e9b18c8f61e98f7e3eaff23ec2ed8b3fbfdb30ac11
+  data.tar.gz: 7c0e134bbfa2c2b78b33727be8739c2e548f169f7b7c76a451bedea257babc82bf062c89e0020daf9426d8b47a1cf72542670e24ef73cc94782e8d9ecec86263

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.6.2)
+    fluent-plugin-google-cloud (0.6.3)
       fluentd (~> 0.10)
       google-api-client (~> 0.9.0)
       google-cloud-logging (~> 0.23.2)
       googleapis-common-protos (~> 1.3)
       googleauth (~> 0.4)
-      grpc (~> 1.0)
+      grpc (~> 1.0, < 1.3)
       json (~> 1.8)
 
 GEM
@@ -18,13 +18,13 @@ GEM
     ast (2.3.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
-    cool.io (1.4.6)
+    cool.io (1.5.0)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     faraday (0.12.1)
       multipart-post (>= 1.2, < 3)
-    fluentd (0.14.15)
-      cool.io (~> 1.4.5)
+    fluentd (0.14.17)
+      cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       msgpack (>= 0.7.0, < 2.0.0)
       serverengine (>= 2.0.4, < 3.0.0)
@@ -57,7 +57,7 @@ GEM
       googleauth (~> 0.5.1)
       grpc (~> 1.0)
       rly (~> 0.2.3)
-    google-protobuf (3.3.0)
+    google-protobuf (3.3.0-universal-darwin)
     googleapis-common-protos (1.3.5)
       google-protobuf (~> 3.2)
       grpc (~> 1.0)
@@ -69,7 +69,7 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
-    grpc (1.2.5)
+    grpc (1.2.5-universal-darwin)
       google-protobuf (~> 3.1)
       googleauth (~> 0.5.1)
     hashdiff (0.3.4)
@@ -125,7 +125,7 @@ GEM
       multi_json (~> 1.10)
     stackdriver-core (0.21.0)
     strptime (0.1.9)
-    test-unit (3.2.3)
+    test-unit (3.2.4)
       power_assert
     thread_safe (0.3.6)
     tins (1.6.0)
@@ -152,4 +152,4 @@ DEPENDENCIES
   webmock (~> 1.17)
 
 BUNDLED WITH
-   1.14.6
+   1.15.0

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.2'
+  gem.version       = '0.6.3'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -24,7 +24,7 @@ eos
   gem.add_runtime_dependency 'google-api-client', '~> 0.9.0'
   gem.add_runtime_dependency 'google-cloud-logging', '~> 0.23.2'
   gem.add_runtime_dependency 'googleauth', '~> 0.4'
-  gem.add_runtime_dependency 'grpc', '~> 1.0'
+  gem.add_runtime_dependency 'grpc', '~> 1.0', '< 1.3'
   gem.add_runtime_dependency 'json', '~> 1.8'
 
   gem.add_development_dependency 'mocha', '~> 1.1'

data/lib/fluent/plugin/out_google_cloud.rb

@@ -73,7 +73,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
-    PLUGIN_VERSION = '0.6.2'
+    PLUGIN_VERSION = '0.6.3'
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
@@ -450,8 +450,9 @@ module Fluent
         # Do this here to avoid having to repeat it for each record.
         match_data = @compiled_kubernetes_tag_regexp.match(tag)
         if match_data
-          group_resource.labels['container_name'] =
-            match_data['container_name']
+          group_resource.labels['container_name'] = match_data['container_name']
+          group_resource.labels['namespace_id'] = match_data['namespace_name']
+          group_resource.labels['pod_id'] = match_data['pod_name']
           %w(namespace_name pod_name).each do |field|
             group_common_labels["#{CONTAINER_CONSTANTS[:service]}/#{field}"] =
               match_data[field]

data/test/plugin/base_test.rb

@@ -17,9 +17,11 @@ require 'helper'
 require 'mocha/test_unit'
 require 'webmock/test_unit'
 
+require_relative 'constants'
+
 # Unit tests for Google Cloud Logging plugin
 module BaseTest
-  include Fluent::GoogleCloudOutput::Constants
+  include Constants
 
   def setup
     Fluent::Test.setup
@@ -39,410 +41,6 @@ module BaseTest
     @logs_sent = []
   end
 
-  # generic attributes
-  HOSTNAME = Socket.gethostname
-
-  WRITE_LOG_ENTRIES_URI = 'https://logging.googleapis.com/v2beta1/entries:write'
-
-  # attributes used for the GCE metadata service
-  PROJECT_ID = 'test-project-id'
-  ZONE = 'us-central1-b'
-  FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
-  VM_ID = '9876543210'
-
-  # attributes used for custom (overridden) configs
-  CUSTOM_PROJECT_ID = 'test-custom-project-id'
-  CUSTOM_ZONE = 'us-custom-central1-b'
-  CUSTOM_FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
-  CUSTOM_VM_ID = 'C9876543210'
-  CUSTOM_HOSTNAME = 'custom.hostname.org'
-
-  # attributes used for the EC2 metadata service
-  EC2_PROJECT_ID = 'test-ec2-project-id'
-  EC2_ZONE = 'us-west-2b'
-  EC2_PREFIXED_ZONE = 'aws:' + EC2_ZONE
-  EC2_VM_ID = 'i-81c16767'
-  EC2_ACCOUNT_ID = '123456789012'
-
-  # The formatting here matches the format used on the VM.
-  EC2_IDENTITY_DOCUMENT = %({
-    "accountId" : "#{EC2_ACCOUNT_ID}",
-    "availabilityZone" : "#{EC2_ZONE}",
-    "instanceId" : "#{EC2_VM_ID}"
-  })
-
-  # Managed VMs specific labels
-  MANAGED_VM_BACKEND_NAME = 'default'
-  MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
-
-  # Container Engine / Kubernetes specific labels
-  CONTAINER_CLUSTER_NAME = 'cluster-1'
-  CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
-  CONTAINER_NAMESPACE_NAME = 'kube-system'
-  CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
-  CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
-  CONTAINER_CONTAINER_NAME = 'redis'
-  CONTAINER_LABEL_KEY = 'component'
-  CONTAINER_LABEL_VALUE = 'redis-component'
-  CONTAINER_STREAM = 'stdout'
-  CONTAINER_SEVERITY = 'INFO'
-  # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch
-  CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
-  CONTAINER_SECONDS_EPOCH = 1_234_567_890
-  CONTAINER_NANOS = 987_654_321
-
-  # Cloud Functions specific labels
-  CLOUDFUNCTIONS_FUNCTION_NAME = '$My_Function.Name-@1'
-  CLOUDFUNCTIONS_REGION = 'us-central1'
-  CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
-  CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
-  CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
-  CLOUDFUNCTIONS_POD_NAME = 'd.dc.myu.uc.functionp.pc.name-a.a1.987-c0l82'
-  CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
-
-  # Dataflow specific labels
-  DATAFLOW_REGION = 'us-central1'
-  DATAFLOW_JOB_NAME = 'job_name_1'
-  DATAFLOW_JOB_ID = 'job_id_1'
-  DATAFLOW_STEP_ID = 'step_1'
-  DATAFLOW_TAG = 'dataflow.googleapis.com/worker'
-
-  # ML specific labels
-  ML_REGION = 'us-central1'
-  ML_JOB_ID = 'job_name_1'
-  ML_TASK_NAME = 'task_name_1'
-  ML_TRIAL_ID = 'trial_id_1'
-  ML_LOG_AREA = 'log_area_1'
-  ML_TAG = 'master-replica-0'
-
-  # Parameters used for authentication
-  AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
-  FAKE_AUTH_TOKEN = 'abc123'
-
-  # Information about test credentials files.
-  # path: Path to the credentials file.
-  # project_id: ID of the project, which must correspond to the file contents.
-  IAM_CREDENTIALS = {
-    path: 'test/plugin/data/iam-credentials.json',
-    project_id: 'fluent-test-project'
-  }
-  LEGACY_CREDENTIALS = {
-    path: 'test/plugin/data/credentials.json',
-    project_id: '847859579879'
-  }
-  INVALID_CREDENTIALS = {
-    path: 'test/plugin/data/invalid_credentials.json',
-    project_id: ''
-  }
-
-  # Configuration files for various test scenarios
-  APPLICATION_DEFAULT_CONFIG = %(
-  )
-
-  # rubocop:disable Metrics/LineLength
-  PRIVATE_KEY_CONFIG = %(
-     auth_method private_key
-     private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
-     private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
-  )
-  # rubocop:enable Metrics/LineLength
-
-  REQUIRE_VALID_TAGS_CONFIG = %(
-    require_valid_tags true
-  )
-
-  NO_METADATA_SERVICE_CONFIG = %(
-    use_metadata_service false
-  )
-
-  NO_DETECT_SUBSERVICE_CONFIG = %(
-    detect_subservice false
-  )
-
-  CUSTOM_METADATA_CONFIG = %(
-    project_id #{CUSTOM_PROJECT_ID}
-    zone #{CUSTOM_ZONE}
-    vm_id #{CUSTOM_VM_ID}
-    vm_name #{CUSTOM_HOSTNAME}
-  )
-
-  CONFIG_MISSING_METADATA_PROJECT_ID = %(
-    zone #{CUSTOM_ZONE}
-    vm_id #{CUSTOM_VM_ID}
-  )
-  CONFIG_MISSING_METADATA_ZONE = %(
-    project_id #{CUSTOM_PROJECT_ID}
-    vm_id #{CUSTOM_VM_ID}
-  )
-  CONFIG_MISSING_METADATA_VM_ID = %(
-    project_id #{CUSTOM_PROJECT_ID}
-    zone #{CUSTOM_ZONE}
-  )
-  CONFIG_MISSING_METADATA_ALL = %(
-  )
-
-  CONFIG_EC2_PROJECT_ID = %(
-    project_id #{EC2_PROJECT_ID}
-  )
-
-  CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %(
-    project_id #{EC2_PROJECT_ID}
-    vm_id #{CUSTOM_VM_ID}
-  )
-
-  CONFIG_DATAFLOW = %(
-    subservice_name "#{DATAFLOW_CONSTANTS[:service]}"
-    labels {
-      "#{DATAFLOW_CONSTANTS[:service]}/region" : "#{DATAFLOW_REGION}",
-      "#{DATAFLOW_CONSTANTS[:service]}/job_name" : "#{DATAFLOW_JOB_NAME}",
-      "#{DATAFLOW_CONSTANTS[:service]}/job_id" : "#{DATAFLOW_JOB_ID}"
-    }
-    label_map { "step": "#{DATAFLOW_CONSTANTS[:service]}/step_id" }
-  )
-
-  CONFIG_ML = %(
-    subservice_name "#{ML_CONSTANTS[:service]}"
-    labels {
-      "#{ML_CONSTANTS[:service]}/job_id" : "#{ML_JOB_ID}",
-      "#{ML_CONSTANTS[:service]}/task_name" : "#{ML_TASK_NAME}",
-      "#{ML_CONSTANTS[:service]}/trial_id" : "#{ML_TRIAL_ID}"
-    }
-    label_map { "name": "#{ML_CONSTANTS[:service]}/job_id/log_area" }
-  )
-
-  # Service configurations for various services
-  COMPUTE_PARAMS = {
-    resource: {
-      type: COMPUTE_CONSTANTS[:resource_type],
-      labels: {
-        'instance_id' => VM_ID,
-        'zone' => ZONE
-      }
-    },
-    log_name: 'test',
-    project_id: PROJECT_ID,
-    labels: {
-      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
-    }
-  }
-
-  VMENGINE_PARAMS = {
-    resource: {
-      type: APPENGINE_CONSTANTS[:resource_type],
-      labels: {
-        'module_id' => MANAGED_VM_BACKEND_NAME,
-        'version_id' => MANAGED_VM_BACKEND_VERSION
-      }
-    },
-    log_name: "#{APPENGINE_CONSTANTS[:service]}%2Ftest",
-    project_id: PROJECT_ID,
-    labels: {
-      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
-      "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
-    }
-  }
-
-  CONTAINER_TAG = "kubernetes.#{CONTAINER_POD_NAME}_" \
-                  "#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
-
-  CONTAINER_FROM_METADATA_PARAMS = {
-    resource: {
-      type: CONTAINER_CONSTANTS[:resource_type],
-      labels: {
-        'cluster_name' => CONTAINER_CLUSTER_NAME,
-        'namespace_id' => CONTAINER_NAMESPACE_ID,
-        'instance_id' => VM_ID,
-        'pod_id' => CONTAINER_POD_ID,
-        'container_name' => CONTAINER_CONTAINER_NAME,
-        'zone' => ZONE
-      }
-    },
-    log_name: CONTAINER_CONTAINER_NAME,
-    project_id: PROJECT_ID,
-    labels: {
-      "#{CONTAINER_CONSTANTS[:service]}/namespace_name" =>
-        CONTAINER_NAMESPACE_NAME,
-      "#{CONTAINER_CONSTANTS[:service]}/pod_name" => CONTAINER_POD_NAME,
-      "#{CONTAINER_CONSTANTS[:service]}/stream" => CONTAINER_STREAM,
-      "label/#{CONTAINER_LABEL_KEY}" => CONTAINER_LABEL_VALUE,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
-    }
-  }
-
-  # Almost the same as from metadata, but missing namespace_id and pod_id.
-  CONTAINER_FROM_TAG_PARAMS = {
-    resource: {
-      type: CONTAINER_CONSTANTS[:resource_type],
-      labels: {
-        'cluster_name' => CONTAINER_CLUSTER_NAME,
-        'instance_id' => VM_ID,
-        'container_name' => CONTAINER_CONTAINER_NAME,
-        'zone' => ZONE
-      }
-    },
-    log_name: CONTAINER_CONTAINER_NAME,
-    project_id: PROJECT_ID,
-    labels: {
-      "#{CONTAINER_CONSTANTS[:service]}/namespace_name" =>
-        CONTAINER_NAMESPACE_NAME,
-      "#{CONTAINER_CONSTANTS[:service]}/pod_name" => CONTAINER_POD_NAME,
-      "#{CONTAINER_CONSTANTS[:service]}/stream" => CONTAINER_STREAM,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
-    }
-  }
-
-  CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
-                        "#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
-                        "#{CLOUDFUNCTIONS_CONTAINER_NAME}"
-
-  CLOUDFUNCTIONS_PARAMS = {
-    resource: {
-      type: CLOUDFUNCTIONS_CONSTANTS[:resource_type],
-      labels: {
-        'function_name' => CLOUDFUNCTIONS_FUNCTION_NAME,
-        'region' => CLOUDFUNCTIONS_REGION
-      }
-    },
-    log_name: 'cloud-functions',
-    project_id: PROJECT_ID,
-    labels: {
-      'execution_id' => CLOUDFUNCTIONS_EXECUTION_ID,
-      "#{CONTAINER_CONSTANTS[:service]}/instance_id" => VM_ID,
-      "#{CONTAINER_CONSTANTS[:service]}/cluster_name" =>
-        CLOUDFUNCTIONS_CLUSTER_NAME,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
-      "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
-    }
-  }
-
-  CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS = {
-    resource: {
-      type: CLOUDFUNCTIONS_CONSTANTS[:resource_type],
-      labels: {
-        'function_name' => CLOUDFUNCTIONS_FUNCTION_NAME,
-        'region' => CLOUDFUNCTIONS_REGION
-      }
-    },
-    log_name: 'cloud-functions',
-    project_id: PROJECT_ID,
-    labels: {
-      "#{CONTAINER_CONSTANTS[:service]}/instance_id" => VM_ID,
-      "#{CONTAINER_CONSTANTS[:service]}/cluster_name" =>
-        CLOUDFUNCTIONS_CLUSTER_NAME,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
-      "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
-    }
-  }
-
-  DATAFLOW_PARAMS = {
-    resource: {
-      type: DATAFLOW_CONSTANTS[:resource_type],
-      labels: {
-        'job_name' => DATAFLOW_JOB_NAME,
-        'job_id' => DATAFLOW_JOB_ID,
-        'step_id' => DATAFLOW_STEP_ID,
-        'region' => DATAFLOW_REGION
-      }
-    },
-    log_name: DATAFLOW_TAG,
-    project_id: PROJECT_ID,
-    labels: {
-      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
-      "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
-    }
-  }
-
-  ML_PARAMS = {
-    resource: {
-      type: ML_CONSTANTS[:resource_type],
-      labels: {
-        'job_id' => ML_JOB_ID,
-        'task_name' => ML_TASK_NAME
-      }
-    },
-    log_name: ML_TAG,
-    project_id: PROJECT_ID,
-    labels: {
-      "#{ML_CONSTANTS[:service]}/trial_id" => ML_TRIAL_ID,
-      "#{ML_CONSTANTS[:service]}/job_id/log_area" => ML_LOG_AREA,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
-      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
-      "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
-    }
-  }
-
-  CUSTOM_PARAMS = {
-    resource: {
-      type: COMPUTE_CONSTANTS[:resource_type],
-      labels: {
-        'instance_id' => CUSTOM_VM_ID,
-        'zone' => CUSTOM_ZONE
-      }
-    },
-    log_name: 'test',
-    project_id: CUSTOM_PROJECT_ID,
-    labels: {
-      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => CUSTOM_HOSTNAME
-    }
-  }
-
-  EC2_PARAMS = {
-    resource: {
-      type: EC2_CONSTANTS[:resource_type],
-      labels: {
-        'instance_id' => EC2_VM_ID,
-        'region' => EC2_PREFIXED_ZONE,
-        'aws_account' => EC2_ACCOUNT_ID
-      }
-    },
-    log_name: 'test',
-    project_id: EC2_PROJECT_ID,
-    labels: {
-      "#{EC2_CONSTANTS[:service]}/resource_name" => HOSTNAME
-    }
-  }
-
-  HTTP_REQUEST_MESSAGE = {
-    'requestMethod' => 'POST',
-    'requestUrl' => 'http://example/',
-    'requestSize' => 210,
-    'status' => 200,
-    'responseSize' => 65,
-    'userAgent' => 'USER AGENT 1.0',
-    'remoteIp' => '55.55.55.55',
-    'referer' => 'http://referer/',
-    'cacheHit' => true,
-    'cacheValidatedWithOriginServer' => true
-  }
-
-  # Tags and their sanitized and encoded version.
-  VALID_TAGS = {
-    'test' => 'test',
-    'germanß' => 'german%C3%9F',
-    'chinese中' => 'chinese%E4%B8%AD',
-    'specialCharacter/_-.' => 'specialCharacter%2F_-.',
-    'abc@&^$*' => 'abc%40%26%5E%24%2A',
-    '@&^$*' => '%40%26%5E%24%2A'
-  }
-  INVALID_TAGS = {
-    # Non-string tags.
-    123 => '123',
-    1.23 => '1.23',
-    [1, 2, 3] => '%5B1%2C%202%2C%203%5D',
-    { key: 'value' } => '%7B%22key%22%3D%3E%22value%22%7D',
-    # Non-utf8 string tags.
-    "nonutf8#{[0x92].pack('C*')}" => 'nonutf8%20',
-    "abc#{[0x92].pack('C*')}" => 'abc%20',
-    "#{[0x92].pack('C*')}" => '%20',
-    # Empty string tag.
-    '' => '_'
-  }
-  ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
-
   # Shared tests.
 
   def test_configure_service_account_application_default

data/test/plugin/constants.rb

@@ -0,0 +1,424 @@
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Constants used by unit tests for Google Cloud Logging plugin.
+module Constants
+  include Fluent::GoogleCloudOutput::Constants
+
+  # Generic attributes.
+  HOSTNAME = Socket.gethostname
+
+  # TODO(qingling128) Separate constants into different submodules.
+  # Attributes used for the GCE metadata service.
+  PROJECT_ID = 'test-project-id'
+  ZONE = 'us-central1-b'
+  FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
+  VM_ID = '9876543210'
+
+  # Attributes used for custom (overridden) configs.
+  CUSTOM_PROJECT_ID = 'test-custom-project-id'
+  CUSTOM_ZONE = 'us-custom-central1-b'
+  CUSTOM_FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
+  CUSTOM_VM_ID = 'C9876543210'
+  CUSTOM_HOSTNAME = 'custom.hostname.org'
+
+  # Attributes used for the EC2 metadata service.
+  EC2_PROJECT_ID = 'test-ec2-project-id'
+  EC2_ZONE = 'us-west-2b'
+  EC2_PREFIXED_ZONE = 'aws:' + EC2_ZONE
+  EC2_VM_ID = 'i-81c16767'
+  EC2_ACCOUNT_ID = '123456789012'
+
+  # The formatting here matches the format used on the VM.
+  EC2_IDENTITY_DOCUMENT = %({
+    "accountId" : "#{EC2_ACCOUNT_ID}",
+    "availabilityZone" : "#{EC2_ZONE}",
+    "instanceId" : "#{EC2_VM_ID}"
+  })
+
+  # Managed VMs specific labels.
+  MANAGED_VM_BACKEND_NAME = 'default'
+  MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
+
+  # Container Engine / Kubernetes specific labels.
+  CONTAINER_CLUSTER_NAME = 'cluster-1'
+  CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
+  CONTAINER_NAMESPACE_NAME = 'kube-system'
+  CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
+  CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
+  CONTAINER_CONTAINER_NAME = 'redis'
+  CONTAINER_LABEL_KEY = 'component'
+  CONTAINER_LABEL_VALUE = 'redis-component'
+  CONTAINER_STREAM = 'stdout'
+  CONTAINER_SEVERITY = 'INFO'
+  # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch.
+  CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
+  CONTAINER_SECONDS_EPOCH = 1_234_567_890
+  CONTAINER_NANOS = 987_654_321
+
+  # Cloud Functions specific labels.
+  CLOUDFUNCTIONS_FUNCTION_NAME = '$My_Function.Name-@1'
+  CLOUDFUNCTIONS_REGION = 'us-central1'
+  CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
+  CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
+  CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
+  CLOUDFUNCTIONS_POD_NAME = 'd.dc.myu.uc.functionp.pc.name-a.a1.987-c0l82'
+  CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
+
+  # Dataflow specific labels.
+  DATAFLOW_REGION = 'us-central1'
+  DATAFLOW_JOB_NAME = 'job_name_1'
+  DATAFLOW_JOB_ID = 'job_id_1'
+  DATAFLOW_STEP_ID = 'step_1'
+  DATAFLOW_TAG = 'dataflow.googleapis.com/worker'
+
+  # ML specific labels.
+  ML_REGION = 'us-central1'
+  ML_JOB_ID = 'job_name_1'
+  ML_TASK_NAME = 'task_name_1'
+  ML_TRIAL_ID = 'trial_id_1'
+  ML_LOG_AREA = 'log_area_1'
+  ML_TAG = 'master-replica-0'
+
+  # Parameters used for authentication.
+  AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
+  FAKE_AUTH_TOKEN = 'abc123'
+
+  # Information about test credentials files.
+  # path: Path to the credentials file.
+  # project_id: ID of the project, which must correspond to the file contents.
+  IAM_CREDENTIALS = {
+    path: 'test/plugin/data/iam-credentials.json',
+    project_id: 'fluent-test-project'
+  }
+  LEGACY_CREDENTIALS = {
+    path: 'test/plugin/data/credentials.json',
+    project_id: '847859579879'
+  }
+  INVALID_CREDENTIALS = {
+    path: 'test/plugin/data/invalid_credentials.json',
+    project_id: ''
+  }
+
+  # Configuration files for various test scenarios.
+  APPLICATION_DEFAULT_CONFIG = %(
+  )
+
+  # rubocop:disable Metrics/LineLength
+  PRIVATE_KEY_CONFIG = %(
+     auth_method private_key
+     private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
+     private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
+  )
+  # rubocop:enable Metrics/LineLength
+
+  REQUIRE_VALID_TAGS_CONFIG = %(
+    require_valid_tags true
+  )
+
+  NO_METADATA_SERVICE_CONFIG = %(
+    use_metadata_service false
+  )
+
+  NO_DETECT_SUBSERVICE_CONFIG = %(
+    detect_subservice false
+  )
+
+  CUSTOM_METADATA_CONFIG = %(
+    project_id #{CUSTOM_PROJECT_ID}
+    zone #{CUSTOM_ZONE}
+    vm_id #{CUSTOM_VM_ID}
+    vm_name #{CUSTOM_HOSTNAME}
+  )
+
+  CONFIG_MISSING_METADATA_PROJECT_ID = %(
+    zone #{CUSTOM_ZONE}
+    vm_id #{CUSTOM_VM_ID}
+  )
+  CONFIG_MISSING_METADATA_ZONE = %(
+    project_id #{CUSTOM_PROJECT_ID}
+    vm_id #{CUSTOM_VM_ID}
+  )
+  CONFIG_MISSING_METADATA_VM_ID = %(
+    project_id #{CUSTOM_PROJECT_ID}
+    zone #{CUSTOM_ZONE}
+  )
+  CONFIG_MISSING_METADATA_ALL = %(
+  )
+
+  CONFIG_EC2_PROJECT_ID = %(
+    project_id #{EC2_PROJECT_ID}
+  )
+
+  CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %(
+    project_id #{EC2_PROJECT_ID}
+    vm_id #{CUSTOM_VM_ID}
+  )
+
+  CONFIG_DATAFLOW = %(
+    subservice_name "#{DATAFLOW_CONSTANTS[:service]}"
+    labels {
+      "#{DATAFLOW_CONSTANTS[:service]}/region" : "#{DATAFLOW_REGION}",
+      "#{DATAFLOW_CONSTANTS[:service]}/job_name" : "#{DATAFLOW_JOB_NAME}",
+      "#{DATAFLOW_CONSTANTS[:service]}/job_id" : "#{DATAFLOW_JOB_ID}"
+    }
+    label_map { "step": "#{DATAFLOW_CONSTANTS[:service]}/step_id" }
+  )
+
+  CONFIG_ML = %(
+    subservice_name "#{ML_CONSTANTS[:service]}"
+    labels {
+      "#{ML_CONSTANTS[:service]}/job_id" : "#{ML_JOB_ID}",
+      "#{ML_CONSTANTS[:service]}/task_name" : "#{ML_TASK_NAME}",
+      "#{ML_CONSTANTS[:service]}/trial_id" : "#{ML_TRIAL_ID}"
+    }
+    label_map { "name": "#{ML_CONSTANTS[:service]}/job_id/log_area" }
+  )
+
+  # Service configurations for various services.
+  COMPUTE_PARAMS = {
+    resource: {
+      type: COMPUTE_CONSTANTS[:resource_type],
+      labels: {
+        'instance_id' => VM_ID,
+        'zone' => ZONE
+      }
+    },
+    log_name: 'test',
+    project_id: PROJECT_ID,
+    labels: {
+      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
+    }
+  }
+
+  VMENGINE_PARAMS = {
+    resource: {
+      type: APPENGINE_CONSTANTS[:resource_type],
+      labels: {
+        'module_id' => MANAGED_VM_BACKEND_NAME,
+        'version_id' => MANAGED_VM_BACKEND_VERSION
+      }
+    },
+    log_name: "#{APPENGINE_CONSTANTS[:service]}%2Ftest",
+    project_id: PROJECT_ID,
+    labels: {
+      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
+      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
+      "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
+    }
+  }
+
+  CONTAINER_TAG = "kubernetes.#{CONTAINER_POD_NAME}_" \
+                  "#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
+
+  CONTAINER_FROM_METADATA_PARAMS = {
+    resource: {
+      type: CONTAINER_CONSTANTS[:resource_type],
+      labels: {
+        'cluster_name' => CONTAINER_CLUSTER_NAME,
+        'namespace_id' => CONTAINER_NAMESPACE_ID,
+        'instance_id' => VM_ID,
+        'pod_id' => CONTAINER_POD_ID,
+        'container_name' => CONTAINER_CONTAINER_NAME,
+        'zone' => ZONE
+      }
+    },
+    log_name: CONTAINER_CONTAINER_NAME,
+    project_id: PROJECT_ID,
+    labels: {
+      "#{CONTAINER_CONSTANTS[:service]}/namespace_name" =>
+        CONTAINER_NAMESPACE_NAME,
+      "#{CONTAINER_CONSTANTS[:service]}/pod_name" => CONTAINER_POD_NAME,
+      "#{CONTAINER_CONSTANTS[:service]}/stream" => CONTAINER_STREAM,
+      "label/#{CONTAINER_LABEL_KEY}" => CONTAINER_LABEL_VALUE,
+      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
+    }
+  }
+
+  # Almost the same as from metadata, but namespace_id and pod_id come from
+  # namespace and pod names.
+  CONTAINER_FROM_TAG_PARAMS = {
+    resource: {
+      type: CONTAINER_CONSTANTS[:resource_type],
+      labels: {
+        'cluster_name' => CONTAINER_CLUSTER_NAME,
+        'namespace_id' => CONTAINER_NAMESPACE_NAME,
+        'instance_id' => VM_ID,
+        'pod_id' => CONTAINER_POD_NAME,
+        'container_name' => CONTAINER_CONTAINER_NAME,
+        'zone' => ZONE
+      }
+    },
+    log_name: CONTAINER_CONTAINER_NAME,
+    project_id: PROJECT_ID,
+    labels: {
+      "#{CONTAINER_CONSTANTS[:service]}/namespace_name" =>
+        CONTAINER_NAMESPACE_NAME,
+      "#{CONTAINER_CONSTANTS[:service]}/pod_name" => CONTAINER_POD_NAME,
+      "#{CONTAINER_CONSTANTS[:service]}/stream" => CONTAINER_STREAM,
+      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
+    }
+  }
+
+  CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
+                        "#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
+                        "#{CLOUDFUNCTIONS_CONTAINER_NAME}"
+
+  CLOUDFUNCTIONS_PARAMS = {
+    resource: {
+      type: CLOUDFUNCTIONS_CONSTANTS[:resource_type],
+      labels: {
+        'function_name' => CLOUDFUNCTIONS_FUNCTION_NAME,
+        'region' => CLOUDFUNCTIONS_REGION
+      }
+    },
+    log_name: 'cloud-functions',
+    project_id: PROJECT_ID,
+    labels: {
+      'execution_id' => CLOUDFUNCTIONS_EXECUTION_ID,
+      "#{CONTAINER_CONSTANTS[:service]}/instance_id" => VM_ID,
+      "#{CONTAINER_CONSTANTS[:service]}/cluster_name" =>
+        CLOUDFUNCTIONS_CLUSTER_NAME,
+      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
+      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
+      "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
+    }
+  }
+
+  CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS = {
+    resource: {
+      type: CLOUDFUNCTIONS_CONSTANTS[:resource_type],
+      labels: {
+        'function_name' => CLOUDFUNCTIONS_FUNCTION_NAME,
+        'region' => CLOUDFUNCTIONS_REGION
+      }
+    },
+    log_name: 'cloud-functions',
+    project_id: PROJECT_ID,
+    labels: {
+      "#{CONTAINER_CONSTANTS[:service]}/instance_id" => VM_ID,
+      "#{CONTAINER_CONSTANTS[:service]}/cluster_name" =>
+        CLOUDFUNCTIONS_CLUSTER_NAME,
+      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
+      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
+      "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
+    }
+  }
+
+  DATAFLOW_PARAMS = {
+    resource: {
+      type: DATAFLOW_CONSTANTS[:resource_type],
+      labels: {
+        'job_name' => DATAFLOW_JOB_NAME,
+        'job_id' => DATAFLOW_JOB_ID,
+        'step_id' => DATAFLOW_STEP_ID,
+        'region' => DATAFLOW_REGION
+      }
+    },
+    log_name: DATAFLOW_TAG,
+    project_id: PROJECT_ID,
+    labels: {
+      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
+      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
+      "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
+    }
+  }
+
+  ML_PARAMS = {
+    resource: {
+      type: ML_CONSTANTS[:resource_type],
+      labels: {
+        'job_id' => ML_JOB_ID,
+        'task_name' => ML_TASK_NAME
+      }
+    },
+    log_name: ML_TAG,
+    project_id: PROJECT_ID,
+    labels: {
+      "#{ML_CONSTANTS[:service]}/trial_id" => ML_TRIAL_ID,
+      "#{ML_CONSTANTS[:service]}/job_id/log_area" => ML_LOG_AREA,
+      "#{COMPUTE_CONSTANTS[:service]}/resource_id" => VM_ID,
+      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME,
+      "#{COMPUTE_CONSTANTS[:service]}/zone" => ZONE
+    }
+  }
+
+  CUSTOM_PARAMS = {
+    resource: {
+      type: COMPUTE_CONSTANTS[:resource_type],
+      labels: {
+        'instance_id' => CUSTOM_VM_ID,
+        'zone' => CUSTOM_ZONE
+      }
+    },
+    log_name: 'test',
+    project_id: CUSTOM_PROJECT_ID,
+    labels: {
+      "#{COMPUTE_CONSTANTS[:service]}/resource_name" => CUSTOM_HOSTNAME
+    }
+  }
+
+  EC2_PARAMS = {
+    resource: {
+      type: EC2_CONSTANTS[:resource_type],
+      labels: {
+        'instance_id' => EC2_VM_ID,
+        'region' => EC2_PREFIXED_ZONE,
+        'aws_account' => EC2_ACCOUNT_ID
+      }
+    },
+    log_name: 'test',
+    project_id: EC2_PROJECT_ID,
+    labels: {
+      "#{EC2_CONSTANTS[:service]}/resource_name" => HOSTNAME
+    }
+  }
+
+  HTTP_REQUEST_MESSAGE = {
+    'requestMethod' => 'POST',
+    'requestUrl' => 'http://example/',
+    'requestSize' => 210,
+    'status' => 200,
+    'responseSize' => 65,
+    'userAgent' => 'USER AGENT 1.0',
+    'remoteIp' => '55.55.55.55',
+    'referer' => 'http://referer/',
+    'cacheHit' => true,
+    'cacheValidatedWithOriginServer' => true
+  }
+
+  # Tags and their sanitized and encoded version.
+  VALID_TAGS = {
+    'test' => 'test',
+    'germanß' => 'german%C3%9F',
+    'chinese中' => 'chinese%E4%B8%AD',
+    'specialCharacter/_-.' => 'specialCharacter%2F_-.',
+    'abc@&^$*' => 'abc%40%26%5E%24%2A',
+    '@&^$*' => '%40%26%5E%24%2A'
+  }
+  INVALID_TAGS = {
+    # Non-string tags.
+    123 => '123',
+    1.23 => '1.23',
+    [1, 2, 3] => '%5B1%2C%202%2C%203%5D',
+    { key: 'value' } => '%7B%22key%22%3D%3E%22value%22%7D',
+    # Non-utf8 string tags.
+    "nonutf8#{[0x92].pack('C*')}" => 'nonutf8%20',
+    "abc#{[0x92].pack('C*')}" => 'abc%20',
+    "#{[0x92].pack('C*')}" => '%20',
+    # Empty string tag.
+    '' => '_'
+  }
+  ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
+end

data/test/plugin/test_out_google_cloud.rb

@@ -197,6 +197,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
 
   private
 
+  WRITE_LOG_ENTRIES_URI = 'https://logging.googleapis.com/v2beta1/entries:write'
+
   def rename_key(hash, old_key, new_key)
     hash.merge(new_key => hash[old_key]).reject { |k, _| k == old_key }
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-09 00:00:00.000000000 Z
+date: 2017-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -88,6 +88,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -95,6 +98,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -196,10 +202,12 @@ files:
 - LICENSE
 - README.rdoc
 - Rakefile
+- fluent-plugin-google-cloud-0.6.2.gem
 - fluent-plugin-google-cloud.gemspec
 - lib/fluent/plugin/out_google_cloud.rb
 - test/helper.rb
 - test/plugin/base_test.rb
+- test/plugin/constants.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
 - test/plugin/data/credentials.json
 - test/plugin/data/iam-credentials.json
@@ -226,13 +234,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: fluentd output plugin for the Stackdriver Logging API
 test_files:
 - test/helper.rb
 - test/plugin/base_test.rb
+- test/plugin/constants.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
 - test/plugin/data/credentials.json
 - test/plugin/data/iam-credentials.json