fluent-plugin-google-cloud 0.5.5.pre.1 → 0.5.5.pre.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 04347e11378a890ae709eb943fae378e4c653948
-  data.tar.gz: 48f4f8d264c03eb6d4fe6d16cc19f29bbf64b4fb
+  metadata.gz: 08888a8334b99ea4dfcf75f9cca5e4bd9c3adfb2
+  data.tar.gz: 3ad4733b6f10307039d813527044a3450443fe1a
 SHA512:
-  metadata.gz: 9a1baec70a6600ef0b3f086ec1b21ccce2159252da2ff820eaad55a83a8a12eafed606b9336fa43fbaa6ffdc9e5061738f3d74327c4d59e3a79e8267457e3d81
-  data.tar.gz: d50f6e7f48c8117b6183d8adb91c5b329f9ec58b48c023f4665412f7c293dcd6be50ce7c8bcd3763c17cfb108d5027b131f82270830e1920a34f0c16f660d0af
+  metadata.gz: adf57d8a4cf841974cf4992587134a0b335e86b2ca473bda6dc6edbf8cbc9f254e9a0699076d2a9e74f10969eadd07dce054e76bd5fd50fe6c62b2dce5f3c7d8
+  data.tar.gz: dea62f362060c090a560a5da926f90054c89874759c28cbe0b3b7bf523672184d5eaac738d98be55a0fa25e55d2e89251dd130bb5234ab76e84698a4bbd9335b

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.5.5.pre.1)
+    fluent-plugin-google-cloud (0.5.5.pre.2)
       fluentd (~> 0.10)
       google-api-client (> 0.9)
       googleapis-common-protos (~> 1.3)
@@ -32,7 +32,7 @@ GEM
       tzinfo (~> 1.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
-    google-api-client (0.9.26)
+    google-api-client (0.9.28)
       addressable (~> 2.3)
       googleauth (~> 0.5)
       httpclient (~> 2.7)
@@ -41,7 +41,7 @@ GEM
       mime-types (>= 1.6)
       representable (~> 2.3.0)
       retriable (~> 2.0)
-    google-protobuf (3.0.2)
+    google-protobuf (3.0.2-x86_64-linux)
     googleapis-common-protos (1.3.4)
       google-protobuf (~> 3.0)
       grpc (~> 1.0)
@@ -53,7 +53,7 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
-    grpc (1.0.1)
+    grpc (1.0.1-x86_64-linux)
       google-protobuf (~> 3.0.2)
       googleauth (~> 0.5.1)
     hashdiff (0.3.2)
@@ -77,7 +77,7 @@ GEM
     multi_json (1.12.1)
     multipart-post (2.0.0)
     os (0.9.6)
-    parser (2.3.3.1)
+    parser (2.4.0.0)
       ast (~> 2.2)
     power_assert (1.0.1)
     powerpack (0.1.1)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.5.5.pre.1'
+  gem.version       = '0.5.5.pre.2'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/lib/fluent/plugin/out_google_cloud.rb

@@ -39,7 +39,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
-    PLUGIN_VERSION = '0.5.5.pre.1'
+    PLUGIN_VERSION = '0.5.5.pre.2'
 
     # Constants for service names.
     APPENGINE_SERVICE = 'appengine.googleapis.com'
@@ -87,6 +87,11 @@ module Fluent
     # The subservice_name overrides the subservice detection, if provided.
     config_param :subservice_name, :string, :default => nil
 
+    # Whether to reject log entries with invalid tags. If this option is set to
+    # false, tags will be made valid by converting any non-string tag to a
+    # string, and sanitizing any non-utf8 or other invalid characters.
+    config_param :require_valid_tags, :bool, :default => false
+
     # The regular expression to use on Kubernetes logs to extract some basic
     # information about the log source. The regex must contain capture groups
     # for pod_name, namespace_name, and container_name.
@@ -334,12 +339,32 @@ module Fluent
       [tag, time, record].to_msgpack
     end
 
+    # Given a tag, returns the corresponding valid tag if possible, or nil if
+    # the tag should be rejected. If 'require_valid_tags' is false, non-string
+    # tags are converted to strings, and invalid characters are sanitized;
+    # otherwise such tags are rejected.
+    def sanitize_tag(tag)
+      if @require_valid_tags &&
+         (!tag.is_a?(String) || tag == '' || convert_to_utf8(tag) != tag)
+        return nil
+      end
+      tag = convert_to_utf8(tag.to_s)
+      tag = '_' if tag == ''
+      tag
+    end
+
     def write(chunk)
       # Group the entries since we have to make one call per tag.
       grouped_entries = {}
       chunk.msgpack_each do |tag, *arr|
-        grouped_entries[tag] = [] unless grouped_entries.key?(tag)
-        grouped_entries[tag].push(arr)
+        sanitized_tag = sanitize_tag(tag)
+        if sanitized_tag.nil?
+          @log.warn "Dropping log entries with invalid tag: '#{tag}'. " \
+                    'A tag should be a string with utf8 characters.'
+          next
+        end
+        grouped_entries[sanitized_tag] ||= []
+        grouped_entries[sanitized_tag].push(arr)
       end
 
       grouped_entries.each do |tag, arr|
@@ -504,10 +529,9 @@ module Fluent
             labels_utf8_pairs = labels.map do |k, v|
               [k.encode('utf-8'), convert_to_utf8(v)]
             end
-            utf8_log_name = convert_to_utf8(log_name)
 
             write_request = Google::Logging::V1::WriteLogEntriesRequest.new(
-              log_name: "projects/#{@project_id}/logs/#{utf8_log_name}",
+              log_name: "projects/#{@project_id}/logs/#{log_name}",
               common_labels: Hash[labels_utf8_pairs],
               entries: entries
             )
@@ -986,7 +1010,7 @@ module Fluent
 
     def field_to_label(record, field, labels, label)
       return unless record.key?(field)
-      labels[label] = record[field].to_s
+      labels[label] = convert_to_utf8(record[field].to_s)
       record.delete(field)
     end
 
@@ -1081,18 +1105,22 @@ module Fluent
 
     def log_name(tag, common_labels)
       if @service_name == CLOUDFUNCTIONS_SERVICE
-        return 'cloud-functions'
+        tag = 'cloud-functions'
       elsif @running_on_managed_vm
         # Add a prefix to Managed VM logs to prevent namespace collisions.
-        return "#{APPENGINE_SERVICE}/#{tag}"
+        tag = "#{APPENGINE_SERVICE}/#{tag}"
       elsif @service_name == CONTAINER_SERVICE
         # For Kubernetes logs, use just the container name as the log name
         # if we have it.
         container_name_key = "#{CONTAINER_SERVICE}/container_name"
         if common_labels && common_labels.key?(container_name_key)
-          return common_labels[container_name_key]
+          sanitized_log_name = sanitize_tag(common_labels[container_name_key])
+          tag = sanitized_log_name unless sanitized_log_name.nil?
         end
       end
+      # Only encode the log name for the grpc path, since the non-grpc client
+      # lib already handles encoding.
+      tag = ERB::Util.url_encode(tag) if @use_grpc
       tag
     end
 

data/test/plugin/base_test.rb

@@ -128,6 +128,10 @@ module BaseTest
   )
   # rubocop:enable Metrics/LineLength
 
+  REQUIRE_VALID_TAGS_CONFIG = %(
+    require_valid_tags true
+  )
+
   NO_METADATA_SERVICE_CONFIG = %(
     use_metadata_service false
   )
@@ -320,6 +324,30 @@ module BaseTest
     'cacheValidatedWithOriginServer' => true
   }
 
+  # Tags and their sanitized and encoded version.
+  VALID_TAGS = {
+    'test' => 'test',
+    'germanß' => 'german%C3%9F',
+    'chinese中' => 'chinese%E4%B8%AD',
+    'specialCharacter/_-.' => 'specialCharacter%2F_-.',
+    'abc@&^$*' => 'abc%40%26%5E%24%2A',
+    '@&^$*' => '%40%26%5E%24%2A'
+  }
+  INVALID_TAGS = {
+    # Non-string tags.
+    123 => '123',
+    1.23 => '1.23',
+    [1, 2, 3] => '%5B1%2C%202%2C%203%5D',
+    { key: 'value' } => '%7B%22key%22%3D%3E%22value%22%7D',
+    # Non-utf8 string tags.
+    "nonutf8#{[0x92].pack('C*')}" => 'nonutf8%20',
+    "abc#{[0x92].pack('C*')}" => 'abc%20',
+    "#{[0x92].pack('C*')}" => '%20',
+    # Empty string tag.
+    '' => '_'
+  }
+  ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
+
   # Shared tests.
 
   def test_configure_service_account_application_default
@@ -626,6 +654,157 @@ module BaseTest
     end
   end
 
+  # Verify that we drop the log entries when 'require_valid_tags' is true and
+  # any non-string tags or tags with non-utf8 characters are detected.
+  def test_reject_invalid_tags_with_require_valid_tags_true
+    setup_gce_metadata_stubs
+    INVALID_TAGS.keys.each do |tag|
+      setup_logging_stubs do
+        @logs_sent = []
+        d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
+        d.emit('msg' => log_entry(0))
+        d.run
+      end
+      verify_log_entries(0, COMPUTE_PARAMS, 'structPayload')
+    end
+  end
+
+  # Verify that empty string container name should fail the kubernetes regex
+  # match, thus the original tag is used as the log name.
+  def test_handle_empty_container_name
+    setup_gce_metadata_stubs
+    setup_container_metadata_stubs
+    container_name = ''
+    # This tag will not match the kubernetes regex because it requires a
+    # non-empty container name.
+    tag = container_tag_with_container_name(container_name)
+    params = CONTAINER_FROM_METADATA_PARAMS.merge(
+      labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
+        "#{CONTAINER_SERVICE_NAME}/container_name" => container_name),
+      log_name: tag)
+    setup_logging_stubs([params]) do
+      @logs_sent = []
+      d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
+      d.emit(container_log_entry_with_metadata(log_entry(0), container_name))
+      d.run
+    end
+    verify_log_entries(1, params, 'textPayload')
+    assert_equal "projects/#{PROJECT_ID}/logs/#{tag}", @logs_sent[0]['logName']
+  end
+
+  # Verify that container names with non-utf8 characters should be rejected when
+  # 'require_valid_tags' is true.
+  def test_reject_non_utf8_container_name_with_require_valid_tags_true
+    setup_gce_metadata_stubs
+    setup_container_metadata_stubs
+    non_utf8_tags = INVALID_TAGS.select do |tag, _|
+      tag.is_a?(String) && !tag.empty?
+    end
+    non_utf8_tags.each do |container_name, encoded_name|
+      params = CONTAINER_FROM_METADATA_PARAMS.merge(
+        labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
+          "#{CONTAINER_SERVICE_NAME}/container_name" =>
+            URI.decode(encoded_name)),
+        log_name: encoded_name)
+      setup_logging_stubs([params]) do
+        @logs_sent = []
+        d = create_driver(REQUIRE_VALID_TAGS_CONFIG,
+                          container_tag_with_container_name(container_name))
+        d.emit(container_log_entry_with_metadata(log_entry(0), container_name))
+        d.run
+      end
+      verify_log_entries(0, params, 'textPayload')
+    end
+  end
+
+  # Verify that tags are properly encoded. When 'require_valid_tags' is true, we
+  # only accept string tags with utf8 characters.
+  def test_encode_tags_with_require_valid_tags_true
+    setup_gce_metadata_stubs
+    VALID_TAGS.each do |tag, encoded_tag|
+      setup_logging_stubs([COMPUTE_PARAMS.merge(log_name: encoded_tag)]) do
+        @logs_sent = []
+        d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
+        d.emit('msg' => log_entry(0))
+        d.run
+      end
+      verify_log_entries(1, COMPUTE_PARAMS, 'structPayload')
+      assert_equal "projects/#{PROJECT_ID}/logs/#{encoded_tag}",
+                   @logs_sent[0]['logName']
+    end
+  end
+
+  # Verify that tags extracted from container names are properly encoded.
+  def test_encode_tags_from_container_name_with_require_valid_tags_true
+    setup_gce_metadata_stubs
+    setup_container_metadata_stubs
+    VALID_TAGS.each do |tag, encoded_tag|
+      params = CONTAINER_FROM_METADATA_PARAMS.merge(
+        labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
+          "#{CONTAINER_SERVICE_NAME}/container_name" => tag),
+        log_name: encoded_tag)
+      setup_logging_stubs([params]) do
+        @logs_sent = []
+        d = create_driver(REQUIRE_VALID_TAGS_CONFIG,
+                          container_tag_with_container_name(tag))
+        d.emit(container_log_entry_with_metadata(log_entry(0), tag))
+        d.run
+      end
+      verify_log_entries(1, params, 'textPayload')
+      assert_equal "projects/#{PROJECT_ID}/logs/#{encoded_tag}",
+                   @logs_sent[0]['logName']
+    end
+  end
+
+  # Verify that tags are properly encoded and sanitized. When
+  # 'require_valid_tags' is false, we try to convert any non-string tags to
+  # strings, and replace non-utf8 characters with a replacement string.
+  def test_sanitize_tags_with_require_valid_tags_false
+    setup_gce_metadata_stubs
+    ALL_TAGS.each do |tag, sanitized_tag|
+      setup_logging_stubs([COMPUTE_PARAMS.merge(log_name: sanitized_tag)]) do
+        @logs_sent = []
+        d = create_driver(APPLICATION_DEFAULT_CONFIG, tag)
+        d.emit('msg' => log_entry(0))
+        d.run
+      end
+      verify_log_entries(1, COMPUTE_PARAMS, 'structPayload')
+      assert_equal "projects/#{PROJECT_ID}/logs/#{sanitized_tag}",
+                   @logs_sent[0]['logName']
+    end
+  end
+
+  # Verify that tags extracted from container names are properly encoded and
+  # sanitized.
+  def test_sanitize_tags_from_container_name_with_require_valid_tags_false
+    setup_gce_metadata_stubs
+    setup_container_metadata_stubs
+    # Log names are derived from container names for containers. And container
+    # names are extracted from the tag based on a regex match pattern. As a
+    # prerequisite, the tag should already be a string, thus we only test
+    # non-empty string cases here.
+    string_tags = ALL_TAGS.select { |tag, _| tag.is_a?(String) && !tag.empty? }
+    string_tags.each do |container_name, encoded_container_name|
+      # Container name in the label is sanitized but not encoded, while the log
+      # name is encoded.
+      params = CONTAINER_FROM_METADATA_PARAMS.merge(
+        labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
+          "#{CONTAINER_SERVICE_NAME}/container_name" =>
+            URI.decode(encoded_container_name)),
+        log_name: encoded_container_name)
+      setup_logging_stubs([params]) do
+        @logs_sent = []
+        d = create_driver(APPLICATION_DEFAULT_CONFIG,
+                          container_tag_with_container_name(container_name))
+        d.emit(container_log_entry_with_metadata(log_entry(0), container_name))
+        d.run
+      end
+      verify_log_entries(1, params, 'textPayload')
+      assert_equal "projects/#{PROJECT_ID}/logs/#{encoded_container_name}",
+                   @logs_sent[0]['logName']
+    end
+  end
+
   def test_timestamps
     setup_gce_metadata_stubs
     expected_ts = []
@@ -1196,7 +1375,13 @@ module BaseTest
                           CLOUDFUNCTIONS_REGION)
   end
 
-  def container_log_entry_with_metadata(log)
+  def container_tag_with_container_name(container_name)
+    "kubernetes.#{CONTAINER_POD_NAME}_#{CONTAINER_NAMESPACE_NAME}_" \
+      "#{container_name}"
+  end
+
+  def container_log_entry_with_metadata(
+      log, container_name = CONTAINER_CONTAINER_NAME)
     {
       log: log,
       stream: CONTAINER_STREAM,
@@ -1206,7 +1391,7 @@ module BaseTest
         namespace_name: CONTAINER_NAMESPACE_NAME,
         pod_id: CONTAINER_POD_ID,
         pod_name: CONTAINER_POD_NAME,
-        container_name: CONTAINER_CONTAINER_NAME,
+        container_name: container_name,
         labels: {
           CONTAINER_LABEL_KEY => CONTAINER_LABEL_VALUE
         }

data/test/plugin/test_out_google_cloud.rb

@@ -229,12 +229,15 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   # Set up http stubs to mock the external calls.
-  def setup_logging_stubs
-    [COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
-     CONTAINER_FROM_METADATA_PARAMS, CLOUDFUNCTIONS_PARAMS, CUSTOM_PARAMS,
-     EC2_PARAMS].each do |params|
+  def setup_logging_stubs(override_stub_params = nil)
+    stub_params = override_stub_params || \
+                  [COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
+                   CONTAINER_FROM_METADATA_PARAMS, CLOUDFUNCTIONS_PARAMS,
+                   CUSTOM_PARAMS, EC2_PARAMS]
+    stub_params.each do |params|
       stub_request(:post, uri_for_log(params)).to_return do |request|
-        @logs_sent << JSON.parse(request.body)
+        log_name = "projects/#{params[:project_id]}/logs/#{params[:log_name]}"
+        @logs_sent << JSON.parse(request.body).merge('logName' => log_name)
         { body: '' }
       end
     end

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -31,7 +31,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     { 8 => 'ResourceExhausted',
       12 => 'Unimplemented',
       16 => 'Unauthenticated' }.each_with_index do |(code, message), index|
-      setup_logging_stubs(true, code, message) do
+      setup_logging_stubs(nil, true, code, message) do
         d = create_driver(USE_GRPC_CONFIG, 'test',
                           GRPCLoggingMockFailingService.rpc_stub_class)
         # The API Client should not retry this and the plugin should consume the
@@ -51,7 +51,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
       13 => 'Internal',
       14 => 'Unavailable' }.each_with_index do |(code, message), index|
       exception_count = 0
-      setup_logging_stubs(true, code, message) do
+      setup_logging_stubs(nil, true, code, message) do
         d = create_driver(USE_GRPC_CONFIG, 'test',
                           GRPCLoggingMockFailingService.rpc_stub_class)
         # The API client should retry this once, then throw an exception which
@@ -224,12 +224,17 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
 
   # GRPC logging mock that successfully logs the records.
   class GRPCLoggingMockService < Google::Logging::V1::LoggingService::Service
-    def initialize(requests_received)
+    def initialize(stub_params, requests_received)
       super()
       @requests_received = requests_received
+      @expected_log_names = stub_params.map do |param|
+        "projects/#{param[:project_id]}/logs/#{param[:log_name]}"
+      end
     end
 
     def write_log_entries(request, _call)
+      fail GRPC::BadStatus.new(99, "Unexpected request: #{request.inspect}") \
+        unless @expected_log_names.include?(request.log_name)
       @requests_received << request
       WriteLogEntriesResponse.new
     end
@@ -283,14 +288,19 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
   end
 
   # Set up grpc stubs to mock the external calls.
-  def setup_logging_stubs(should_fail = false, code = 0, message = 'Ok')
+  def setup_logging_stubs(override_stub_params = nil, should_fail = false,
+                          code = 0, message = 'Ok')
+    stub_params = override_stub_params || \
+                  [COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
+                   CONTAINER_FROM_METADATA_PARAMS, CLOUDFUNCTIONS_PARAMS,
+                   CUSTOM_PARAMS, EC2_PARAMS]
     srv = GRPC::RpcServer.new
     @failed_attempts = []
     @requests_sent = []
     if should_fail
       grpc = GRPCLoggingMockFailingService.new(code, message, @failed_attempts)
     else
-      grpc = GRPCLoggingMockService.new(@requests_sent)
+      grpc = GRPCLoggingMockService.new(stub_params, @requests_sent)
     end
     srv.handle(grpc)
     srv.add_http2_port(GRPC_MOCK_HOST, :this_port_is_insecure)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.5.5.pre.1
+  version: 0.5.5.pre.2
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-31 00:00:00.000000000 Z
+date: 2017-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd