fluent-plugin-google-cloud 0.6.23 → 0.6.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dbbe816a86396963d80686517b9dbd529fa6262c
-  data.tar.gz: b2cbe43dabcd06979dfcdef8949ad725456d338b
+  metadata.gz: e9d4e0c7112b19f2fdf43ac7bad7c2d030932f63
+  data.tar.gz: ea83fcd3888666a51aae9c4a03ea0ffb0bdba527
 SHA512:
-  metadata.gz: 80a44dbf1c339c0cbd5e0dfb8d776d7bbd62ae8c10dcd334618fe29c9b409094528c04c6a90f49f4b8174a2fea05845a68e4b288b1ade37098e01e2cd6f93357
-  data.tar.gz: 0e432cc308b9956b4b8a38d1fcac33cfd911518d384ff7a57e158d55761816df12b8f197fd47c47d4b506bb3c50d0f122444cce60c67397126bb43ed800fe9a9
+  metadata.gz: 2b179f5f31c11216c3e04b5d071cb244700f8352ed11fe1fa00fcf3282e5016e52a02e1880859d92a445d641e9bc1280a861a19896e5166ee493ad9b81569b2a
+  data.tar.gz: 3e9e799eb94b3e35156ce44cb8511f942082ac2ebc6bde9d21590543b421148f029f24f499c5373c7ca60a57054e066bbce3c9b19e8c312fa562f902420edf1d

data/README.rdoc

@@ -1,9 +1,10 @@
 = Google Cloud Logging plugin for {fluentd}[http://github.com/fluent/fluentd]
 
-fluent-plugin-google-cloud is an
-{output plugin for fluentd}[http://docs.fluentd.org/articles/output-plugin-overview]
-which sends logs to the
-{Stackdriver Logging API}[https://cloud.google.com/logging/docs/api/].
+fluent-plugin-google-cloud gem includes two plugins:
+1. A {filter plugin for fluentd}[http://docs.fluentd.org/articles/filter-plugin-overview]
+that embeds insertIds into log entries to guarantee order and uniqueness.
+2. An {output plugin for fluentd}[http://docs.fluentd.org/articles/output-plugin-overview]
+which sends logs to the {Stackdriver Logging API}[https://cloud.google.com/logging/docs/api/].
 
 This is an official Google Ruby gem.
 
@@ -12,8 +13,7 @@ This is an official Google Ruby gem.
 
 == Installation
 
-This gem is hosted at
-{RubyGems.org}[https://rubygems.org/gems/fluent-plugin-google-cloud]
+This gem is hosted at {RubyGems.org}[https://rubygems.org/gems/fluent-plugin-google-cloud]
 and can be installed using:
 
     $ gem install fluent-plugin-google-cloud
@@ -23,22 +23,31 @@ will also install and configure the gem.
 
 == Configuration
 
-To send logs to Google Cloud Logging, specify <code>type google_cloud</code>
-in a
-{match clause}[http://docs.fluentd.org/articles/config-file#2-ldquomatchrdquo-tell-fluentd-what-to-do]
-of your fluentd configuration file, for example:
+To embed insertIds into log entries, specify <code>@type add_insert_ids</code>
+in a {filter clause}[https://docs.fluentd.org/v1.0/articles/config-file#(3)-%E2%80%9Cfilter%E2%80%9D:-event-processing-pipeline]
+of your Fluentd configuration file, for example:
+
+    <filter **>
+      @type add_insert_ids
+      insert_id_key my_insert_id_field_name # Optional.
+    </filter>
+
+insert_id_key can be used to customize the insertId field name.
+
+To send logs to Google Cloud Logging, specify <code>@type google_cloud</code>
+in a {match clause}[http://docs.fluentd.org/articles/config-file#2-ldquomatchrdquo-tell-fluentd-what-to-do]
+of your Fluentd configuration file, for example:
 
     <match **>
-      type google_cloud
+      @type google_cloud
     </match>
 
-No further configuration is required. The plugin uses
+See detailed instructions on how to configure this output plugin {here}[https://cloud.google.com/logging/docs/agent/configuration#cloud-fluentd-config].
+The plugin uses
 {Google Application Default Credentials}[https://developers.google.com/identity/protocols/application-default-credentials]
-for authorization - for additional information see 
+for authorization - for additional information see
 {here}[https://cloud.google.com/logging/docs/agent/authorization].
 
-<em>The previously documented parameters auth_method, private_key_email,
-and private_key_path are removed, and can no longer be used.</em>
 
 == Copyright
 

data/fluent-plugin-google-cloud.gemspec

@@ -1,17 +1,17 @@
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-google-cloud'
   gem.description   = <<-eos
-   Fluentd output plugin for the Stackdriver Logging API, which will make
-   logs viewable in the Developer Console's log viewer and can optionally
-   store them in Google Cloud Storage and/or BigQuery.
+   Fluentd plugins for the Stackdriver Logging API, which will make logs
+   viewable in the Stackdriver Logs Viewer and can optionally store them
+   in Google Cloud Storage and/or BigQuery.
    This is an official Google Ruby gem.
 eos
-  gem.summary       = 'fluentd output plugin for the Stackdriver Logging API'
+  gem.summary       = 'fluentd plugins for the Stackdriver Logging API'
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.23'
-  gem.authors       = ['Ling Huang', 'Igor Peshansky']
+  gem.version       = '0.6.24'
+  gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
 

data/lib/fluent/plugin/filter_add_insert_ids.rb

@@ -0,0 +1,103 @@
+# Copyright 2018 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/filter'
+
+module Fluent
+  module Plugin
+    # Fluentd filter plugin for adding insertIds to guarantee log entry order
+    # and uniqueness.
+    # Sample log entries enriched by this plugin:
+    # {
+    #   "timestamp": "2017-08-22 13:35:28",
+    #   "message": "1",
+    #   "logging.googleapis.com/insertId": "aye7eakuf23h41aef0"
+    # }
+    # {
+    #   "timestamp": "2017-08-22 13:35:28",
+    #   "message": "2",
+    #   "logging.googleapis.com/insertId": "aye7eakuf23h41aef1"
+    # }
+    # {
+    #   "timestamp": "2017-08-22 13:35:28",
+    #   "message": "3",
+    #   "logging.googleapis.com/insertId": "aye7eakuf23h41aef2"
+    # }
+    class AddInsertIdsFilter < Filter
+      Fluent::Plugin.register_filter('add_insert_ids', self)
+
+      # Constants for configuration.
+      module ConfigConstants
+        # The default field name of insertIds in the log entry.
+        DEFAULT_INSERT_ID_KEY = 'logging.googleapis.com/insertId'.freeze
+        # The character size of the insertIds. This matches the setup in the
+        # Stackdriver Logging backend.
+        INSERT_ID_SIZE = 17
+        # The characters that are allowed in the insertIds. This matches the
+        # allowed collection by the Stackdriver Logging Backend.
+        ALLOWED_CHARS = (Array(0..9) + Array('a'..'z')).freeze
+      end
+
+      include self::ConfigConstants
+
+      desc 'The field name for insertIds in the log record.'
+      config_param :insert_id_key, :string, default: DEFAULT_INSERT_ID_KEY
+
+      # Expose attr_readers for testing.
+      attr_reader :insert_id_key
+
+      def start
+        super
+        @log = $log # rubocop:disable Style/GlobalVars
+
+        # Initialize the insertID.
+        @log.info "Started the add_insert_ids plugin with #{@insert_id_key}" \
+                  ' as the insert ID key.'
+        @insert_id = generate_initial_insert_id
+        @log.info "Initialized the insert ID key to #{@insert_id}."
+      end
+
+      def configure(conf)
+        super
+      end
+
+      def shutdown
+        super
+      end
+
+      # rubocop:disable Style/UnusedMethodArgument
+      def filter(tag, time, record)
+        # Only generate and add an insertId field if the record is a hash and
+        # the insert ID field is not already set (or set to an empty string).
+        if record.is_a?(Hash) && record[@insert_id_key].to_s.empty?
+          record[@insert_id_key] = increment_insert_id
+        end
+        record
+      end
+      # rubocop:enable Style/UnusedMethodArgument
+
+      private
+
+      # Generate a random string as the initial insertId.
+      def generate_initial_insert_id
+        Array.new(INSERT_ID_SIZE) { ALLOWED_CHARS.sample }.join
+      end
+
+      # Increment the insertId and return the new value.
+      def increment_insert_id
+        @insert_id = @insert_id.next
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_google_cloud.rb

@@ -148,6 +148,7 @@ module Fluent
         'logging.googleapis.com/sourceLocation'.freeze
       DEFAULT_TRACE_KEY = 'logging.googleapis.com/trace'.freeze
       DEFAULT_SPAN_ID_KEY = 'logging.googleapis.com/spanId'.freeze
+      DEFAULT_INSERT_ID_KEY = 'logging.googleapis.com/insertId'.freeze
 
       DEFAULT_METADATA_AGENT_URL =
         'http://local-metadata-agent.stackdriver.com:8000'.freeze
@@ -227,7 +228,16 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
-    PLUGIN_VERSION = '0.6.23'.freeze
+    # Extract plugin version by finding the spec this file was loaded from.
+    PLUGIN_VERSION = begin
+      dependency = Gem::Dependency.new('fluent-plugin-google-cloud')
+      all_specs, = Gem::SpecFetcher.fetcher.spec_for_dependency(dependency)
+      matching_spec, = all_specs.grep(
+        proc { |spec,| __FILE__.include?(spec.full_gem_path) }) do |spec,|
+          spec.version.to_s
+        end
+      matching_spec
+    end.freeze
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
@@ -273,6 +283,7 @@ module Fluent
       DEFAULT_SOURCE_LOCATION_KEY
     config_param :trace_key, :string, :default => DEFAULT_TRACE_KEY
     config_param :span_id_key, :string, :default => DEFAULT_SPAN_ID_KEY
+    config_param :insert_id_key, :string, :default => DEFAULT_INSERT_ID_KEY
 
     # Whether to try to detect if the record is a text log entry with JSON
     # content that needs to be parsed.
@@ -573,16 +584,17 @@ module Fluent
 
           is_json = false
           if @detect_json
-            # Save the timestamp and severity if available, then clear it out to
+            # Save the following fields if available, then clear them out to
             # allow for determining whether we should parse the log or message
             # field.
-            timestamp = record.delete('time')
-            severity = record.delete('severity')
+            preserved_keys = [
+              'time', 'severity', @trace_key, @span_id_key, @insert_id_key
+            ]
 
             # If the log is json, we want to export it as a structured log
             # unless there is additional metadata that would be lost.
             record_json = nil
-            if record.length == 1
+            if (record.keys - preserved_keys).length == 1
               %w(log message msg).each do |field|
                 if record.key?(field)
                   record_json = parse_json_or_nil(record[field])
@@ -590,13 +602,15 @@ module Fluent
               end
             end
             unless record_json.nil?
+              # Propagate these if necessary. Note that we don't want to
+              # override these keys in the JSON we've just parsed.
+              preserved_keys.each do |key|
+                record_json[key] ||= record[key] if record.key?(key)
+              end
+
               record = record_json
               is_json = true
             end
-            # Restore timestamp and severity if necessary. Note that we don't
-            # want to override these keys in the JSON we've just parsed.
-            record['time'] ||= timestamp if timestamp
-            record['severity'] ||= severity if severity
           end
 
           ts_secs, ts_nanos = compute_timestamp(
@@ -610,12 +624,12 @@ module Fluent
                                             ts_secs,
                                             ts_nanos)
 
-          # Get fully-qualified trace id for LogEntry "trace" field.
-          fq_trace_id = record.delete(@trace_key)
-          entry.trace = fq_trace_id if fq_trace_id
-
+          trace = record.delete(@trace_key)
+          entry.trace = trace if trace
           span_id = record.delete(@span_id_key)
           entry.span_id = span_id if span_id
+          insert_id = record.delete(@insert_id_key)
+          entry.insert_id = insert_id if insert_id
 
           set_log_entry_fields(record, entry)
           set_payload(entry_level_resource.type, record, entry, is_json)
@@ -1983,8 +1997,13 @@ module Fluent
           creds = :this_channel_is_insecure
         end
         port = ":#{uri.port}" if uri.port
+        user_agent = \
+          "#{PLUGIN_NAME}/#{PLUGIN_VERSION} grpc-ruby/#{GRPC::VERSION} " \
+          "#{Google::Apis::OS_VERSION}"
         @client = Google::Cloud::Logging::V2::LoggingServiceV2Client.new(
-          credentials: GRPC::Core::Channel.new("#{host}#{port}", nil, creds))
+          credentials: GRPC::Core::Channel.new(
+            "#{host}#{port}", { 'grpc.primary_user_agent' => user_agent },
+            creds))
       else
         # TODO: Use a non-default ClientOptions object.
         Google::Apis::ClientOptions.default.application_name = PLUGIN_NAME

data/test/plugin/base_test.rb

@@ -1221,13 +1221,32 @@ module BaseTest
 
   def test_log_entry_trace_field
     verify_field_key('trace', DEFAULT_TRACE_KEY, 'custom_trace_key',
-                     CONFIG_CUSTOM_TRACE_KEY_SPECIFIED,
-                     'projects/proj1/traces/1234567890abcdef1234567890abcdef')
+                     CONFIG_CUSTOM_TRACE_KEY_SPECIFIED, TRACE)
   end
 
   def test_log_entry_span_id_field
     verify_field_key('spanId', DEFAULT_SPAN_ID_KEY, 'custom_span_id_key',
-                     CONFIG_CUSTOM_SPAN_ID_KEY_SPECIFIED, '000000000000004a')
+                     CONFIG_CUSTOM_SPAN_ID_KEY_SPECIFIED, SPAN_ID)
+  end
+
+  def test_log_entry_insert_id_field
+    verify_field_key('insertId', DEFAULT_INSERT_ID_KEY, 'custom_insert_id_key',
+                     CONFIG_CUSTOM_INSERT_ID_KEY_SPECIFIED, INSERT_ID)
+  end
+
+  def test_cascading_json_detection_with_log_entry_trace_field
+    verify_cascading_json_detection_with_log_entry_fields(
+      'trace', DEFAULT_TRACE_KEY, TRACE, TRACE2)
+  end
+
+  def test_cascading_json_detection_with_log_entry_span_id_field
+    verify_cascading_json_detection_with_log_entry_fields(
+      'spanId', DEFAULT_SPAN_ID_KEY, SPAN_ID, SPAN_ID2)
+  end
+
+  def test_cascading_json_detection_with_log_entry_insert_id_field
+    verify_cascading_json_detection_with_log_entry_fields(
+      'insertId', DEFAULT_INSERT_ID_KEY, INSERT_ID, INSERT_ID2)
   end
 
   # Metadata Agent related tests.
@@ -1855,6 +1874,13 @@ module BaseTest
     }
   end
 
+  def structured_log_entry
+    {
+      'name' => 'test name',
+      'code' => 'test code'
+    }
+  end
+
   def log_entry(i)
     "test log entry #{i}"
   end
@@ -2004,6 +2030,71 @@ module BaseTest
     end
   end
 
+  # Cascading JSON detection is only triggered when the record has one field
+  # left with name "log", "message" or "msg". This test verifies additional
+  # LogEntry fields like spanId and traceId do not disable that by accident.
+  def verify_cascading_json_detection_with_log_entry_fields(
+      log_entry_field, default_key, root_level_value, nested_level_value)
+    setup_gce_metadata_stubs
+
+    # {
+    #   "logging.googleapis.com/XXX' => 'sample value'
+    #   "msg": {
+    #     "name": "test name",
+    #     "code": "test code"
+    #   }
+    # }
+    log_entry_with_root_level_field = {
+      default_key => root_level_value,
+      'msg' => structured_log_entry.to_json
+    }
+    # {
+    #   "msg": {
+    #     "logging.googleapis.com/XXX' => 'another value',
+    #     "name": "test name",
+    #     "code": "test code"
+    #   }
+    # }
+    log_entry_with_nested_level_field = {
+      'msg' => {
+        default_key => nested_level_value
+      }.merge(structured_log_entry).to_json
+    }
+    # {
+    #   "logging.googleapis.com/XXX' => 'sample value'
+    #   "msg": {
+    #     "logging.googleapis.com/XXX' => 'another value',
+    #     "name": "test name",
+    #     "code": "test code"
+    #   }
+    # }
+    log_entry_with_both_level_fields = log_entry_with_nested_level_field.merge(
+      default_key => root_level_value)
+
+    {
+      log_entry_with_root_level_field => root_level_value,
+      log_entry_with_nested_level_field => nested_level_value,
+      log_entry_with_both_level_fields => nested_level_value
+    }.each_with_index do |(input_log_entry, expected_value), index|
+      setup_logging_stubs do
+        @logs_sent = []
+        d = create_driver(DETECT_JSON_CONFIG)
+        d.emit(input_log_entry)
+        d.run
+      end
+      verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+        assert_equal expected_value, entry[log_entry_field],
+                     "Index #{index} failed. #{expected_value} is expected" \
+                     " for #{log_entry_field} field."
+        payload_fields = get_fields(entry['jsonPayload'])
+        assert_equal structured_log_entry.size, payload_fields.size
+        payload_fields.each do |key, value|
+          assert_equal structured_log_entry[key], get_string(value)
+        end
+      end
+    end
+  end
+
   def verify_field_key(log_entry_field, default_key, custom_key,
                        custom_key_config, sample_value)
     setup_gce_metadata_stubs

data/test/plugin/constants.rb

@@ -98,6 +98,14 @@ module Constants
   MANAGED_VM_BACKEND_NAME = 'default'.freeze
   MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'.freeze
 
+  # LogEntry fields for extraction.
+  TRACE = 'projects/proj1/traces/1234567890abcdef1234567890abcdef'.freeze
+  TRACE2 = 'projects/proj1/traces/1234567890abcdef1234567890fedcba'.freeze
+  SPAN_ID = '000000000000004a'.freeze
+  SPAN_ID2 = '000000000000007e'.freeze
+  INSERT_ID = 'fah7yr7iw64tg857y'.freeze
+  INSERT_ID2 = 'fah7yr7iw64tgaeuf'.freeze
+
   # Docker Container labels.
   DOCKER_CONTAINER_ID =
     '0d0f03ff8d3c42688692536d1af77a28cd135c0a5c531f25a31'.freeze
@@ -348,6 +356,10 @@ module Constants
     span_id_key custom_span_id_key
   ).freeze
 
+  CONFIG_CUSTOM_INSERT_ID_KEY_SPECIFIED = %(
+    insert_id_key custom_insert_id_key
+  ).freeze
+
   # Service configurations for various services.
 
   # GCE.

data/test/plugin/test_filter_add_insert_ids.rb

@@ -0,0 +1,135 @@
+# Copyright 2018 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative '../helper'
+
+require 'fluent/test/driver/filter'
+require 'fluent/plugin/filter_add_insert_ids'
+
+# Unit tests for filter_add_insert_ids plugin.
+class FilterAddInsertIdsTest < Test::Unit::TestCase
+  include Fluent::Plugin::AddInsertIdsFilter::ConfigConstants
+
+  CUSTOM_INSERT_ID_KEY = 'custom_insert_id_key'.freeze
+  INSERT_ID = 'aeyr82r92h249gh9h'.freeze
+  TEST_MESSAGE = 'test message for add_insert_ids plugin.'.freeze
+  APPLICATION_DEFAULT_CONFIG = ''.freeze
+  INSERT_ID_KEY_CONFIG = %(
+    insert_id_key #{CUSTOM_INSERT_ID_KEY}
+  ).freeze
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def test_configure_insert_id_key
+    {
+      APPLICATION_DEFAULT_CONFIG => DEFAULT_INSERT_ID_KEY,
+      INSERT_ID_KEY_CONFIG => CUSTOM_INSERT_ID_KEY
+    }.each do |config, insert_id_key|
+      d = create_driver(config)
+      assert_equal insert_id_key, d.instance.insert_id_key
+    end
+  end
+
+  def test_add_insert_ids
+    total_entry_count = 1000
+    d = create_driver
+    d.run do
+      total_entry_count.times do |index|
+        d.emit(log_entry(index))
+      end
+    end
+    filtered_events = d.filtered_as_array
+
+    assert_equal total_entry_count, filtered_events.size,
+                 "#{total_entry_count} log entries after filtering is" \
+                 " expected. Only #{filtered_events.size} are detected."
+    # The expected insertId will be assigned as we scan the first log entry.
+    expected_insert_id = nil
+    unique_insert_ids = Set.new
+    filtered_events.each_with_index do |event, index|
+      assert_equal 3, event.size, "Index #{index} failed. Log event should" \
+                   ' include 3 elements: tag, time and record.'
+      record = event[2]
+      assert_true record.is_a?(Hash), "Index #{index} failed. Log record" \
+                  " #{record} should be a hash."
+      assert_equal index, record['id'], "Index #{index} failed. Log entries" \
+                   ' should come in order.'
+      assert_equal TEST_MESSAGE, record['message'], "Index #{index} failed."
+
+      # Get the first insertID.
+      expected_insert_id = record[DEFAULT_INSERT_ID_KEY] if index == 0
+      insert_id = record[DEFAULT_INSERT_ID_KEY]
+      assert_equal expected_insert_id, insert_id, "Index #{index} failed."
+      expected_insert_id = expected_insert_id.next
+      assert_true insert_id < expected_insert_id,
+                  "Index #{index} failed. #{insert_id}" \
+                  " < #{expected_insert_id} is false."
+      unique_insert_ids << insert_id
+    end
+    assert_equal total_entry_count, unique_insert_ids.size,
+                 "Expected #{total_entry_count} unique insertIds." \
+                 " Only #{unique_insert_ids.size} found."
+  end
+
+  def test_insert_ids_not_added_if_present
+    log_entry_with_empty_insert_id = log_entry(0).merge(
+      DEFAULT_INSERT_ID_KEY => '')
+    {
+      log_entry(0).merge(DEFAULT_INSERT_ID_KEY => INSERT_ID) => true,
+      # Still generate insertId if it's an empty string
+      log_entry_with_empty_insert_id => false
+    }.each do |test_data|
+      input_log_entry, retain_original_insert_id = test_data
+      # Make a copy because the log entry gets modified by the filter plugin.
+      log_entry = input_log_entry.dup
+      d = create_driver
+      d.run do
+        d.emit(log_entry)
+      end
+      filtered_events = d.filtered_as_array
+
+      assert_equal 1, filtered_events.size, 'Exact 1 log entry after' \
+                   " filtering is expected. Test data: #{test_data}."
+      event = filtered_events[0]
+      assert_equal 3, event.size, 'Log event should include 3 elements: tag,' \
+                   " time and record. Test data: #{test_data}."
+      record = event[2]
+      assert_true record.is_a?(Hash), "Log record #{record} should be a hash." \
+                  " Test data: #{test_data}."
+      assert_equal 0, record['id'], "Test data: #{test_data}."
+      assert_equal TEST_MESSAGE, record['message'], "Test data: #{test_data}."
+      insert_id = record[DEFAULT_INSERT_ID_KEY]
+      assert_false insert_id.to_s.empty?, 'Insert ID should not be empty.' \
+                   " Test data: #{test_data}."
+      assert_equal retain_original_insert_id,
+                   input_log_entry[DEFAULT_INSERT_ID_KEY] == insert_id,
+                   "Input value is #{input_log_entry[DEFAULT_INSERT_ID_KEY]}." \
+                   " Output value is #{insert_id}. Test data: #{test_data}."
+    end
+  end
+
+  def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
+    Fluent::Test::FilterTestDriver.new(
+      Fluent::Plugin::AddInsertIdsFilter).configure(conf, true)
+  end
+
+  def log_entry(index)
+    {
+      'id' => index,
+      'message' => TEST_MESSAGE
+    }
+  end
+end

data/test/plugin/test_out_google_cloud.rb

@@ -25,6 +25,21 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_false d.instance.instance_variable_get(:@use_grpc)
   end
 
+  def test_user_agent
+    setup_gce_metadata_stubs
+    user_agent = nil
+    stub_request(:post, WRITE_LOG_ENTRIES_URI).to_return do |request|
+      user_agent = request.headers['User-Agent']
+      { body: '' }
+    end
+    d = create_driver
+    d.emit('message' => log_entry(0))
+    d.run
+    assert_match Regexp.new("#{Fluent::GoogleCloudOutput::PLUGIN_NAME}/" \
+                            "#{Fluent::GoogleCloudOutput::PLUGIN_VERSION}"), \
+                 user_agent
+  end
+
   def test_client_400
     setup_gce_metadata_stubs
     # The API Client should not retry this and the plugin should consume

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -27,6 +27,26 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     assert_true d.instance.instance_variable_get(:@use_grpc)
   end
 
+  def test_user_agent
+    setup_gce_metadata_stubs
+
+    user_agent = nil
+    # Record user agent when creating a GRPC::Core::Channel.
+    GRPC::Core::Channel.class_eval do
+      old_initialize = instance_method(:initialize)
+      define_method(:initialize) do |url, args, creds|
+        user_agent = args['grpc.primary_user_agent']
+        old_initialize.bind(self).call(url, args, creds)
+      end
+    end
+
+    d = create_driver
+    d.instance.send :init_api_client
+    assert_match Regexp.new("#{Fluent::GoogleCloudOutput::PLUGIN_NAME}/" \
+                            "#{Fluent::GoogleCloudOutput::PLUGIN_VERSION}"), \
+                 user_agent
+  end
+
   def test_client_error
     setup_gce_metadata_stubs
     {

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.23
+  version: 0.6.24
 platform: ruby
 authors:
-- Ling Huang
-- Igor Peshansky
+- Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-10 00:00:00.000000000 Z
+date: 2018-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -220,9 +219,9 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.7.1
 description: |2
-     Fluentd output plugin for the Stackdriver Logging API, which will make
-     logs viewable in the Developer Console's log viewer and can optionally
-     store them in Google Cloud Storage and/or BigQuery.
+     Fluentd plugins for the Stackdriver Logging API, which will make logs
+     viewable in the Stackdriver Logs Viewer and can optionally store them
+     in Google Cloud Storage and/or BigQuery.
      This is an official Google Ruby gem.
 email:
 - stackdriver-agents@google.com
@@ -236,6 +235,7 @@ files:
 - README.rdoc
 - Rakefile
 - fluent-plugin-google-cloud.gemspec
+- lib/fluent/plugin/filter_add_insert_ids.rb
 - lib/fluent/plugin/monitoring.rb
 - lib/fluent/plugin/out_google_cloud.rb
 - test/helper.rb
@@ -247,6 +247,7 @@ files:
 - test/plugin/data/invalid_credentials.json
 - test/plugin/data/new-style-credentials.json
 - test/plugin/test_driver.rb
+- test/plugin/test_filter_add_insert_ids.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb
 homepage: https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
@@ -272,7 +273,7 @@ rubyforge_project:
 rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
-summary: fluentd output plugin for the Stackdriver Logging API
+summary: fluentd plugins for the Stackdriver Logging API
 test_files:
 - test/helper.rb
 - test/plugin/base_test.rb
@@ -283,5 +284,6 @@ test_files:
 - test/plugin/data/invalid_credentials.json
 - test/plugin/data/new-style-credentials.json
 - test/plugin/test_driver.rb
+- test/plugin/test_filter_add_insert_ids.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb