fluent-plugin-google-cloud 0.7.5 → 0.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c8f41809aee8029c651ce49f3214853c27afa693e8396759cc51a940aa8474b
-  data.tar.gz: 595f84df87e4d92c2b554091731eb677bf5ef22f82e3a4b80a91468a9a404882
+  metadata.gz: 2a93bebee6c0d361009e34f871a735f85766599f9c42100d1d8d0580318177c3
+  data.tar.gz: 067150e565ba137c65846bd22ba0f1b47a0500a7ef16ae5f51a5c30284db5586
 SHA512:
-  metadata.gz: 34479043c6ad5f90f4859f8fc08e916a09ec70330cb81e5a3762c77d2db2a9daba743a985415de9ae3b1419150a2ad19c09c2935009988110130b7b325a9937f
-  data.tar.gz: f7f1826c54523041287c1ac1fd6cf5620c5eeb40d08d0a0988845d263ba4f71d8d57aa655e8afb8b33ca2003a9ba19eca407f3d15ebd6777e5a63de532de1877
+  metadata.gz: fa5799c20d5c118bdc333b7924b89cbca158f8a259efbd625280c963d0e9fff89c5bf842c7cf481dec325455bc25cdbc666b78edc1baba2a155e07440a84af81
+  data.tar.gz: 2701198d7c27c923a26cb44d26d3d03dae62c0b3135a6cb898bc302edd071c5dc5ee1232b650b105f8e20e352aca84d9cef6915d7586a46eb36656d7442b7514

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.7.5)
+    fluent-plugin-google-cloud (0.7.6)
       fluentd (= 1.2.5)
-      google-api-client (= 0.23.9)
-      google-cloud-logging (= 1.5.4)
+      google-api-client (= 0.28.4)
+      google-cloud-logging (= 1.6.0)
       google-protobuf (= 3.6.1)
       googleapis-common-protos (= 1.3.7)
       googleauth (= 0.6.6)
-      grpc (= 1.8.3)
+      grpc (= 1.14.2)
       json (= 2.1.0)
 
 GEM
@@ -17,6 +17,7 @@ GEM
     addressable (2.6.0)
       public_suffix (>= 2.0.2, < 4.0)
     ast (2.4.0)
+    concurrent-ruby (1.1.5)
     cool.io (1.5.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
@@ -36,19 +37,20 @@ GEM
       tzinfo (~> 1.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
-    google-api-client (0.23.9)
+    google-api-client (0.28.4)
       addressable (~> 2.5, >= 2.5.1)
-      googleauth (>= 0.5, < 0.7.0)
+      googleauth (>= 0.5, < 0.10.0)
       httpclient (>= 2.8.1, < 3.0)
       mime-types (~> 3.0)
       representable (~> 3.0)
       retriable (>= 2.0, < 4.0)
-      signet (~> 0.9)
+      signet (~> 0.10)
     google-cloud-core (1.3.0)
       google-cloud-env (~> 1.0)
     google-cloud-env (1.0.5)
       faraday (~> 0.11)
-    google-cloud-logging (1.5.4)
+    google-cloud-logging (1.6.0)
+      concurrent-ruby (~> 1.0)
       google-cloud-core (~> 1.2)
       google-gax (~> 1.3)
       googleapis-common-protos-types (>= 1.0.2)
@@ -73,10 +75,9 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.7)
-    grpc (1.8.3)
+    grpc (1.14.2)
       google-protobuf (~> 3.1)
       googleapis-common-protos-types (~> 1.0.0)
-      googleauth (>= 0.5.1, < 0.7)
     hashdiff (0.3.8)
     http_parser.rb (0.6.0)
     httpclient (2.8.3)
@@ -89,7 +90,7 @@ GEM
     mime-types-data (3.2018.0812)
     mocha (1.8.0)
       metaclass (~> 0.0.1)
-    msgpack (1.2.6)
+    msgpack (1.2.9)
     multi_json (1.13.1)
     multipart-post (2.0.0)
     os (1.0.0)
@@ -117,7 +118,7 @@ GEM
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.10.0)
-    safe_yaml (1.0.4)
+    safe_yaml (1.0.5)
     serverengine (2.1.0)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
@@ -137,7 +138,7 @@ GEM
     tzinfo-data (1.2018.9)
       tzinfo (>= 1.0.0)
     uber (0.1.0)
-    unicode-display_width (1.4.1)
+    unicode-display_width (1.5.0)
     webmock (2.3.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.7.5'
+  gem.version       = '0.7.6'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
@@ -22,10 +22,10 @@ eos
   gem.add_runtime_dependency 'fluentd', '1.2.5'
   gem.add_runtime_dependency 'googleapis-common-protos', '1.3.7'
   gem.add_runtime_dependency 'googleauth', '0.6.6'
-  gem.add_runtime_dependency 'google-api-client', '0.23.9'
-  gem.add_runtime_dependency 'google-cloud-logging', '1.5.4'
+  gem.add_runtime_dependency 'google-api-client', '0.28.4'
+  gem.add_runtime_dependency 'google-cloud-logging', '1.6.0'
   gem.add_runtime_dependency 'google-protobuf', '3.6.1'
-  gem.add_runtime_dependency 'grpc', '1.8.3'
+  gem.add_runtime_dependency 'grpc', '1.14.2'
   gem.add_runtime_dependency 'json', '2.1.0'
 
   gem.add_development_dependency 'mocha', '~> 1.1'

data/lib/fluent/plugin/out_google_cloud.rb

@@ -153,6 +153,7 @@ module Fluent
         'logging.googleapis.com/sourceLocation'.freeze
       DEFAULT_SPAN_ID_KEY = 'logging.googleapis.com/spanId'.freeze
       DEFAULT_TRACE_KEY = 'logging.googleapis.com/trace'.freeze
+      DEFAULT_TRACE_SAMPLED_KEY = 'logging.googleapis.com/trace_sampled'.freeze
 
       DEFAULT_METADATA_AGENT_URL =
         'http://local-metadata-agent.stackdriver.com:8000'.freeze
@@ -245,7 +246,7 @@ module Fluent
     PLUGIN_VERSION = begin
       # Extract plugin version from file path.
       match_data = __FILE__.match(
-        %r{fluent-plugin-google-cloud-(?<version>[0-9a-zA-Z\.]*)/})
+        %r{fluent-plugin-google-cloud-(?<version>[^/]*)/})
       if match_data
         match_data['version']
       else
@@ -306,6 +307,8 @@ module Fluent
       DEFAULT_SOURCE_LOCATION_KEY
     config_param :span_id_key, :string, :default => DEFAULT_SPAN_ID_KEY
     config_param :trace_key, :string, :default => DEFAULT_TRACE_KEY
+    config_param :trace_sampled_key, :string, :default =>
+      DEFAULT_TRACE_SAMPLED_KEY
 
     # Whether to try to detect if the record is a text log entry with JSON
     # content that needs to be parsed.
@@ -626,7 +629,8 @@ module Fluent
               @operation_key,
               @source_location_key,
               @span_id_key,
-              @trace_key
+              @trace_key,
+              @trace_sampled_key
             ]
 
             # If the log is json, we want to export it as a structured log
@@ -643,7 +647,8 @@ module Fluent
               # Propagate these if necessary. Note that we don't want to
               # override these keys in the JSON we've just parsed.
               preserved_keys.each do |key|
-                record_json[key] ||= record[key] if record.key?(key)
+                record_json[key] ||= record[key] if
+                  record.key?(key) && !record_json.key?(key)
               end
 
               record = record_json
@@ -667,13 +672,15 @@ module Fluent
                                             ts_secs,
                                             ts_nanos)
 
-          trace = record.delete(@trace_key)
-          entry.trace = compute_trace(trace) if trace
-
-          span_id = record.delete(@span_id_key)
-          entry.span_id = span_id if span_id
           insert_id = record.delete(@insert_id_key)
           entry.insert_id = insert_id if insert_id
+          span_id = record.delete(@span_id_key)
+          entry.span_id = span_id if span_id
+          trace = record.delete(@trace_key)
+          entry.trace = compute_trace(trace) if trace
+          trace_sampled = record.delete(@trace_sampled_key)
+          entry.trace_sampled = parse_bool(trace_sampled) unless
+            trace_sampled.nil?
 
           set_log_entry_fields(record, entry)
           set_payload(entry_level_resource.type, record, entry, is_json)
@@ -716,6 +723,10 @@ module Fluent
       end
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     private
 
     def compute_trace(trace)
@@ -969,8 +980,7 @@ module Fluent
     end
 
     def parse_json_or_nil(input)
-      # Only here to please rubocop...
-      return nil if input.nil?
+      return nil unless input.is_a?(String)
 
       input.each_codepoint do |c|
         if c == 123
@@ -1997,6 +2007,8 @@ module Fluent
       ret
     end
 
+    # TODO(qingling128): Fix the inconsistent behavior of 'message', 'log' and
+    # 'msg' in the next major version 1.0.0.
     def set_payload(resource_type, record, entry, is_json)
       # Only one of {text_payload, json_payload} will be set.
       text_payload = nil
@@ -2032,6 +2044,7 @@ module Fluent
                                json_payload
                              end
       elsif text_payload
+        text_payload = text_payload.to_s
         entry.text_payload = if @use_grpc
                                convert_to_utf8(text_payload)
                              else

data/test/plugin/base_test.rb

@@ -536,6 +536,78 @@ module BaseTest
     end
   end
 
+  # TODO(qingling128): Fix the inconsistent behavior of 'message', 'log' and
+  # 'msg' in the next major version 1.0.0.
+  def test_structured_payload_json_log_detect_json_with_hash_input
+    hash_value = {
+      'msg' => 'test log entry 0',
+      'tag2' => 'test',
+      'data' => 5000,
+      'some_null_field' => nil
+    }
+    [
+      {
+        config: APPLICATION_DEFAULT_CONFIG,
+        field_name: 'log',
+        expected_payload: 'jsonPayload'
+      },
+      {
+        config: APPLICATION_DEFAULT_CONFIG,
+        field_name: 'msg',
+        expected_payload: 'jsonPayload'
+      },
+      {
+        config: APPLICATION_DEFAULT_CONFIG,
+        field_name: 'message',
+        expected_payload: 'textPayload'
+      },
+      {
+        config: DETECT_JSON_CONFIG,
+        field_name: 'log',
+        expected_payload: 'jsonPayload'
+      },
+      {
+        config: DETECT_JSON_CONFIG,
+        field_name: 'msg',
+        expected_payload: 'jsonPayload'
+      },
+      {
+        config: DETECT_JSON_CONFIG,
+        field_name: 'message',
+        expected_payload: 'textPayload'
+      }
+    ].each do |test_params|
+      new_stub_context do
+        setup_gce_metadata_stubs
+        setup_logging_stubs do
+          d = create_driver(test_params[:config])
+          d.emit(test_params[:field_name] => hash_value)
+          d.run
+        end
+        if test_params[:expected_payload] == 'textPayload'
+          verify_log_entries(1, COMPUTE_PARAMS, 'textPayload') do |entry|
+            text_payload = entry['textPayload']
+            assert_equal '{"msg"=>"test log entry 0", "tag2"=>"test", ' \
+                         '"data"=>5000, "some_null_field"=>nil}',
+                         text_payload, entry
+          end
+        else
+          verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+            json_payload = get_fields(entry['jsonPayload'])
+            assert_equal 1, json_payload.size, entry
+            fields = get_fields(
+              get_struct(json_payload[test_params[:field_name]]))
+            assert_equal 4, fields.size, entry
+            assert_equal 'test log entry 0', get_string(fields['msg']), entry
+            assert_equal 'test', get_string(fields['tag2']), entry
+            assert_equal 5000, get_number(fields['data']), entry
+            assert_equal null_value, fields['some_null_field'], entry
+          end
+        end
+      end
+    end
+  end
+
   def test_structured_payload_json_log_detect_json_parsed
     setup_gce_metadata_stubs
     json_string = '{"msg": "test log entry 0", "tag2": "test", ' \
@@ -1442,6 +1514,15 @@ module BaseTest
                      sample_value: TRACE)
   end
 
+  def test_log_entry_trace_sampled_field
+    verify_field_key('traceSampled',
+                     default_key: DEFAULT_TRACE_SAMPLED_KEY,
+                     custom_key: 'custom_trace_sampled_key',
+                     custom_key_config:
+                       CONFIG_CUSTOM_TRACE_SAMPLED_KEY_SPECIFIED,
+                     sample_value: TRACE_SAMPLED)
+  end
+
   # Verify the cascading JSON detection of LogEntry fields.
 
   def test_cascading_json_detection_with_log_entry_insert_id_field
@@ -1490,6 +1571,15 @@ module BaseTest
       nested_level_value: TRACE2)
   end
 
+  def test_cascading_json_detection_with_log_entry_trace_sampled_field
+    verify_cascading_json_detection_with_log_entry_fields(
+      'traceSampled', DEFAULT_TRACE_SAMPLED_KEY,
+      root_level_value: TRACE_SAMPLED,
+      nested_level_value: TRACE_SAMPLED2,
+      default_value_from_root: false,
+      default_value_from_nested: false)
+  end
+
   # Verify that labels present in multiple inputs respect the expected priority
   # order:
   # 1. Labels from the field "logging.googleapis.com/labels" in payload.
@@ -2506,6 +2596,10 @@ module BaseTest
       :expected_value_from_root, root_level_value)
     expected_value_from_nested = expectation.fetch(
       :expected_value_from_nested, nested_level_value)
+    default_value_from_root = expectation.fetch(
+      :default_value_from_root, nil)
+    default_value_from_nested = expectation.fetch(
+      :default_value_from_nested, nil)
 
     setup_gce_metadata_stubs
 
@@ -2543,21 +2637,34 @@ module BaseTest
     log_entry_with_both_level_fields = log_entry_with_nested_level_field.merge(
       default_key => root_level_value)
 
-    {
-      log_entry_with_root_level_field => expected_value_from_root,
-      log_entry_with_nested_level_field => expected_value_from_nested,
-      log_entry_with_both_level_fields => expected_value_from_nested
-    }.each_with_index do |(input_log_entry, expected_value), index|
+    [
+      [
+        log_entry_with_root_level_field,
+        expected_value_from_root,
+        default_value_from_root
+      ],
+      [
+        log_entry_with_nested_level_field,
+        expected_value_from_nested,
+        default_value_from_nested
+      ],
+      [
+        log_entry_with_both_level_fields,
+        expected_value_from_nested,
+        default_value_from_nested
+      ]
+    ].each_with_index do |(log_entry, expected_value, default_value), index|
       setup_logging_stubs do
         @logs_sent = []
         d = create_driver(DETECT_JSON_CONFIG)
-        d.emit(input_log_entry)
+        d.emit(log_entry)
         d.run
       end
       verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload', false) do |entry|
-        assert_equal expected_value, entry[log_entry_field],
-                     "Index #{index} failed. #{expected_value} is expected" \
-                     " for #{log_entry_field} field."
+        assert_equal_with_default \
+          entry[log_entry_field], expected_value, default_value,
+          "Index #{index} failed. #{expected_value} is expected for " \
+          "#{log_entry_field} field."
         payload_fields = get_fields(entry['jsonPayload'])
         assert_equal structured_log_entry.size, payload_fields.size
         payload_fields.each do |key, value|

data/test/plugin/constants.rb

@@ -99,12 +99,14 @@ module Constants
   MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'.freeze
 
   # LogEntry fields for extraction.
-  TRACE = 'projects/proj1/traces/1234567890abcdef1234567890abcdef'.freeze
-  TRACE2 = 'projects/proj1/traces/1234567890abcdef1234567890fedcba'.freeze
-  SPAN_ID = '000000000000004a'.freeze
-  SPAN_ID2 = '000000000000007e'.freeze
   INSERT_ID = 'fah7yr7iw64tg857y'.freeze
   INSERT_ID2 = 'fah7yr7iw64tgaeuf'.freeze
+  SPAN_ID = '000000000000004a'.freeze
+  SPAN_ID2 = '000000000000007e'.freeze
+  TRACE = 'projects/proj1/traces/1234567890abcdef1234567890abcdef'.freeze
+  TRACE2 = 'projects/proj1/traces/1234567890abcdef1234567890fedcba'.freeze
+  TRACE_SAMPLED = true
+  TRACE_SAMPLED2 = false
 
   STACKDRIVER_TRACE_ID = '1234567890abcdef1234567890abcdef'.freeze
   FULL_STACKDRIVER_TRACE = \
@@ -396,6 +398,10 @@ module Constants
     trace_key custom_trace_key
   ).freeze
 
+  CONFIG_CUSTOM_TRACE_SAMPLED_KEY_SPECIFIED = %(
+    trace_sampled_key custom_trace_sampled_key
+  ).freeze
+
   # For 'labels' config.
   LABELS_FROM_LABELS_CONFIG = {
     'a_label_from_labels_config' => 'some_value',
@@ -1106,6 +1112,7 @@ module Constants
     DEFAULT_OPERATION_KEY => OPERATION_MESSAGE,
     DEFAULT_SOURCE_LOCATION_KEY => SOURCE_LOCATION_MESSAGE,
     DEFAULT_SPAN_ID_KEY => SPAN_ID,
-    DEFAULT_TRACE_KEY => TRACE
+    DEFAULT_TRACE_KEY => TRACE,
+    DEFAULT_TRACE_SAMPLED_KEY => TRACE_SAMPLED
   }.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.7.6
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-17 00:00:00.000000000 Z
+date: 2019-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -58,28 +58,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.23.9
+        version: 0.28.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.23.9
+        version: 0.28.4
 - !ruby/object:Gem::Dependency
   name: google-cloud-logging
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.5.4
+        version: 1.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.5.4
+        version: 1.6.0
 - !ruby/object:Gem::Dependency
   name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.8.3
+        version: 1.14.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.8.3
+        version: 1.14.2
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -260,7 +260,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 2.7.9
 signing_key: 
 specification_version: 4
 summary: fluentd plugins for the Stackdriver Logging API