fluent-plugin-google-cloud 0.6.25.1 → 0.7.0.pre.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bb53a82f49086e32adb57a617f0ad53812493815
-  data.tar.gz: 6964a23f146726e01f71a74378c0fa4327c5b582
+  metadata.gz: 20379f9f088d04bfb6b446d03347d5574ed78e03
+  data.tar.gz: c873feb14beedffe40b75714de0de47e0e117ec7
 SHA512:
-  metadata.gz: a9074434a1978f326a44fc4651d692a1e0f524c6c8ddeebda4efe89968059df9289749f21d85983a5308ee8629a208fb3963974ac234e8462a85a5c48aab94cb
-  data.tar.gz: 57e333f19ca5a7e5a9dbd2bd4b4bb65e461222bde2036b9a98a0e99ed2d95159d690c718b2b6bf95fe208dbbea2c7d5c1147a68b7d2dc39ff7f7dbfed1909664
+  metadata.gz: 07bba814b28cd239cd8150fbf5d8bb0a5d26a8d8acc42ed1f2905232e7e387d394ef74cb5a18125b585e02b79de3c0c0443e1b91ff0face222f07541769cb6f0
+  data.tar.gz: 286f822a8d7d4ea719326c7a8181ad6c666860bb351b8d724ee65ea1d508c9b3f1e0bfa2d5020b05218b1a2719f3bb77b1027cd084a04dbba65dc00312b2f7e0

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.25.1'
+  gem.version       = '0.7.0.pre.4'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
@@ -19,13 +19,13 @@ eos
   gem.test_files    = gem.files.grep(/^(test)/)
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency 'fluentd', '~> 0.10'
+  gem.add_runtime_dependency 'fluentd', '~> 1.2', '>= 1.2.5'
   gem.add_runtime_dependency 'googleapis-common-protos', '~> 1.3'
   gem.add_runtime_dependency 'google-api-client', '~> 0.17'
   gem.add_runtime_dependency 'google-cloud-logging', '~> 1.3', '>= 1.3.2'
   gem.add_runtime_dependency 'googleauth', '~> 0.6'
   gem.add_runtime_dependency 'grpc', '~> 1.0'
-  gem.add_runtime_dependency 'json', '~> 2.1'
+  gem.add_runtime_dependency 'json', '~> 1.8'
   gem.add_runtime_dependency 'google-protobuf', '~> 3.6', '>= 3.6.1'
 
   gem.add_development_dependency 'mocha', '~> 1.1'

data/lib/fluent/plugin/out_google_cloud.rb

@@ -165,6 +165,11 @@ module Fluent
       # monitored resource from Stackdriver Metadata agent.
       LOCAL_RESOURCE_ID_KEY = 'logging.googleapis.com/local_resource_id'.freeze
 
+      # The regexp matches stackdriver trace id format: 32-byte hex string.
+      # The format is documented in
+      # https://cloud.google.com/trace/docs/reference/v2/rpc/google.devtools.cloudtrace.v1#trace
+      STACKDRIVER_TRACE_ID_REGEXP = Regexp.new('^\h{32}$').freeze
+
       # Map from each field name under LogEntry to corresponding variables
       # required to perform field value extraction from the log record.
       LOG_ENTRY_FIELDS_MAP = {
@@ -410,6 +415,12 @@ module Fluent
     # Whether to attempt adjusting invalid log entry timestamps.
     config_param :adjust_invalid_timestamps, :bool, :default => true
 
+    # Whether to autoformat value of "logging.googleapis.com/trace" to
+    # comply with Stackdriver Trace format
+    # "projects/[PROJECT-ID]/traces/[TRACE-ID]" when setting
+    # LogEntry.trace.
+    config_param :autoformat_stackdriver_trace, :bool, :default => true
+
     # rubocop:enable Style/HashSyntax
 
     # TODO: Add a log_name config option rather than just using the tag?
@@ -627,7 +638,8 @@ module Fluent
                                             ts_nanos)
 
           trace = record.delete(@trace_key)
-          entry.trace = trace if trace
+          entry.trace = compute_trace(trace) if trace
+
           span_id = record.delete(@span_id_key)
           entry.span_id = span_id if span_id
           insert_id = record.delete(@insert_id_key)
@@ -676,6 +688,12 @@ module Fluent
 
     private
 
+    def compute_trace(trace)
+      return trace unless @autoformat_stackdriver_trace &&
+                          STACKDRIVER_TRACE_ID_REGEXP.match(trace)
+      "projects/#{@project_id}/traces/#{trace}"
+    end
+
     def construct_log_entry_in_grpc_format(labels,
                                            resource,
                                            severity,

data/test/plugin/base_test.rb

@@ -338,6 +338,98 @@ module BaseTest
     end
   end
 
+  def test_autoformat_enabled_with_stackdriver_trace_id_as_trace
+    [
+      APPLICATION_DEFAULT_CONFIG,
+      ENABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG
+    ].each do |config|
+      new_stub_context do
+        setup_gce_metadata_stubs
+        setup_logging_stubs do
+          d = create_driver(config)
+          d.emit(DEFAULT_TRACE_KEY => STACKDRIVER_TRACE_ID)
+          d.run
+          verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+            assert_equal FULL_STACKDRIVER_TRACE, entry['trace'],
+                         'stackdriver trace id should be autoformatted ' \
+                         'when autoformat_stackdriver_trace is enabled.'
+          end
+        end
+      end
+    end
+  end
+
+  def test_autoformat_disabled_with_stackdriver_trace_id_as_trace
+    setup_gce_metadata_stubs
+    setup_logging_stubs do
+      d = create_driver(DISABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG)
+      d.emit(DEFAULT_TRACE_KEY => STACKDRIVER_TRACE_ID)
+      d.run
+      verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+        assert_equal STACKDRIVER_TRACE_ID, entry['trace'],
+                     'trace as stackdriver trace id should not be ' \
+                     'autoformatted with config ' \
+                     "#{DISABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG}."
+      end
+    end
+  end
+
+  def test_no_trace_when_trace_key_not_exists_with_any_autoformat_config
+    [
+      APPLICATION_DEFAULT_CONFIG,
+      ENABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG,
+      DISABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG
+    ].each do |config|
+      new_stub_context do
+        setup_gce_metadata_stubs
+        setup_logging_stubs do
+          d = create_driver(config)
+          d.emit('msg' => log_entry(0))
+          d.run
+          verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+            assert_nil entry['trace']
+          end
+        end
+      end
+    end
+  end
+
+  def test_non_stackdriver_trace_id_compliant_trace_with_any_autoformat_config
+    configs = [
+      APPLICATION_DEFAULT_CONFIG,
+      ENABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG,
+      DISABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG
+    ]
+    traces = [
+      TRACE, # Full trace won't be modified.
+      EMPTY_STRING,
+      INVALID_SHORT_STACKDRIVER_TRACE_ID,
+      INVALID_LONG_STACKDRIVER_TRACE_ID,
+      INVALID_NON_HEX_STACKDRIVER_TRACE_ID,
+      INVALID_TRACE_NO_TRACE_ID,
+      INVALID_TRACE_NO_PROJECT_ID,
+      INVALID_TRACE_WITH_SHORT_TRACE_ID,
+      INVALID_TRACE_WITH_LONG_TRACE_ID,
+      INVALID_TRACE_WITH_NON_HEX_TRACE_ID
+    ]
+    configs.product(traces).each do |config, trace|
+      new_stub_context do
+        setup_gce_metadata_stubs
+        setup_logging_stubs do
+          d = create_driver(config)
+          d.emit(DEFAULT_TRACE_KEY => trace)
+          d.run
+          verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+            assert_equal_with_default \
+              entry['trace'], trace, '',
+              'trace as non stackdriver trace id should not be ' \
+              "autoformatted with config #{config}."
+          end
+        end
+      end
+    end
+  end
+
   def test_structured_payload_malformatted_log
     setup_gce_metadata_stubs
     message = 'test message'

data/test/plugin/constants.rb

@@ -106,6 +106,30 @@ module Constants
   INSERT_ID = 'fah7yr7iw64tg857y'.freeze
   INSERT_ID2 = 'fah7yr7iw64tgaeuf'.freeze
 
+  STACKDRIVER_TRACE_ID = '1234567890abcdef1234567890abcdef'.freeze
+  FULL_STACKDRIVER_TRACE = \
+    "projects/#{PROJECT_ID}/traces/#{STACKDRIVER_TRACE_ID}".freeze
+
+  # Invalid trace id for stackdriver.
+  EMPTY_STRING = ''.freeze
+  INVALID_SHORT_STACKDRIVER_TRACE_ID = '1234567890abcdef'.freeze
+  INVALID_LONG_STACKDRIVER_TRACE_ID = \
+    '1234567890abcdef1234567890abcdef123'.freeze
+  INVALID_NON_HEX_STACKDRIVER_TRACE_ID = \
+    '1234567890abcdef1234567890abcdeZ'.freeze
+
+  # Invalid full format of stackdriver trace.
+  INVALID_TRACE_NO_TRACE_ID = "projects/#{PROJECT_ID}/traces/".freeze
+  INVALID_TRACE_NO_PROJECT_ID = \
+    "projects//traces/#{STACKDRIVER_TRACE_ID}".freeze
+  INVALID_TRACE_WITH_SHORT_TRACE_ID = \
+    "projects/#{PROJECT_ID}/traces/#{INVALID_SHORT_STACKDRIVER_TRACE_ID}".freeze
+  INVALID_TRACE_WITH_LONG_TRACE_ID = \
+    "projects/#{PROJECT_ID}/traces/#{INVALID_LONG_STACKDRIVER_TRACE_ID}".freeze
+  INVALID_TRACE_WITH_NON_HEX_TRACE_ID = \
+    "projects/#{PROJECT_ID}/" \
+    "traces/#{INVALID_NON_HEX_STACKDRIVER_TRACE_ID}".freeze
+
   # Docker Container labels.
   DOCKER_CONTAINER_ID =
     '0d0f03ff8d3c42688692536d1af77a28cd135c0a5c531f25a31'.freeze
@@ -265,6 +289,14 @@ module Constants
     enable_metadata_agent false
   ).freeze
 
+  ENABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG = %(
+    autoformat_stackdriver_trace true
+  ).freeze
+
+  DISABLE_AUTOFORMAT_STACKDRIVER_TRACE_CONFIG = %(
+    autoformat_stackdriver_trace false
+  ).freeze
+
   DOCKER_CONTAINER_CONFIG = %(
     enable_metadata_agent true
     label_map { "source": "#{DOCKER_CONSTANTS[:service]}/stream" }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.25.1
+  version: 0.7.0.pre.4
 platform: ruby
 authors:
 - Stackdriver Agents Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-13 00:00:00.000000000 Z
+date: 2018-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.5
 - !ruby/object:Gem::Dependency
   name: googleapis-common-protos
   requirement: !ruby/object:Gem::Requirement
@@ -106,14 +112,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
@@ -234,6 +240,7 @@ files:
 - LICENSE
 - README.rdoc
 - Rakefile
+- fluent-plugin-google-cloud-0.7.0.gem
 - fluent-plugin-google-cloud.gemspec
 - lib/fluent/plugin/filter_add_insert_ids.rb
 - lib/fluent/plugin/monitoring.rb
@@ -265,9 +272,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.14