fluent-plugin-google-cloud 0.6.13.pre.memory.2 → 0.6.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 26e55203b2c40d92dfa4087065c41b547291cdf0
-  data.tar.gz: dc3d1822611b4dfc35e00980449e525ab54e75e1
+  metadata.gz: 9d51deb1fd15ce338a6165a33e853704f7244051
+  data.tar.gz: 111bc6001f81947c1fd46e50a1d138487d864bc6
 SHA512:
-  metadata.gz: 670621b542a34fe36eac6b2ca0dda0a59043116476a61068060c7622c4def2def173e7d11342483e4f1b4bcf192cab2e0f736745558160e700c6ee7575189c2a
-  data.tar.gz: 1b37a3134df21f2fa17685f93f1ba2f2b1bb6a260add960a26b6dc6cd764f1cc42fff2581ac2bcfb739bddc01bd168bf6d23cecce8a422b022a8e9b306f7436e
+  metadata.gz: 7490512b67ea3bcdd2dcf60c12e29f8a2aa2f7232532a5d7f5a3e878c15913122bb7ee03b8c82b352d64b3224a90d900af3e2db94415ab694d5dce08eddd0193
+  data.tar.gz: 63e06d74b7148efc7c2ec31272ac15a999b921556ac08ef8f46807b110c22de0609f9a17573bb850a8ab2d6da7f88f90db2a1f6d9adb0cb64afd9ad70f07930b

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.6.13.pre.memory.2)
+    fluent-plugin-google-cloud (0.6.13)
       fluentd (~> 0.10)
       google-api-client (~> 0.14)
       google-cloud-logging (~> 1.2.3)
       googleapis-common-protos (~> 1.3)
       googleauth (~> 0.5)
-      grpc (~> 1.8.3)
+      grpc (~> 1.2.5)
       json (~> 1.8)
 
 GEM
@@ -70,10 +70,9 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
-    grpc (1.8.3)
+    grpc (1.2.5)
       google-protobuf (~> 3.1)
-      googleapis-common-protos-types (~> 1.0.0)
-      googleauth (>= 0.5.1, < 0.7)
+      googleauth (~> 0.5.1)
     hashdiff (0.3.7)
     http_parser.rb (0.6.0)
     httpclient (2.8.3)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.13.pre.memory.2'
+  gem.version       = '0.6.13'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -24,7 +24,7 @@ eos
   gem.add_runtime_dependency 'google-api-client', '~> 0.14'
   gem.add_runtime_dependency 'google-cloud-logging', '~> 1.2.3'
   gem.add_runtime_dependency 'googleauth', '~> 0.5'
-  gem.add_runtime_dependency 'grpc', '~> 1.8.3'
+  gem.add_runtime_dependency 'grpc', '~> 1.2.5'
   gem.add_runtime_dependency 'json', '~> 1.8'
 
   gem.add_development_dependency 'mocha', '~> 1.1'

data/lib/fluent/plugin/out_google_cloud.rb

@@ -195,7 +195,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
-    PLUGIN_VERSION = '0.6.13.pre.memory.2'.freeze
+    PLUGIN_VERSION = '0.6.13'.freeze
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
@@ -341,6 +341,10 @@ module Fluent
     config_param :metadata_agent_url, :string,
                  :default => DEFAULT_METADATA_AGENT_URL
 
+    # Whether to split log entries with different log tags into different
+    # requests when talking to Stackdriver Logging API.
+    config_param :split_logs_by_tag, :bool, :default => true
+
     # rubocop:enable Style/HashSyntax
 
     # TODO: Add a log_name config option rather than just using the tag?
@@ -555,8 +559,24 @@ module Fluent
         }
       end
 
-      requests_to_send.each do |request|
-        @write_request.call(request)
+      if @split_logs_by_tag
+        requests_to_send.each do |request|
+          @write_request.call(request)
+        end
+      else
+        # Combine all requests into one. The request level "log_name" will be
+        # ported to the entry level. The request level "resource" and "labels"
+        # are ignored as they should have been folded into the entry level
+        # "resource" and "labels" already anyway.
+        combined_entries = []
+        requests_to_send.each do |request|
+          request[:entries].each do |entry|
+            # Modify entries in-place as they are not needed later on.
+            entry.log_name = request[:log_name]
+          end
+          combined_entries.concat(request[:entries])
+        end
+        @write_request.call(entries: combined_entries)
       end
     end
 
@@ -607,17 +627,23 @@ module Fluent
       )
     end
 
-    def write_request_via_grpc(entries:, log_name:, resource:, labels:)
+    def write_request_via_grpc(entries:,
+                               log_name: '',
+                               resource: nil,
+                               labels: {})
       client = api_client
       entries_count = entries.length
       client.write_log_entries(
         # Ignore partial_success for gRPC path.
         entries,
         log_name: log_name,
-        resource: Google::Api::MonitoredResource.new(
-          type: resource.type,
-          labels: resource.labels.to_h
-        ),
+        # Leave resource nil if it's nil.
+        resource: if resource
+                    Google::Api::MonitoredResource.new(
+                      type: resource.type,
+                      labels: resource.labels.to_h
+                    )
+                  end,
         labels: labels.map do |k, v|
           [k.encode('utf-8'), convert_to_utf8(v)]
         end.to_h
@@ -705,7 +731,10 @@ module Fluent
                  error: error.to_s
     end
 
-    def write_request_via_rest(entries:, log_name:, resource:, labels:)
+    def write_request_via_rest(entries:,
+                               log_name: '',
+                               resource: nil,
+                               labels: {})
       client = api_client
       entries_count = entries.length
       client.write_entry_log_entries(

data/test/plugin/base_test.rb

@@ -605,6 +605,56 @@ module BaseTest
     end
   end
 
+  def test_configure_split_logs_by_tag
+    setup_gce_metadata_stubs
+    {
+      APPLICATION_DEFAULT_CONFIG => true,
+      DISABLE_SPLIT_LOGS_BY_TAG_CONFIG => false
+    }.each do |(config, split_logs_by_tag)|
+      d = create_driver(config)
+      assert_equal split_logs_by_tag,
+                   d.instance.instance_variable_get(:@split_logs_by_tag)
+    end
+  end
+
+  def test_split_logs_by_tag
+    setup_gce_metadata_stubs
+    log_entry_count = 5
+    dynamic_log_names = (0..log_entry_count - 1).map do |index|
+      "projects/test-project-id/logs/tag#{index}"
+    end
+    [
+      # [] returns nil for any index.
+      [APPLICATION_DEFAULT_CONFIG, log_entry_count, dynamic_log_names, []],
+      [DISABLE_SPLIT_LOGS_BY_TAG_CONFIG, 1, [''], dynamic_log_names]
+    ].each do |(config, request_count, request_log_names, entry_log_names)|
+      setup_prometheus
+      setup_logging_stubs do
+        @logs_sent = []
+        d = create_driver(config + PROMETHEUS_ENABLE_CONFIG, 'test', true)
+        log_entry_count.times do |i|
+          d.emit("tag#{i}", 'message' => log_entry(i))
+        end
+        d.run
+      end
+      @logs_sent.zip(request_log_names).each do |request, log_name|
+        assert_equal log_name, request['logName']
+      end
+      verify_log_entries(log_entry_count, COMPUTE_PARAMS_NO_LOG_NAME,
+                         'textPayload') do |entry, entry_index|
+        verify_default_log_entry_text(entry['textPayload'], entry_index,
+                                      entry)
+        assert_equal entry_log_names[entry_index], entry['logName']
+      end
+      # Verify the number of requests is different based on whether the
+      # 'split_logs_by_tag' flag is enabled.
+      assert_prometheus_metric_value(:stackdriver_successful_requests_count,
+                                     request_count, :aggregate)
+      assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
+                                     log_entry_count, :aggregate)
+    end
+  end
+
   def test_timestamps
     setup_gce_metadata_stubs
     current_time = Time.now
@@ -1591,12 +1641,13 @@ module BaseTest
 
   # The caller can optionally provide a block which is called for each entry.
   def verify_json_log_entries(n, params, payload_type = 'textPayload')
-    i = 0
+    entry_count = 0
     @logs_sent.each do |request|
       request['entries'].each do |entry|
         unless payload_type.empty?
-          assert entry.key?(payload_type), "Entry ##{i} did not contain " \
-            "expected #{payload_type} key: #{entry}"
+          assert entry.key?(payload_type),
+                 "Entry ##{entry_count} did not contain expected" \
+                 " #{payload_type} key: #{entry}."
         end
 
         # per-entry resource or log_name overrides the corresponding field
@@ -1608,22 +1659,27 @@ module BaseTest
         labels ||= request['labels']
         labels.merge!(entry['labels'] || {})
 
-        assert_equal \
-          "projects/#{params[:project_id]}/logs/#{params[:log_name]}", log_name
+        if params[:log_name]
+          assert_equal \
+            "projects/#{params[:project_id]}/logs/#{params[:log_name]}",
+            log_name
+        end
         assert_equal params[:resource][:type], resource['type']
         check_labels resource['labels'], params[:resource][:labels]
         check_labels labels, params[:labels]
         if block_given?
-          yield(entry, i)
+          yield(entry, entry_count)
         elsif payload_type == 'textPayload'
           # Check the payload for textPayload, otherwise it's up to the caller.
-          verify_default_log_entry_text(entry['textPayload'], i, entry)
+          verify_default_log_entry_text(entry['textPayload'], entry_count,
+                                        entry)
         end
-        i += 1
-        assert i <= n, "Number of entries #{i} exceeds expected number #{n}"
+        entry_count += 1
+        assert entry_count <= n,
+               "Number of entries #{entry_count} exceeds expected number #{n}."
       end
     end
-    assert i == n, "Number of entries #{i} does not match expected number #{n}"
+    assert_equal n, entry_count
   end
 
   def verify_container_logs(log_entry_factory, expected_params)
@@ -1781,7 +1837,13 @@ module BaseTest
   def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
     metric = Prometheus::Client.registry.get(metric_name)
     assert_not_nil(metric)
-    assert_equal(expected_value, metric.get(labels))
+    metric_value = if labels == :aggregate
+                     # Sum up all metric values regardless of the labels.
+                     metric.values.values.reduce(0.0, :+)
+                   else
+                     metric.get(labels)
+                   end
+    assert_equal(expected_value, metric_value)
   end
 
   # Get the fields of the payload.

data/test/plugin/constants.rb

@@ -166,6 +166,10 @@ module Constants
     detect_subservice false
   ).freeze
 
+  DISABLE_SPLIT_LOGS_BY_TAG_CONFIG = %(
+    split_logs_by_tag false
+  ).freeze
+
   PROMETHEUS_ENABLE_CONFIG = %(
     enable_monitoring true
     monitoring_type prometheus
@@ -243,7 +247,7 @@ module Constants
   # Service configurations for various services.
 
   # GCE.
-  COMPUTE_PARAMS = {
+  COMPUTE_PARAMS_NO_LOG_NAME = {
     resource: {
       type: COMPUTE_CONSTANTS[:resource_type],
       labels: {
@@ -251,12 +255,14 @@ module Constants
         'zone' => ZONE
       }
     },
-    log_name: 'test',
     project_id: PROJECT_ID,
     labels: {
       "#{COMPUTE_CONSTANTS[:service]}/resource_name" => HOSTNAME
     }
   }.freeze
+  COMPUTE_PARAMS = COMPUTE_PARAMS_NO_LOG_NAME.merge(
+    log_name: 'test'
+  ).freeze
   COMPUTE_PARAMS_WITH_METADATA_VM_ID_AND_ZONE = COMPUTE_PARAMS.merge(
     resource: COMPUTE_PARAMS[:resource].merge(
       labels: {

data/test/plugin/test_driver.rb

@@ -0,0 +1,55 @@
+# Copyright 2018 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/engine'
+require 'fluent/event'
+require 'fluent/test/input_test'
+
+module Fluent
+  module Test
+    # Similar to the standard BufferedOutputTestDriver, but allows multiple tags
+    # to exist in one chunk.
+    class MultiTagBufferedOutputTestDriver < InputTestDriver
+      def initialize(klass, &block)
+        super(klass, &block)
+        @entries = []
+      end
+
+      def emit(tag, record, time = Engine.now)
+        es = ArrayEventStream.new([[time, record]])
+        data = @instance.format_stream(tag, es)
+        @entries << data
+        self
+      end
+
+      def run(num_waits = 10)
+        result = nil
+        super(num_waits) do
+          chunk = @instance.buffer.generate_chunk(
+            @instance.metadata(nil, nil, nil)).staged!
+          @entries.each do |entry|
+            chunk.concat(entry, 1)
+          end
+
+          begin
+            result = @instance.write(chunk)
+          ensure
+            chunk.purge
+          end
+        end
+        result
+      end
+    end
+  end
+end

data/test/plugin/test_out_google_cloud.rb

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 require_relative 'base_test'
+require_relative 'test_driver'
 
 # Unit tests for Google Cloud Logging plugin
 class GoogleCloudOutputTest < Test::Unit::TestCase
@@ -308,9 +309,17 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   # Create a Fluentd output test driver with the Google Cloud Output plugin.
-  def create_driver(conf = APPLICATION_DEFAULT_CONFIG, tag = 'test')
-    Fluent::Test::BufferedOutputTestDriver.new(
-      Fluent::GoogleCloudOutput, tag).configure(conf, true)
+  def create_driver(conf = APPLICATION_DEFAULT_CONFIG,
+                    tag = 'test',
+                    multi_tags = false)
+    driver = if multi_tags
+               Fluent::Test::MultiTagBufferedOutputTestDriver.new(
+                 Fluent::GoogleCloudOutput)
+             else
+               Fluent::Test::BufferedOutputTestDriver.new(
+                 Fluent::GoogleCloudOutput, tag)
+             end
+    driver.configure(conf, true)
   end
 
   # Verify the number and the content of the log entries match the expectation.

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -15,6 +15,7 @@
 require 'grpc'
 
 require_relative 'base_test'
+require_relative 'test_driver'
 
 # Unit tests for Google Cloud Logging plugin
 class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
@@ -223,10 +224,18 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
   # grpc enabled. The signature of this method is different between the grpc
   # path and the non-grpc path. For grpc, an additional grpc stub class can be
   # passed in to construct the mock used by the test driver.
-  def create_driver(conf = APPLICATION_DEFAULT_CONFIG, tag = 'test')
+  def create_driver(conf = APPLICATION_DEFAULT_CONFIG,
+                    tag = 'test',
+                    multi_tags = false)
     conf += USE_GRPC_CONFIG
-    Fluent::Test::BufferedOutputTestDriver.new(
-      GoogleCloudOutputWithGRPCMock.new(@grpc_stub), tag).configure(conf, true)
+    driver = if multi_tags
+               Fluent::Test::MultiTagBufferedOutputTestDriver.new(
+                 GoogleCloudOutputWithGRPCMock.new(@grpc_stub))
+             else
+               Fluent::Test::BufferedOutputTestDriver.new(
+                 GoogleCloudOutputWithGRPCMock.new(@grpc_stub), tag)
+             end
+    driver.configure(conf, true)
   end
 
   # Google Cloud Fluent output stub with grpc mock.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.13.pre.memory.2
+  version: 0.6.13
 platform: ruby
 authors:
 - Todd Derr
@@ -87,14 +87,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.3
+        version: 1.2.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.3
+        version: 1.2.5
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +220,7 @@ files:
 - test/plugin/data/credentials.json
 - test/plugin/data/iam-credentials.json
 - test/plugin/data/invalid_credentials.json
+- test/plugin/test_driver.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb
 homepage: https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
@@ -237,9 +238,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.8
@@ -254,5 +255,6 @@ test_files:
 - test/plugin/data/credentials.json
 - test/plugin/data/iam-credentials.json
 - test/plugin/data/invalid_credentials.json
+- test/plugin/test_driver.rb
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb