vmik-fluent-plugin-google-cloud 0.5.5 → 0.6.4.pre.alpha

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -31,7 +31,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     { 8 => 'ResourceExhausted',
       12 => 'Unimplemented',
       16 => 'Unauthenticated' }.each_with_index do |(code, message), index|
-      setup_logging_stubs(nil, true, code, message) do
+      setup_logging_stubs(true, code, message) do
         d = create_driver(USE_GRPC_CONFIG, 'test',
                           GRPCLoggingMockFailingService.rpc_stub_class)
         # The API Client should not retry this and the plugin should consume the
@@ -51,7 +51,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
       13 => 'Internal',
       14 => 'Unavailable' }.each_with_index do |(code, message), index|
       exception_count = 0
-      setup_logging_stubs(nil, true, code, message) do
+      setup_logging_stubs(true, code, message) do
         d = create_driver(USE_GRPC_CONFIG, 'test',
                           GRPCLoggingMockFailingService.rpc_stub_class)
         # The API client should retry this once, then throw an exception which
@@ -72,31 +72,50 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
   end
 
-  def test_http_request_from_record_with_referer_nil
+  # TODO: The code in the non-gRPC and gRPC tests is nearly identical.
+  # Refactor and remove duplication.
+  # TODO: Use status codes instead of int literals.
+  def test_prometheus_metrics
     setup_gce_metadata_stubs
-    setup_logging_stubs do
-      d = create_driver
-      d.emit('httpRequest' => http_request_message_with_nil_referer)
-      d.run
-    end
-    verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
-      assert_equal http_request_message_with_absent_referer,
-                   entry['httpRequest'], entry
-      assert_nil get_fields(entry['structPayload'])['httpRequest'], entry
-    end
-  end
-
-  def test_http_request_from_record_with_referer_absent
-    setup_gce_metadata_stubs
-    setup_logging_stubs do
-      d = create_driver
-      d.emit('httpRequest' => http_request_message_with_absent_referer)
-      d.run
-    end
-    verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
-      assert_equal http_request_message_with_absent_referer,
-                   entry['httpRequest'], entry
-      assert_nil get_fields(entry['structPayload'])['httpRequest'], entry
+    [
+      # Single successful request.
+      [false, 0, 1, 1, [1, 0, 1, 0]],
+      # Several successful requests.
+      [false, 0, 2, 1, [2, 0, 2, 0]],
+      # Single successful request with several entries.
+      [false, 0, 1, 2, [1, 0, 2, 0]],
+      # Single failed request that causes logs to be dropped.
+      [true, 16, 1, 1, [0, 1, 0, 1]],
+      # Single failed request that escalates without logs being dropped.
+      [true, 13, 1, 1, [0, 1, 0, 0]]
+    ].each do |should_fail, code, request_count, entry_count, metric_values|
+      setup_prometheus
+      setup_logging_stubs(should_fail, code, 'SomeMessage') do
+        (1..request_count).each do
+          d = create_driver(USE_GRPC_CONFIG + PROMETHEUS_ENABLE_CONFIG, 'test',
+                            GRPCLoggingMockFailingService.rpc_stub_class)
+          (1..entry_count).each do |i|
+            d.emit('message' => log_entry(i.to_s))
+          end
+          # rubocop:disable Lint/HandleExceptions
+          begin
+            d.run
+          rescue GRPC::BadStatus
+          end
+          # rubocop:enable Lint/HandleExceptions
+        end
+      end
+      successful_requests_count, failed_requests_count,
+        ingested_entries_count, dropped_entries_count = metric_values
+      assert_prometheus_metric_value(:stackdriver_successful_requests_count,
+                                     successful_requests_count, grpc: true)
+      assert_prometheus_metric_value(:stackdriver_failed_requests_count,
+                                     failed_requests_count,
+                                     grpc: true, code: code)
+      assert_prometheus_metric_value(:stackdriver_ingested_entries_count,
+                                     ingested_entries_count)
+      assert_prometheus_metric_value(:stackdriver_dropped_entries_count,
+                                     dropped_entries_count)
     end
   end
 
@@ -124,7 +143,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
     verify_index = 0
     verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
-      assert_equal_with_default(entry['metadata']['severity'],
+      assert_equal_with_default(entry['severity'],
                                 expected_severity[verify_index],
                                 'DEFAULT', entry)
       verify_index += 1
@@ -143,8 +162,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
              'null_field' => nil)
       d.run
     end
-    verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
-      fields = get_fields(entry['structPayload'])
+    verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+      fields = get_fields(entry['jsonPayload'])
       assert_equal 5, fields.size, entry
       assert_equal 'test log entry 0', get_string(fields['msg']), entry
       assert_equal 'test non utf8', get_string(fields['normal_key']), entry
@@ -174,9 +193,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
         d.run
       end
       verify_log_entries(1, COMPUTE_PARAMS) do |entry|
-        assert_equal expected, entry['metadata']['timestamp'],
-                     "Test with timestamp '#{input}' failed for " \
-                     "entry: '#{entry}'."
+        assert_equal expected, entry['timestamp'], 'Test with timestamp ' \
+                     "'#{input}' failed for entry: '#{entry}'."
       end
     end
   end
@@ -185,8 +203,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
 
   GRPC_MOCK_HOST = 'localhost:56789'
 
-  WriteLogEntriesRequest = Google::Logging::V1::WriteLogEntriesRequest
-  WriteLogEntriesResponse = Google::Logging::V1::WriteLogEntriesResponse
+  WriteLogEntriesRequest = Google::Logging::V2::WriteLogEntriesRequest
+  WriteLogEntriesResponse = Google::Logging::V2::WriteLogEntriesResponse
 
   USE_GRPC_CONFIG = %(
     use_grpc true
@@ -223,18 +241,13 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
   end
 
   # GRPC logging mock that successfully logs the records.
-  class GRPCLoggingMockService < Google::Logging::V1::LoggingService::Service
-    def initialize(stub_params, requests_received)
+  class GRPCLoggingMockService < Google::Logging::V2::LoggingServiceV2::Service
+    def initialize(requests_received)
       super()
       @requests_received = requests_received
-      @expected_log_names = stub_params.map do |param|
-        "projects/#{param[:project_id]}/logs/#{param[:log_name]}"
-      end
     end
 
     def write_log_entries(request, _call)
-      fail GRPC::BadStatus.new(99, "Unexpected request: #{request.inspect}") \
-        unless @expected_log_names.include?(request.log_name)
       @requests_received << request
       WriteLogEntriesResponse.new
     end
@@ -248,15 +261,17 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
 
     alias_method :list_logs, :_undefined
+    alias_method :list_log_entries, :_undefined
     alias_method :list_log_services, :_undefined
     alias_method :list_log_service_indexes, :_undefined
+    alias_method :list_monitored_resource_descriptors, :_undefined
     alias_method :delete_log, :_undefined
     undef_method :_undefined
   end
 
   # GRPC logging mock that fails and returns server side or client side errors.
   class GRPCLoggingMockFailingService <
-      Google::Logging::V1::LoggingService::Service
+      Google::Logging::V2::LoggingServiceV2::Service
     # 'code_sent' and 'message_sent' are references of external variables. We
     #  will assert the values of them later. 'code_value' and 'message_value'
     #  are actual error code and message we expect this mock to return.
@@ -281,26 +296,23 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
 
     alias_method :list_logs, :_undefined
+    alias_method :list_log_entries, :_undefined
     alias_method :list_log_services, :_undefined
     alias_method :list_log_service_indexes, :_undefined
+    alias_method :list_monitored_resource_descriptors, :_undefined
     alias_method :delete_log, :_undefined
     undef_method :_undefined
   end
 
   # Set up grpc stubs to mock the external calls.
-  def setup_logging_stubs(override_stub_params = nil, should_fail = false,
-                          code = 0, message = 'Ok')
-    stub_params = override_stub_params || \
-                  [COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
-                   CONTAINER_FROM_METADATA_PARAMS, CLOUDFUNCTIONS_PARAMS,
-                   CUSTOM_PARAMS, EC2_PARAMS]
+  def setup_logging_stubs(should_fail = false, code = 0, message = 'Ok')
     srv = GRPC::RpcServer.new
     @failed_attempts = []
     @requests_sent = []
     if should_fail
       grpc = GRPCLoggingMockFailingService.new(code, message, @failed_attempts)
     else
-      grpc = GRPCLoggingMockService.new(stub_params, @requests_sent)
+      grpc = GRPCLoggingMockService.new(@requests_sent)
     end
     srv.handle(grpc)
     srv.add_http2_port(GRPC_MOCK_HOST, :this_port_is_insecure)
@@ -320,8 +332,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
   # Verify the number and the content of the log entries match the expectation.
   # The caller can optionally provide a block which is called for each entry.
   def verify_log_entries(n, params, payload_type = 'textPayload', &block)
-    @requests_sent.each do |batch|
-      @logs_sent << JSON.parse(batch.to_json)
+    @requests_sent.each do |request|
+      @logs_sent << JSON.parse(request.to_json)
     end
     verify_json_log_entries(n, params, payload_type, &block)
   end
@@ -347,16 +359,9 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
   end
 
-  # Unset the 'referer' field.
-  def http_request_message_with_absent_referer
-    http_request_message.reject do |k, _|
-      k == 'referer'
-    end
-  end
-
-  # Get the fields of the struct payload.
-  def get_fields(struct_payload)
-    struct_payload['fields']
+  # Get the fields of the payload.
+  def get_fields(payload)
+    payload['fields']
   end
 
   # Get the value of a struct field.

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vmik-fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.5.5
-  prerelease: 
+  version: 0.6.4.pre.alpha
 platform: ruby
 authors:
 - Todd Derr
@@ -10,281 +9,256 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-15 00:00:00.000000000 Z
+date: 2017-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.12'
-    - - <
-      - !ruby/object:Gem::Version
-        version: 0.15.0
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.12'
-    - - <
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.15.0
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: googleapis-common-protos
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: google-api-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.0
-    - - <
-      - !ruby/object:Gem::Version
-        version: 0.10.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.0
-    - - <
+- !ruby/object:Gem::Dependency
+  name: google-cloud-logging
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 0.23.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.23.2
 - !ruby/object:Gem::Dependency
   name: googleauth
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.4'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.4'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: grpc
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
-    - - <
+        version: '1.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
-    - - <
+        version: '1.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.8'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '1.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.8'
-    - - <
-      - !ruby/object:Gem::Version
-        version: '1.9'
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.3'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.35.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.35.0
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.17'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-description: ! "   Fluentd output plugin for the Stackdriver Logging API, which will
-  make\n   logs viewable in the Developer Console's log viewer and can optionally\n
-  \  store them in Google Cloud Storage and/or BigQuery.\n   This is an official Google
-  Ruby gem.\n"
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+description: |2
+     Fluentd output plugin for the Stackdriver Logging API, which will make
+     logs viewable in the Developer Console's log viewer and can optionally
+     store them in Google Cloud Storage and/or BigQuery.
+     This is an official Google Ruby gem.
 email:
 - salty@google.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- fluent-plugin-google-cloud.gemspec
-- README.rdoc
+- CONTRIBUTING
 - Gemfile
+- Gemfile.lock
 - LICENSE
-- CONTRIBUTING
+- README.rdoc
+- Rakefile
+- fluent-plugin-google-cloud.gemspec
+- lib/fluent/plugin/monitoring.rb
+- lib/fluent/plugin/out_google_cloud.rb
 - test/helper.rb
 - test/plugin/base_test.rb
-- test/plugin/data/iam-credentials.json
-- test/plugin/data/credentials.json
+- test/plugin/constants.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
+- test/plugin/data/credentials.json
+- test/plugin/data/iam-credentials.json
 - test/plugin/data/invalid_credentials.json
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb
-- Rakefile
-- lib/fluent/plugin/out_google_cloud.rb
-- lib/google/logging/v1/log_entry_pb.rb
-- lib/google/logging/v1/logging_pb.rb
-- lib/google/logging/v1/logging_services_pb.rb
-- lib/google/logging/type/http_request_pb.rb
-- lib/google/logging/type/log_severity_pb.rb
 homepage: https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
 licenses:
 - Apache-2.0
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.6.12
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: fluentd output plugin for the Stackdriver Logging API
 test_files:
 - test/helper.rb
 - test/plugin/base_test.rb
-- test/plugin/data/iam-credentials.json
-- test/plugin/data/credentials.json
+- test/plugin/constants.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
+- test/plugin/data/credentials.json
+- test/plugin/data/iam-credentials.json
 - test/plugin/data/invalid_credentials.json
 - test/plugin/test_out_google_cloud.rb
 - test/plugin/test_out_google_cloud_grpc.rb

data/lib/google/logging/type/http_request_pb.rb

@@ -1,30 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: google/logging/type/http_request.proto
-
-require 'google/protobuf'
-
-Google::Protobuf::DescriptorPool.generated_pool.build do
-  add_message "google.logging.type.HttpRequest" do
-    optional :request_method, :string, 1
-    optional :request_url, :string, 2
-    optional :request_size, :int64, 3
-    optional :status, :int32, 4
-    optional :response_size, :int64, 5
-    optional :user_agent, :string, 6
-    optional :remote_ip, :string, 7
-    optional :server_ip, :string, 13
-    optional :referer, :string, 8
-    optional :cache_lookup, :bool, 11
-    optional :cache_hit, :bool, 9
-    optional :cache_validated_with_origin_server, :bool, 10
-    optional :cache_fill_bytes, :int64, 12
-  end
-end
-
-module Google
-  module Logging
-    module Type
-      HttpRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.logging.type.HttpRequest").msgclass
-    end
-  end
-end

data/lib/google/logging/type/log_severity_pb.rb

@@ -1,26 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: google/logging/type/log_severity.proto
-
-require 'google/protobuf'
-
-Google::Protobuf::DescriptorPool.generated_pool.build do
-  add_enum "google.logging.type.LogSeverity" do
-    value :DEFAULT, 0
-    value :DEBUG, 100
-    value :INFO, 200
-    value :NOTICE, 300
-    value :WARNING, 400
-    value :ERROR, 500
-    value :CRITICAL, 600
-    value :ALERT, 700
-    value :EMERGENCY, 800
-  end
-end
-
-module Google
-  module Logging
-    module Type
-      LogSeverity = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.logging.type.LogSeverity").enummodule
-    end
-  end
-end