fluent-plugin-google-cloud 0.5.6 → 0.6.0.v2.alpha.1

data/test/plugin/test_out_google_cloud.rb

@@ -28,18 +28,18 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     setup_gce_metadata_stubs
     # The API Client should not retry this and the plugin should consume
     # the exception.
-    stub_request(:post, uri_for_log(COMPUTE_PARAMS))
+    stub_request(:post, WRITE_LOG_ENTRIES_URI)
       .to_return(status: 400, body: 'Bad Request')
     d = create_driver
     d.emit('message' => log_entry(0))
     d.run
-    assert_requested(:post, uri_for_log(COMPUTE_PARAMS), times: 1)
+    assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
   end
 
   # All credentials errors resolve to a 401.
   def test_client_401
     setup_gce_metadata_stubs
-    stub_request(:post, uri_for_log(COMPUTE_PARAMS))
+    stub_request(:post, WRITE_LOG_ENTRIES_URI)
       .to_return(status: 401, body: 'Unauthorized')
     d = create_driver
     d.emit('message' => log_entry(0))
@@ -48,14 +48,14 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     rescue Google::Apis::AuthorizationError => error
       assert_equal 'Unauthorized', error.message
     end
-    assert_requested(:post, uri_for_log(COMPUTE_PARAMS), times: 2)
+    assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 2)
   end
 
   def test_server_error
     setup_gce_metadata_stubs
     # The API client should retry this once, then throw an exception which
     # gets propagated through the plugin.
-    stub_request(:post, uri_for_log(COMPUTE_PARAMS))
+    stub_request(:post, WRITE_LOG_ENTRIES_URI)
       .to_return(status: 500, body: 'Server Error')
     d = create_driver
     d.emit('message' => log_entry(0))
@@ -66,7 +66,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
       assert_equal 'Server error', error.message
       exception_count += 1
     end
-    assert_requested(:post, uri_for_log(COMPUTE_PARAMS), times: 1)
+    assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
     assert_equal 1, exception_count
   end
 
@@ -84,7 +84,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
       # the proto.
       assert_equal http_request_message_with_nil_referer,
                    entry['httpRequest'], entry
-      assert_nil get_fields(entry['structPayload'])['httpRequest'], entry
+      assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
     end
   end
 
@@ -113,7 +113,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     verify_index = 0
     verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
       assert_equal expected_severity[verify_index],
-                   entry['metadata']['severity'], entry
+                   entry['severity'], entry
       verify_index += 1
     end
   end
@@ -181,7 +181,6 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_equal('ERROR', test_obj.parse_severity('ERROR  '))
     assert_equal('ERROR', test_obj.parse_severity('   ERROR  '))
     assert_equal('ERROR', test_obj.parse_severity("\t  ERROR  "))
-
     # space in the middle should not be stripped.
     assert_equal('DEFAULT', test_obj.parse_severity('ER ROR'))
 
@@ -208,9 +207,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
         d.run
       end
       verify_log_entries(1, COMPUTE_PARAMS) do |entry|
-        assert_equal timestamp, entry['metadata']['timestamp'],
-                     "Test with timestamp '#{timestamp}' failed for " \
-                     "entry: '#{entry}'."
+        assert_equal timestamp, entry['timestamp'], 'Test with timestamp ' \
+                     "'#{timestamp}' failed for entry: '#{entry}'."
       end
     end
   end
@@ -221,25 +219,11 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     hash.merge(new_key => hash[old_key]).reject { |k, _| k == old_key }
   end
 
-  # The REST path uses old bindings that were generated prior to the field
-  # rename, and has to use the old name, which is 'validatedWithOriginServer'.
-  def http_request_message
-    rename_key(super, 'cacheValidatedWithOriginServer',
-               'validatedWithOriginServer')
-  end
-
   # Set up http stubs to mock the external calls.
-  def setup_logging_stubs(override_stub_params = nil)
-    stub_params = override_stub_params || \
-                  [COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
-                   CONTAINER_FROM_METADATA_PARAMS, CLOUDFUNCTIONS_PARAMS,
-                   CUSTOM_PARAMS, EC2_PARAMS]
-    stub_params.each do |params|
-      stub_request(:post, uri_for_log(params)).to_return do |request|
-        log_name = "projects/#{params[:project_id]}/logs/#{params[:log_name]}"
-        @logs_sent << JSON.parse(request.body).merge('logName' => log_name)
-        { body: '' }
-      end
+  def setup_logging_stubs
+    stub_request(:post, WRITE_LOG_ENTRIES_URI).to_return do |request|
+      @logs_sent << JSON.parse(request.body)
+      { body: '' }
     end
     yield
   end
@@ -270,9 +254,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     end
   end
 
-  # Get the fields of the struct payload.
-  def get_fields(struct_payload)
-    struct_payload
+  # Get the fields of the payload.
+  def get_fields(payload)
+    payload
   end
 
   # Get the value of a struct field.

data/test/plugin/test_out_google_cloud_grpc.rb

@@ -31,7 +31,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     { 8 => 'ResourceExhausted',
       12 => 'Unimplemented',
       16 => 'Unauthenticated' }.each_with_index do |(code, message), index|
-      setup_logging_stubs(nil, true, code, message) do
+      setup_logging_stubs(true, code, message) do
         d = create_driver(USE_GRPC_CONFIG, 'test',
                           GRPCLoggingMockFailingService.rpc_stub_class)
         # The API Client should not retry this and the plugin should consume the
@@ -51,7 +51,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
       13 => 'Internal',
       14 => 'Unavailable' }.each_with_index do |(code, message), index|
       exception_count = 0
-      setup_logging_stubs(nil, true, code, message) do
+      setup_logging_stubs(true, code, message) do
         d = create_driver(USE_GRPC_CONFIG, 'test',
                           GRPCLoggingMockFailingService.rpc_stub_class)
         # The API client should retry this once, then throw an exception which
@@ -82,7 +82,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
       assert_equal http_request_message_with_absent_referer,
                    entry['httpRequest'], entry
-      assert_nil get_fields(entry['structPayload'])['httpRequest'], entry
+      assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
     end
   end
 
@@ -96,7 +96,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
       assert_equal http_request_message_with_absent_referer,
                    entry['httpRequest'], entry
-      assert_nil get_fields(entry['structPayload'])['httpRequest'], entry
+      assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
     end
   end
 
@@ -124,7 +124,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
     verify_index = 0
     verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
-      assert_equal_with_default(entry['metadata']['severity'],
+      assert_equal_with_default(entry['severity'],
                                 expected_severity[verify_index],
                                 'DEFAULT', entry)
       verify_index += 1
@@ -143,8 +143,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
              'null_field' => nil)
       d.run
     end
-    verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
-      fields = get_fields(entry['structPayload'])
+    verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
+      fields = get_fields(entry['jsonPayload'])
       assert_equal 5, fields.size, entry
       assert_equal 'test log entry 0', get_string(fields['msg']), entry
       assert_equal 'test non utf8', get_string(fields['normal_key']), entry
@@ -174,9 +174,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
         d.run
       end
       verify_log_entries(1, COMPUTE_PARAMS) do |entry|
-        assert_equal expected, entry['metadata']['timestamp'],
-                     "Test with timestamp '#{input}' failed for " \
-                     "entry: '#{entry}'."
+        assert_equal expected, entry['timestamp'], 'Test with timestamp ' \
+                     "'#{input}' failed for entry: '#{entry}'."
       end
     end
   end
@@ -185,8 +184,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
 
   GRPC_MOCK_HOST = 'localhost:56789'
 
-  WriteLogEntriesRequest = Google::Logging::V1::WriteLogEntriesRequest
-  WriteLogEntriesResponse = Google::Logging::V1::WriteLogEntriesResponse
+  WriteLogEntriesRequest = Google::Logging::V2::WriteLogEntriesRequest
+  WriteLogEntriesResponse = Google::Logging::V2::WriteLogEntriesResponse
 
   USE_GRPC_CONFIG = %(
     use_grpc true
@@ -223,18 +222,13 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
   end
 
   # GRPC logging mock that successfully logs the records.
-  class GRPCLoggingMockService < Google::Logging::V1::LoggingService::Service
-    def initialize(stub_params, requests_received)
+  class GRPCLoggingMockService < Google::Logging::V2::LoggingServiceV2::Service
+    def initialize(requests_received)
       super()
       @requests_received = requests_received
-      @expected_log_names = stub_params.map do |param|
-        "projects/#{param[:project_id]}/logs/#{param[:log_name]}"
-      end
     end
 
     def write_log_entries(request, _call)
-      fail GRPC::BadStatus.new(99, "Unexpected request: #{request.inspect}") \
-        unless @expected_log_names.include?(request.log_name)
       @requests_received << request
       WriteLogEntriesResponse.new
     end
@@ -248,15 +242,17 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
 
     alias_method :list_logs, :_undefined
+    alias_method :list_log_entries, :_undefined
     alias_method :list_log_services, :_undefined
     alias_method :list_log_service_indexes, :_undefined
+    alias_method :list_monitored_resource_descriptors, :_undefined
     alias_method :delete_log, :_undefined
     undef_method :_undefined
   end
 
   # GRPC logging mock that fails and returns server side or client side errors.
   class GRPCLoggingMockFailingService <
-      Google::Logging::V1::LoggingService::Service
+      Google::Logging::V2::LoggingServiceV2::Service
     # 'code_sent' and 'message_sent' are references of external variables. We
     #  will assert the values of them later. 'code_value' and 'message_value'
     #  are actual error code and message we expect this mock to return.
@@ -281,26 +277,23 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
     end
 
     alias_method :list_logs, :_undefined
+    alias_method :list_log_entries, :_undefined
     alias_method :list_log_services, :_undefined
     alias_method :list_log_service_indexes, :_undefined
+    alias_method :list_monitored_resource_descriptors, :_undefined
     alias_method :delete_log, :_undefined
     undef_method :_undefined
   end
 
   # Set up grpc stubs to mock the external calls.
-  def setup_logging_stubs(override_stub_params = nil, should_fail = false,
-                          code = 0, message = 'Ok')
-    stub_params = override_stub_params || \
-                  [COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
-                   CONTAINER_FROM_METADATA_PARAMS, CLOUDFUNCTIONS_PARAMS,
-                   CUSTOM_PARAMS, EC2_PARAMS]
+  def setup_logging_stubs(should_fail = false, code = 0, message = 'Ok')
     srv = GRPC::RpcServer.new
     @failed_attempts = []
     @requests_sent = []
     if should_fail
       grpc = GRPCLoggingMockFailingService.new(code, message, @failed_attempts)
     else
-      grpc = GRPCLoggingMockService.new(stub_params, @requests_sent)
+      grpc = GRPCLoggingMockService.new(@requests_sent)
     end
     srv.handle(grpc)
     srv.add_http2_port(GRPC_MOCK_HOST, :this_port_is_insecure)
@@ -320,8 +313,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
   # Verify the number and the content of the log entries match the expectation.
   # The caller can optionally provide a block which is called for each entry.
   def verify_log_entries(n, params, payload_type = 'textPayload', &block)
-    @requests_sent.each do |batch|
-      @logs_sent << JSON.parse(batch.to_json)
+    @requests_sent.each do |request|
+      @logs_sent << JSON.parse(request.to_json)
     end
     verify_json_log_entries(n, params, payload_type, &block)
   end
@@ -349,14 +342,14 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
 
   # Unset the 'referer' field.
   def http_request_message_with_absent_referer
-    http_request_message.reject do |k, _|
+    HTTP_REQUEST_MESSAGE.reject do |k, _|
       k == 'referer'
     end
   end
 
-  # Get the fields of the struct payload.
-  def get_fields(struct_payload)
-    struct_payload['fields']
+  # Get the fields of the payload.
+  def get_fields(payload)
+    payload['fields']
   end
 
   # Get the value of a struct field.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.5.6
+  version: 0.6.0.v2.alpha.1
 platform: ruby
 authors:
 - Todd Derr
@@ -53,6 +53,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: google-cloud-logging
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.23.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.23.2
 - !ruby/object:Gem::Dependency
   name: googleauth
   requirement: !ruby/object:Gem::Requirement
@@ -184,11 +198,6 @@ files:
 - Rakefile
 - fluent-plugin-google-cloud.gemspec
 - lib/fluent/plugin/out_google_cloud.rb
-- lib/google/logging/type/http_request_pb.rb
-- lib/google/logging/type/log_severity_pb.rb
-- lib/google/logging/v1/log_entry_pb.rb
-- lib/google/logging/v1/logging_pb.rb
-- lib/google/logging/v1/logging_services_pb.rb
 - test/helper.rb
 - test/plugin/base_test.rb
 - test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
@@ -212,12 +221,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: fluentd output plugin for the Stackdriver Logging API

data/lib/google/logging/type/http_request_pb.rb

@@ -1,30 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: google/logging/type/http_request.proto
-
-require 'google/protobuf'
-
-Google::Protobuf::DescriptorPool.generated_pool.build do
-  add_message "google.logging.type.HttpRequest" do
-    optional :request_method, :string, 1
-    optional :request_url, :string, 2
-    optional :request_size, :int64, 3
-    optional :status, :int32, 4
-    optional :response_size, :int64, 5
-    optional :user_agent, :string, 6
-    optional :remote_ip, :string, 7
-    optional :server_ip, :string, 13
-    optional :referer, :string, 8
-    optional :cache_lookup, :bool, 11
-    optional :cache_hit, :bool, 9
-    optional :cache_validated_with_origin_server, :bool, 10
-    optional :cache_fill_bytes, :int64, 12
-  end
-end
-
-module Google
-  module Logging
-    module Type
-      HttpRequest = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.logging.type.HttpRequest").msgclass
-    end
-  end
-end

data/lib/google/logging/type/log_severity_pb.rb

@@ -1,26 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: google/logging/type/log_severity.proto
-
-require 'google/protobuf'
-
-Google::Protobuf::DescriptorPool.generated_pool.build do
-  add_enum "google.logging.type.LogSeverity" do
-    value :DEFAULT, 0
-    value :DEBUG, 100
-    value :INFO, 200
-    value :NOTICE, 300
-    value :WARNING, 400
-    value :ERROR, 500
-    value :CRITICAL, 600
-    value :ALERT, 700
-    value :EMERGENCY, 800
-  end
-end
-
-module Google
-  module Logging
-    module Type
-      LogSeverity = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.logging.type.LogSeverity").enummodule
-    end
-  end
-end

data/lib/google/logging/v1/log_entry_pb.rb

@@ -1,52 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: google/logging/v1/log_entry.proto
-
-require 'google/protobuf'
-
-require 'google/logging/type/http_request_pb'
-require 'google/logging/type/log_severity_pb'
-require 'google/protobuf/any_pb'
-require 'google/protobuf/struct_pb'
-require 'google/protobuf/timestamp_pb'
-Google::Protobuf::DescriptorPool.generated_pool.build do
-  add_message "google.logging.v1.LogEntry" do
-    optional :metadata, :message, 1, "google.logging.v1.LogEntryMetadata"
-    optional :insert_id, :string, 4
-    optional :log, :string, 5
-    optional :http_request, :message, 7, "google.logging.type.HttpRequest"
-    optional :operation, :message, 8, "google.logging.v1.LogEntryOperation"
-    optional :writer_email_address, :string, 9
-    oneof :payload do
-      optional :proto_payload, :message, 2, "google.protobuf.Any"
-      optional :text_payload, :string, 3
-      optional :struct_payload, :message, 6, "google.protobuf.Struct"
-    end
-  end
-  add_message "google.logging.v1.LogEntryMetadata" do
-    optional :timestamp, :message, 9, "google.protobuf.Timestamp"
-    optional :severity, :enum, 2, "google.logging.type.LogSeverity"
-    optional :project_id, :string, 3
-    optional :project_number, :int64, 10
-    optional :service_name, :string, 4
-    optional :region, :string, 5
-    optional :zone, :string, 6
-    optional :user_id, :string, 7
-    map :labels, :string, :string, 8
-  end
-  add_message "google.logging.v1.LogEntryOperation" do
-    optional :id, :string, 1
-    optional :producer, :string, 2
-    optional :first, :bool, 3
-    optional :last, :bool, 4
-  end
-end
-
-module Google
-  module Logging
-    module V1
-      LogEntry = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.logging.v1.LogEntry").msgclass
-      LogEntryMetadata = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.logging.v1.LogEntryMetadata").msgclass
-      LogEntryOperation = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.logging.v1.LogEntryOperation").msgclass
-    end
-  end
-end