logstash-output-monasca_log_api 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 375ed1a143e91cddf640eada2f64c431d6957e8b
-  data.tar.gz: 9347e47c599a7c364e18025dc0968b5eaae5769e
+  metadata.gz: 6e01dc0526076f71ea01ecd92d2040f3695ee006
+  data.tar.gz: fcd4afa2854382dd30cb63b4c2d244eeb113132c
 SHA512:
-  metadata.gz: 10dae2e9ac05afc48211da532a5090ed9b28f1cacb2f2a1c2590edc343f50d1aa472d20955285ceaef67ea36de1d9214fe81dc17e09af7c37378896b47b3e6a9
-  data.tar.gz: 840994418f26066851bf9d96965b51f0d3ff857d1688492a6402a368707b2289eff76a9fe8c0d21a5793b0e0d9ac81eaaa984620d28f639f5a035594c359bf93
+  metadata.gz: 6f56d1f2a6fc39105ec0bc18ffcfd66a259cc2df32017ecae5d2bb140549b19f97ff6fcff42b030e0947320b64e43d0c4fc4d1187855ef63a22dc4bb782fa75f
+  data.tar.gz: fc0ca2aeccbbbf9e158f42f48ad71ceb6669d1306a17be4f3a3c10900d5d31f3f513cb03286bf661bbe00005b9abaf503271ec07af675beea6169cdb29e307b4

data/README.md

@@ -224,6 +224,11 @@ input {
 
 **Important:** The user which is sending the data on behalf of other tenants requires a specific role. This role is defined in the configuration file of the monasca-log-api as ```delegate_roles```.
 
+#### Exception handling
+
+This output plugin will sleep for maximum 60 seconds after the request to
+Monasca API fails and will retry the request until successful.
+
 ## Open tasks
 * Language translations (Replace hardcoded String messages with a configuration/language file)
-* Exception handling (monasca-api requests)
+* Unit tests for failed requests handling

data/lib/logstash/outputs/monasca_log_api.rb

@@ -209,6 +209,7 @@ class LogStash::Outputs::MonascaLogApi < LogStash::Outputs::Base
 
   def stop_time_check
     #ensure that entries buffered in queue will be  handled before stop
+    @stopping = true
     @mutex.synchronize do
       send_logs
     end
@@ -226,6 +227,7 @@ class LogStash::Outputs::MonascaLogApi < LogStash::Outputs::Base
       token = LogStash::Outputs::Keystone::Token.instance
       @logger.debug("Sending #{@logs[JSON_LOGS].size} logs")
       retry_tries = 5
+      attempt = 0
       begin
         tries ||= retry_tries
         @monasca_log_api_client.send_logs(@logs, token.id, @cross_tenant)
@@ -241,10 +243,19 @@ class LogStash::Outputs::MonascaLogApi < LogStash::Outputs::Base
         else
           @logger.error("Unauthorized: #{e}. Requesting new token failed "\
                         "after #{retry_tries} retries.")
+
+          sleep_for_retry(60)
+          retry
         end
       rescue => e
-        @logger.error('Sending event to monasca-log-api threw exception',
+        attempt += 1
+        sleep_for = get_sleep_time(attempt)
+        @logger.error("Sending event to monasca-log-api threw exception, "\
+                      "will sleep for #{sleep_for} seconds.",
                       :exceptionew => e)
+
+        sleep_for_retry(sleep_for)
+        retry
       end
       @logs.clear
       initialize_logs_object
@@ -270,4 +281,13 @@ class LogStash::Outputs::MonascaLogApi < LogStash::Outputs::Base
       raise LogStash::ConfigurationError, err
     end
   end
+
+  def get_sleep_time(attempt)
+    sleep_for = attempt**2
+    sleep_for <= 60 ? sleep_for : 60
+  end
+
+  def sleep_for_retry(duration)
+    Stud.stoppable_sleep(duration) { @stopping }
+  end
 end

data/logstash-output-monasca_log_api.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-monasca_log_api'
-  s.version         = '1.0.3'
+  s.version         = '1.0.4'
   s.licenses = ['Apache-2.0']
   s.summary = 'This gem is a logstash output plugin to connect via http to monasca-log-api.'
   s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
@@ -22,10 +22,11 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-core', '~> 2.0'
   s.add_runtime_dependency 'logstash-codec-plain', '~> 2.0'
   s.add_runtime_dependency 'logstash-codec-json', '~> 2.0'
+  s.add_runtime_dependency 'stud', '~> 0.0.22'
   s.add_development_dependency 'logstash-devutils', '~> 0.0.14'
   s.add_development_dependency 'simplecov', '~> 0.10'
   s.add_development_dependency 'simplecov-rcov', '~> 0.2.0'
   s.add_development_dependency 'rspec_junit_formatter', '~> 0.2.3'
-  s.add_development_dependency 'rubocop', '~> 0.37.2'
+  s.add_development_dependency 'rubocop', ">= 0.60.0"
   s.add_development_dependency 'webmock', '~> 2.0'
 end

data/spec/outputs/monasca_log_api_spec.rb

@@ -422,23 +422,6 @@ describe 'outputs/monasca_log_api' do
     end
   end
 
-  context 'when sending logs raise exception' do
-    it 'logs a failure' do
-      expect_any_instance_of(Cabin::Channel).to receive(:error)
-
-      monasca_log_api = LogStash::Outputs::MonascaLogApi.new(complete_config)
-      expect_any_instance_of(LogStash::Outputs::Monasca::MonascaLogApiClient)
-        .to receive(:send_logs)
-        .and_raise(Errno::ETIMEDOUT)
-      expect_any_instance_of(LogStash::Outputs::Keystone::KeystoneClient)
-        .to receive(:authenticate).and_return(valid_token)
-
-      monasca_log_api.register
-      monasca_log_api.multi_receive([event, event])
-      expect { monasca_log_api.multi_receive([event]) }.to_not raise_error
-    end
-  end
-
   context 'after sending logs' do
     it 'clears collected logs' do
       expect_any_instance_of(LogStash::Outputs::Monasca::MonascaLogApiClient)
@@ -480,42 +463,6 @@ describe 'outputs/monasca_log_api' do
       monasca_log_api.multi_receive([event, event, event])
       expect(LogStash::Outputs::Keystone::Token.instance.id).to eq(token_id)
     end
-
-    it 'if unauthorized, renew it' do
-      stub_request(:post, monasca_log_api_url_post)
-        .with(:headers =>
-        {
-          'Accept' => '*/*',
-          'Content-Type' => 'application/json',
-          'User-Agent' => 'Ruby',
-          'X-Auth-Token' => 'f8cdafb7dce94444ad781a53ddaff693'
-        })
-        .to_return(:status => 401)
-
-      expect_any_instance_of(LogStash::Outputs::Monasca::MonascaLogApiClient)
-        .to receive(:handle_response).exactly(5).times
-        .and_raise(
-          LogStash::Outputs::Monasca::MonascaLogApiClient::InvalidTokenError
-        )
-
-      expect_any_instance_of(LogStash::Outputs::Keystone::KeystoneClient)
-        .to receive(:authenticate).exactly(6).times
-        .with(
-          complete_config['username'],
-          complete_config['user_domain_name'],
-          complete_config['password'],
-          complete_config['project_name'],
-          complete_config['project_domain_name']
-        )
-        .and_return(expired_token, valid_token)
-
-      monasca_log_api = LogStash::Outputs::MonascaLogApi.new(complete_config)
-      allow(monasca_log_api).to receive(:start_time_check)
-      monasca_log_api.register
-      expect(LogStash::Outputs::Keystone::Token.instance.id).to eq(old_token_id)
-      monasca_log_api.multi_receive([event, event, event])
-      expect(LogStash::Outputs::Keystone::Token.instance.id).to eq(token_id)
-    end
   end
 
   context 'when stopping plugin' do
@@ -528,4 +475,6 @@ describe 'outputs/monasca_log_api' do
     end
   end
 
+  # TODO add testing graceful recovery from exceptions
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-monasca_log_api
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Fujitsu Enabling Software Technology GmbH
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-11-20 00:00:00.000000000 Z
+date: 2018-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  name: stud
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.22
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -111,17 +125,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.37.2
+        version: 0.60.0
   name: rubocop
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.37.2
+        version: 0.60.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: