logstash-output-azure_service_bus 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af1daf5ab0f287bfe29993a09437eb840063c5195d5dc9c247985fae757a09bd
-  data.tar.gz: afb35cc44123159129fc63fe5fcb374c7cb77ac98137b3b2ae0e513e96b12f09
+  metadata.gz: 258ba5d4370f9b1553d0742de66e8bb10cc2e685fb451591b02e773e329be171
+  data.tar.gz: ce24ff104251bf82c1ffc8107364509e9b06eb5f18e96187b22dcaec80ff71d6
 SHA512:
-  metadata.gz: 3727d297acacbee56d2570ac67aca3e05bcc2f677b6ab5ab50c3acfa84aa721c683be70810fdcc26203124d615cbd99e6bb19fc5f1d68d3eccefd859e288d0e9
-  data.tar.gz: a72b4442be6118a0b933b2d3e9b6736a282e3024f32f00dd9bb4506b6096c02a824777ebe7535340d53eb9c152121b8e9bf2bf70c00d9d5b05b9ccb67307306e
+  metadata.gz: fa154761acffb67e497e6705174782d4acc59c72ab702f1b8e1eab08123eaf3530d3aa2cf75cad4a5c5f520b16ee26b0c1433a4290d50e190467151007e3d9c2
+  data.tar.gz: dee42c681c9d8fefcad82be870770d61efd7308ec5a49c91f77c57fddb9e9cf6fdfe89e6b73ad8f306ec6039ffd138b33769cbfad848d9ee1812ad051621aa9f

data/README.md

@@ -7,7 +7,7 @@ This plugin is hosted on RubyGems.org: [https://rubygems.org/gems/logstash-outpu
 ## Install
 To install, use the plugin tool that is part of your Logstash installation:
 ```
-$LOGSTASH_INSTALL/bin/logstash-plugin install logstash-output-azure_service_bus
+$LOGSTASH_PATH/bin/logstash-plugin install logstash-output-azure_service_bus
 ```
 
 ## Pipeline Configuration
@@ -27,17 +27,21 @@ There is one optional setting (`messageid_field`) which sets the Service Bus `Me
 input { ... }
 filter {
     uuid {
-        target => "uuid"
+        target => "[@metadata][uuid]"
     }
 }
 output {
     azure_service_bus {
         service_bus_namespace => "service-bus-name"
         service_bus_entity => "queue-or-topic-name"
-        messageid_field => "uuid"
+        messageid_field => "[@metadata][uuid]"
+    }
+    elasticsearch {
+        ...
+        document_id => "%{[@metadata][uuid]}"
     }
 }
 ```
 
-## Service Bus Configuration
-This plugin will retry sending messages if the Service Bus connection times out or returns a bad response. To avoid idempotence issues, you should enable duplicate detection on the destination queue or topic.
+## Retry
+This plugin will retry sending messages _indefinitely_ if Service Bus times out or returns a [documented bad response](https://docs.microsoft.com/en-us/rest/api/servicebus/send-message-batch#response-codes) (except 400). To avoid idempotence issues, you should enable duplicate detection on the destination queue or topic.

data/lib/logstash/outputs/azure_service_bus.rb

@@ -13,30 +13,33 @@ class LogStash::Outputs::AzureServiceBus < LogStash::Outputs::Base
   config :messageid_field, :validate => :string
 
   def register
-    retry_options = {
-      max: 3,
+    service_bus_retry_options = {
+      max: Float::MAX, # Essentially retries indefinitely
       interval: 1,
       interval_randomness: 0.5,
       backoff_factor: 2,
-      retry_statuses: [429, 500],
       exceptions: [Faraday::ConnectionFailed, Faraday::TimeoutError, Faraday::RetriableResponse],
-      methods: %i[get post],
-      retry_block: ->(env, _options, retries, exception) { @logger.warn("Error (#{exception}) for #{env.method.upcase} #{env.url} - #{retries + 1} retry(s) left") }
+      methods: [], # Empty -> all methods
+      retry_statuses: [401, 403, 404, 410, 429, 500], # https://docs.microsoft.com/en-us/rest/api/servicebus/send-message-batch#response-codes
+      retry_block: lambda do |env, _options, _retries, exception|
+        if env.status.nil?
+          @logger.warn("Problem (#{exception}) for #{env.method.upcase} #{env.url}")
+        else
+          @logger.warn("Problem (HTTP #{env.status}) for #{env.method.upcase} #{env.url}")
+        end
+      end,
+      retry_if: lambda do |env, _exc|
+        refresh_access_token if env.status == 401
+        true # Always retry
+      end
     }
-    @token_conn = Faraday.new(
-      url: 'http://169.254.169.254/metadata/identity/oauth2/token',
-      params: { 'api-version' => '2018-02-01', 'resource' => 'https://servicebus.azure.net/' },
-      headers: { 'Metadata' => 'true' },
-      request: { timeout: 4 }
-    ) do |f|
-      f.request :retry, retry_options
-    end
     @service_bus_conn = Faraday.new(
       url: "https://#{@service_bus_namespace}.servicebus.windows.net/#{@service_bus_entity}/",
       request: { timeout: 10 }
-    ) do |f|
-      f.request :retry, retry_options
+    ) do |conn|
+      conn.request :retry, service_bus_retry_options
     end
+    @access_token = ''
     refresh_access_token
   end
 
@@ -59,32 +62,37 @@ class LogStash::Outputs::AzureServiceBus < LogStash::Outputs::Base
   end
 
   def post_messages(messages)
-    refresh_access_token if access_token_needs_refresh?
     response = @service_bus_conn.post('messages') do |req|
       req.body = JSON.generate(messages)
       req.headers = { 'Authorization' => "Bearer #{@access_token}", 'Content-Type' => 'application/vnd.microsoft.servicebus.json' }
     end
-    raise "Error while sending message to Service Bus: HTTP #{response.status}" if response.status != 201
-
-    @logger.debug("Sent #{messages.length} message(s) to Service Bus")
-  end
-
-  def access_token_needs_refresh?
-    Time.now.to_i - 60 > @access_token_expiration # Refresh the access token if it will expire within 60 seconds.
+  rescue StandardError => e
+    # Hopefully we never make it here and "throw away" messages since we have an agressive retry strategy.
+    @logger.error("Error (#{e}) while sending message to Service Bus")
+  else
+    if response.status == 200
+      @logger.debug("Sent #{messages.length} message(s) to Service Bus")
+    else
+      @logger.error("Error while sending message to Service Bus: HTTP #{response.status}")
+    end
   end
 
   def refresh_access_token
     @logger.info('Refreshing Azure access token')
     begin
-      response = @token_conn.get
-    rescue Faraday::ConnectionFailed => e
-      @logger.error('Unable to connect to the Azure Instance Metadata Service')
-      raise e
+      response = Faraday.get('http://169.254.169.254/metadata/identity/oauth2/token', params: { 'api-version' => '2018-02-01', 'resource' => 'https://servicebus.azure.net/' }) do |req|
+        req.headers = { 'Metadata' => 'true' }
+        req.options.timeout = 4
+      end
+    rescue StandardError => e # We just catch everything and move on since @service_bus_conn will handle retries.
+      @logger.error("Error while fetching access token: #{e}")
+    else
+      if response.status == 200
+        data = JSON.parse(response.body)
+        @access_token = data['access_token']
+      else
+        @logger.error("HTTP error when fetching access token: #{response.body}")
+      end
     end
-    raise "Unable to fetch token: #{response.body}" if response.status != 200
-
-    data = JSON.parse(response.body)
-    @access_token = data['access_token']
-    @access_token_expiration = data['expires_on'].to_i
   end
 end

data/logstash-output-azure_service_bus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-azure_service_bus'
-  s.version       = '0.2.1'
+  s.version       = '0.3.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Send Logstash messages to Azure Service Bus.'
   s.homepage      = 'https://github.com/gharryg/logstash-output-azure_service_bus'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-azure_service_bus
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Harrison Golden
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-06 00:00:00.000000000 Z
+date: 2022-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement