logstash-output-application_insights 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2992e3a202401238e6ce64bb81b14e36d215a13b
-  data.tar.gz: ef6fd4ac9fce577765515ab7fb35f964101b2275
+  metadata.gz: 12c164429fdbd439168dab5277c28185ce2e4985
+  data.tar.gz: 44048dcef57a0bbe542eb8a3b68811f968eb8352
 SHA512:
-  metadata.gz: fccb9334a9591040d6bc9255a8e620517cbb2aa6e1907bd8167eb3235c829b8702c742f31faedb82088c03ac8dbec2c7d2a96ff96e4aa1234754fa462a6b7158
-  data.tar.gz: 066f4e308ab6df69ddc1ce574f987fc82d8a9dcdef9e72dbdd2bf7680d892908f44ba04c0f53323751755114be6fb4669538324ad62bf0567dca6d1834fdf0ce
+  metadata.gz: 8388e00a758efb2fb30215ace69730d28690b573c66ee901be7dd635f16660b5a2236ef0f1bb4c0665924b54e8a3150ea868ce8c1771f42409af3cd5ed251ec3
+  data.tar.gz: cc27c111592a9bd9aca93fb8f697219eb58698ae7691222b43ff98313ce1cd365263feb9fd19fbfbcaa07cca98a9a114c2e0eb3b1a25319b151d7f789f515880

data/README.md

@@ -1,9 +1,12 @@
 # Microsoft Application Insights Output Plugin for Logstash
-
 [![GitHub version](https://badge.fury.io/gh/microsoft%2Flogstash-output-application-insights.svg)](https://badge.fury.io/gh/microsoft%2Flogstash-output-application-insights)
 [![Gem Version](https://badge.fury.io/rb/logstash-output-application_insights.svg)](https://badge.fury.io/rb/logstash-output-application_insights)
+ 
+* This project is a plugin for [Logstash](https://github.com/elastic/logstash). 
+
+* This plugin have to be installed on top of the Logstash core pipeline. It is not a stand-alone program.
 
-This project is a plugin for [Logstash](https://github.com/elastic/logstash).
+* **This plugin outputs events to Microsoft Application Insights Analytics open schema tables.**
 
 # Plugin Features
 
@@ -49,7 +52,7 @@ filter {
 }
 output {
   application_insights {
-    intrumentation_key => "5a6714a3-ec7b-4999-ab96-232f1da92059"
+    instrumentation_key => "5a6714a3-ec7b-4999-ab96-232f1da92059"
     table_id => "c24394e1-f077-420e-8a25-ef6fdf045938"
     storage_account_name_key => [ "my-storage-account", "pfrYTwPgKyYNfKBY2QdF+v5sbgx8/eAQp+FFkGpPBnkMDE1k+ZNK3r3qIPqqw8UsOIUqaF3dXBdPDouGJuxNXQ==" ]
   }
@@ -147,12 +150,12 @@ example:
 azure_storage_blob_prefix => "myprefix"
 ```
 
-### intrumentation_key
-Default Application Insights Analytics intrumentation_key. No default
+### instrumentation_key
+Default Application Insights Analytics instrumentation_key. No default
 It will be used only in case the key is not specified in the tables property associated to a table_id, or as field or metadata fields in the event
 example:
 ```ruby
-intrumentation_key => "5A6714A3-EC7B-4999-AB96-232F1DA92059"
+instrumentation_key => "5A6714A3-EC7B-4999-AB96-232F1DA92059"
 ```
 
 ### table_id
@@ -358,11 +361,11 @@ example:
 ```ruby
 delete_not_notified_blobs => true
 ```
-### validate_endpoint
+### validate_notification
 When set to true, access to application insights will be validated at initialization and if validation fail, logstash process will abort. Default false
 example:
 ```ruby
-validate_endpoint => true
+validate_notification => true
 ```
 
 ### validate_storage
@@ -404,22 +407,32 @@ example:
 stop_on_unknown_io_errors => true
 ```
 
-### notification_endpoint
-when set notification are sent to an alternative endpoint. Default "https://dc.services.visualstudio.com/v2/track"
-Used for troubleshooting
+### azure_storage_host_suffix
+when set an alternative storage service will be used. Default "core.windows.net"
 example:
 ```ruby
-stop_on_unknown_io_errors => true
+azure_storage_host_suffix => "core.windows.net"
+```
+
+### application_insights_endpoint
+when set blob ready notification are sent to an alternative endpoint. Default "https://dc.services.visualstudio.com/v2/track"
+example:
+```ruby
+application_insights_endpoint => "https://dc.services.visualstudio.com/v2/track"
 ```
 
 ### notification_version
 Advanced, internal, should not be set, the only current valid value is 1
+example:
+```ruby
+notification_version => 1
+```
 
 ### tables
 Allow to support multiple tables, and to configure each table with its own parameters, using the global parameters as defaults.
 It is only required if the plugin need to support mutiple table.
 Tables is Hash, where the key is the table_id and the value is a has of specific properties, that their defualt value are the global properties.
-The specific properties are: intrumentation_key, table_columns, blob_max_delay, csv_default_value, serialized_event_field, blob_serialization, csv_separator
+The specific properties are: instrumentation_key, table_columns, blob_max_delay, csv_default_value, serialized_event_field, blob_serialization, csv_separator
 template:
 ```ruby
 tables => { "table_id1" => { properties } "table_id2" => { properties } }
@@ -427,12 +440,12 @@ tables => { "table_id1" => { properties } "table_id2" => { properties } }
 
 Examples:
 ```ruby
-tables => { "6f29a89e-1385-4317-85af-3ac1cea48058" => { "intrumentation_key" => "76c3b8e9-dfc6-4afd-8d4c-3b02fdadb19f", "blob_max_delay" => 60 } }
+tables => { "6f29a89e-1385-4317-85af-3ac1cea48058" => { "instrumentation_key" => "76c3b8e9-dfc6-4afd-8d4c-3b02fdadb19f", "blob_max_delay" => 60 } }
 ```
 
 ```ruby
-tables => { "6f29a89e-1385-4317-85af-3ac1cea48058" => { "intrumentation_key" => "76c3b8e9-dfc6-4afd-8d4c-3b02fdadb19f", "blob_max_delay" => 60 }
-            "2e1b46aa-56d2-4e13-a742-d0db516d66fc" => { "intrumentation_key" => "76c3b8e9-dfc6-4afd-8d4c-3b02fdadb19f", "blob_max_delay" => 120 "ext" => "csv" "serialized_event_field" => "message" } 
+tables => { "6f29a89e-1385-4317-85af-3ac1cea48058" => { "instrumentation_key" => "76c3b8e9-dfc6-4afd-8d4c-3b02fdadb19f", "blob_max_delay" => 60 }
+            "2e1b46aa-56d2-4e13-a742-d0db516d66fc" => { "instrumentation_key" => "76c3b8e9-dfc6-4afd-8d4c-3b02fdadb19f", "blob_max_delay" => 120 "ext" => "csv" "serialized_event_field" => "message" } 
           }
 ```
 

data/lib/logstash/outputs/application_insights.rb

@@ -54,6 +54,12 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
   require "logstash/outputs/application_insights/blob" 
   autoload :Block, "logstash/outputs/application_insights/block"
   autoload :Storage_cleanup, "logstash/outputs/application_insights/storage_cleanup"
+  autoload :Shutdown_recovery, "logstash/outputs/application_insights/shutdown_recovery"
+  autoload :Storage_recovery, "logstash/outputs/application_insights/storage_recovery"
+  autoload :Notification_recovery, "logstash/outputs/application_insights/notification_recovery"
+  autoload :Validate_storage, "logstash/outputs/application_insights/validate_storage"
+  autoload :Validate_notification, "logstash/outputs/application_insights/validate_notification"
+  
 
   autoload :Clients, "logstash/outputs/application_insights/clients" 
   autoload :Client, "logstash/outputs/application_insights/client" 
@@ -101,10 +107,10 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
   # string may include only characters that are allowed in any valid url
   config :azure_storage_blob_prefix, :validate => :string
   
-  # Default Application Insights Analytics intrumentation_key
+  # Default Application Insights Analytics instrumentation_key
   # will be used only in case it is not specified as a table_id property in tables
   # or as part of the event's fields or event's metadata fields
-  config :intrumentation_key, :validate => :string
+  config :instrumentation_key, :validate => :string
 
   # Default Application Insights Analytics table_id
   # will be used only in case it is not specified as part o
@@ -121,8 +127,8 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
   # A hash of table_ids, where each table_id points to a set of properties
   # the properties are a hash, where the keys are are the properties
   # current supported properties per table_id are:
-  # intrumentation_key, ext, table_columns, csv_default_value, csv_separator, blob_max_delay, event_separator, serialized_event_field
-  # intrumentation_key, Application Insights Analytics intrumentation_key, will be used in case not specified in any of the event's fields or events's metadata fileds
+  # instrumentation_key, ext, table_columns, csv_default_value, csv_separator, blob_max_delay, event_separator, serialized_event_field
+  # instrumentation_key, Application Insights Analytics instrumentation_key, will be used in case not specified in any of the event's fields or events's metadata fileds
   # serialized_event_field, specifies the field that may contain the full serialized event (either as json or csv), 
   #             when specified, the ext property should be set either to csv or to json (json is the default)
   #             if event. does not conatin the field, value will be created based on the fileds in the evnt, according to table_columns if configured, or all fileds in event
@@ -150,7 +156,7 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
   #                can be specified only together with table_columns
   #
   # Example json table_id
-  #   tables => {"a679fbd2-702c-4c46-8548-80082c66ef28" => {"intrumentation_key" => "abee940b-e648-4242-b6b3-f2826667bf96", "blob_max_delay" => 60} }
+  #   tables => {"a679fbd2-702c-4c46-8548-80082c66ef28" => {"instrumentation_key" => "abee940b-e648-4242-b6b3-f2826667bf96", "blob_max_delay" => 60} }
   # Example json table_id, input in serialized_event_field
   #   {"ab6a3584-aef0-4a82-8725-2f2336e59f3e" => {"serialized_event_field" => "message". "ext" => "json"} }
   # Example csv table_id, input in serialized_event_field
@@ -271,25 +277,30 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
   # When set to true, process will stop if an unknown IO error is found
   config :stop_on_unknown_io_errors, :validate => :boolean
 
-  # Advanced, internal, should not be set, the default is Application Insights production endpoint
   # when set notification are sent to an alternative endpoint, used for internal testing
-  config :notification_endpoint, :validate => :string
+  config :application_insights_endpoint, :validate => :string
+
+  # when set an alternative storage service will be used. 
+  config :azure_storage_host_suffix, :validate => :string
 
   # Advanced, internal, should not be set, the only current valid value is 1
   config :notification_version, :validate => :number
 
   # When set to true, access to application insights will be validated at initialization
   # and if validation fail, logstash process will abort
-  config :validate_endpoint, :validate => :boolean, :default => true
+  config :validate_notification, :validate => :boolean
 
   # When set to true, access to azure storage for each of the configured accounts will be validated at initialization
   # and if validation fail, logstash process will abort
-  config :validate_storage, :validate => :boolean, :default => true
+  config :validate_storage, :validate => :boolean
 
   public
 
   def register
 
+    # logstash define: @original_params = original_params
+    # logstash define: @config = params
+
     # set configuration
     Config.validate_and_adjust_configuration( default_configuration )
     configuration = Config.current
@@ -305,9 +316,27 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
     configuration[:telemetry_channel] = @telemetry.telemetry_channel
 
     Timer.config( configuration )
-    Blob.config( configuration )
-    Blob.validate_endpoint if @validate_endpoint
-    Blob.validate_storage if @validate_storage
+
+    if @validate_notification
+      status = Validate_notification.new.validate
+      raise ConfigurationError, "Failed to access application insights at #{configuration[:application_insights_endpoint]}, due to error #{status[:error].inspect}" unless status[:success]
+    end
+
+    if @validate_storage
+      result = Validate_storage.new.validate
+      result.each do |storage_account_name, status|
+        raise ConfigurationError, "Failed access azure storage account #{storage_account_name}, due to error #{status[:error].inspect}" unless status[:success]
+      end
+    end
+
+    @notification_recovery = Notification_recovery.instance
+    @notification_recovery.start
+
+    @storage_recovery = Storage_recovery.instance
+    @storage_recovery.start
+
+    @shutdown_recovery = Shutdown_recovery.instance
+    @shutdown_recovery.start
 
     @shutdown = Shutdown.instance
     @channels = Channels.instance
@@ -320,7 +349,7 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
     #   @channels.receive( event, encoded_event )
     # end
 
-    Telemetry.instance.track_event("register", {:properties => configuration})
+    @telemetry.track_event("register", {:properties => configuration})
 
 
     return "ok\n"
@@ -334,60 +363,16 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
   end
 
   def close
-    Telemetry.instance.track_event( "close" )
-    Telemetry.instance.flush
+    @telemetry.track_event( "close" )
+    @telemetry.flush
+    @shutdown_recovery.close
+    @storage_recovery.close
+    @notification_recovery.close
     @shutdown.submit
   end
 
   private
 
   # -----------------------------------------------
-
-
-  def list_blob_names
-    blob_names = Set.new []
-    loop do
-      continuation_token = NIL
-      entries = @azure_blob.list_blobs(@container, { :timeout => 10, :marker => continuation_token})
-      @@logger.debug { 'blob entries: #{entries}' }
-      entries.each do |entry|
-        @@logger.debug { 'blob entry name: #{entry.name}' }
-        blob_names << entry.name
-      end
-      continuation_token = entries.continuation_token
-      break if continuation_token.empty?
-    end
-    return blob_names
-  end # def list_blobs
-
-
-  def list_container_names
-    container_names = Set.new []
-    loop do
-      continuation_token = NIL
-      containers = @azure_blob.list_containers()
-      @@logger.debug { 'containers: #{containers}' }
-      containers.each do |container|
-        @@logger.debug { 'container entry name:' + container.name }
-        container_names << container.name
-        upload(container.name, "blob-append-" + container.name, "test - " + container.name)
-        blobs = @azure_blob.list_blobs(container.name)
-        blobs.each do |blob|
-          @@logger.debug { 'blob name: ' + blob.name }
-        end
-      end
-      continuation_token = containers.continuation_token
-      break if continuation_token.empty?
-    end
-    return container_names
-  end # def list_blobs
-
-  def create_container (container_name)
-    begin
-      @azure_blob.create_container(container_name)
-    rescue
-      @@logger.debug { $! }
-    end
-  end
 end
 

data/lib/logstash/outputs/application_insights/blob.rb

@@ -21,7 +21,7 @@
 class LogStash::Outputs::Application_insights
   class Blob
 
-    attr_reader :intrumentation_key
+    attr_reader :instrumentation_key
     attr_reader :table_id
     attr_reader :storage_account_name
     attr_reader :container_name
@@ -36,37 +36,7 @@ class LogStash::Outputs::Application_insights
 
     public
 
-    def self.config ( configuration )
-      @@configuration = configuration
-
-      @@logger = configuration[:logger]
-      @@io_retry_delay = configuration[:io_retry_delay]
-      @@io_max_retries = configuration[:io_max_retries]
-      @@blob_max_bytesize = configuration[:blob_max_bytesize]
-      @@blob_max_events = configuration[:blob_max_events]
-      @@state_table_name = "#{configuration[:azure_storage_table_prefix]}#{STATE_TABLE_NAME}"
-      @@save_notified_blobs_records = configuration[:save_notified_blobs_records]
-
-      @@closing = false
-
-      # queues, per storage_account_name, for failed blob commit, will continue to try resending
-      @@failed_on_commit_retry_Qs = {}
-      launch_storage_recovery_threads( @@failed_on_commit_retry_Qs, :commit, :io_failure )
-      launch_storage_recovery_table_threads( :uploading )
-
-      # queues, per storage_account_name, for failed notify, will continue to try resending
-      @@failed_on_notify_retry_Qs = {}
-      launch_storage_recovery_threads( @@failed_on_notify_retry_Qs, :notify, :notify_failed_blob_not_accessible )
-      launch_storage_recovery_table_threads( :committed )
-
-      # for failed to notify due to endpoint, will continue to try resending
-      launch_endpoint_recovery_thread
-
-      # queues, per storage_account_name, for failed to log to table, will continue to try resending
-      @@failed_on_log_to_table_retry_Qs = {}
-      launch_storage_recovery_threads( @@failed_on_log_to_table_retry_Qs, :log_to_table_update, :io_failure )
-
-    end
+    @@closing = false
 
     def self.close
       @@closing = true
@@ -76,160 +46,22 @@ class LogStash::Outputs::Application_insights
       @@closing
     end
 
-    def self.launch_endpoint_recovery_thread
-      @@failed_on_notification_endpoint_retry_Q = Queue.new
-      storage_recovery_thread( nil, @@failed_on_notification_endpoint_retry_Q, :notify, :io_failure )
-    end
-
-    def self.launch_storage_recovery_threads ( queues, method, failure_reason )
-      @@configuration[:storage_account_name_key].each do |storage_account_name, storage_account_keys|
-        queues[storage_account_name] = Queue.new
-        # a threads, per storage  account name
-        storage_recovery_thread( storage_account_name, queues[storage_account_name], method, failure_reason )
-      end
-    end
-
-    def self.launch_storage_recovery_table_threads ( state )
-      @@configuration[:storage_account_name_key].each do |storage_account_name, storage_account_keys|
-        recovery_table_thread( storage_account_name, state)
-      end
-    end
-
-    #return thread
-    def self.recovery_table_thread( storage_account_name, state )
-      Thread.new( storage_account_name, state ) do |storage_account_name, state|
-        
-        blob = Blob.new
-
-        committed_tuples = [  ]
-        uncommitted_tuples = [  ]
-        upload_empty_tuples = [  ]
-        token = nil
-        finished = false
-        filter = "#{:PartitionKey} eq '#{@@configuration[:azure_storage_blob_prefix]}-#{state}'"
-
-        # should exit thread after fetching data from table, and submit recovery, the loop is only for case of failure
-        until finished || stopped? do
-          entities = blob.log_to_table_query( storage_account_name, filter, token )
-          if entities
-            token = entities.continuation_token
-
-            if :committed == state
-              entities.each do |entity|
-                State.instance.inc_pending_notifications
-                tuple = blob.table_entity_to_tuple( entity.properties )
-                @@failed_on_notification_endpoint_retry_Q << tuple
-              end
-
-            elsif :uploading == state
-              # first tuples are collected, before send to queues, to make sure blob states don't change in between
-              entities.each do |entity|
-                typed_tuple = nil
-                until typed_tuple || stopped?
-                  typed_tuple = blob.update_commited_or_uncommited_list( entity.properties )
-                  Stud.stoppable_sleep(60, 1) { stopped? } unless typed_tuple
-                end
-                next if stopped?
-
-                if typed_tuple[:committed]
-                  committed_tuples << typed_tuple[:committed]
-                elsif typed_tuple[:uncommitted]
-                  uncommitted_tuples << typed_tuple[:uncommitted]
-                else
-                  upload_empty_tuples << typed_tuple[:upload_empty]
-                end
-              end
-            end
-
-            next if token
-            committed_tuples.each do |tuple|
-              State.instance.inc_pending_commits
-              @@failed_on_log_to_table_retry_Qs[storage_account_name] << tuple
-            end
-            uncommitted_tuples.each do |tuple|
-              State.instance.inc_pending_commits
-              @@failed_on_commit_retry_Qs[storage_account_name] << tuple
-            end
-            upload_empty_tuples.each do |tuple|
-              @@failed_on_log_to_table_retry_Qs[storage_account_name] << tuple
-            end
-            finished = true
-          else
-            Stud.stoppable_sleep(60, 1) { stopped? }
-          end
-        end
-        @@logger.info { "exit table recovery thread, storage: #{storage_account_name}, state: #{state}, entities: #{entities ? entities.length : nil}" }
-      end
-    end
-
-    def self.state_on? ( storage_account_name, blob, failure_reason )
-      if blob
-        if :io_failure == failure_reason
-          @@endpoint_state_on ||= blob.test_notification_endpoint( @@configuration[:storage_account_name_key][0][0] )
-        else
-          Clients.instance.storage_account_state_on?( storage_account_name )
-        end
-      elsif storage_account_name
-        Clients.instance.storage_account_state_on?( storage_account_name )
-      else
-        Clients.instance.storage_account_state_on?
-      end
-    end
-
-    def self.storage_recovery_thread( storage_account_name, queue, method, failure_reason )
-      # a threads, per storage  account name, that retries failed blob commits / notification / table updates
-      Thread.new( storage_account_name, queue, method, failure_reason ) do |storage_account_name, queue, method, failure_reason|
-        blob = Blob.new if :notify == method
-        semaphore = Mutex.new
-        action = {:method => method, :semaphore => semaphore, :counter => 0 }
-        loop do
-          tuple ||= queue.pop
-          until state_on?( storage_account_name, blob, failure_reason ) do sleep( 1 ) end
-
-          not_busy = nil
-          semaphore.synchronize {
-            not_busy = action[:counter] += 1 if 10 > action[:counter]
-          }
-          if not_busy
-            Thread.new( action, tuple ) do |action, tuple|
-              Blob.new.send( action[:method], tuple )
-              action[:semaphore].synchronize {
-                action[:counter] -= 1
-              }
-            end
-            tuple = nil # release for GC
-          else
-            Stud.stoppable_sleep(60, 1) { 10 > action[:counter] }
-            next
-          end 
-        end
-      end
-    end
-
-    def self.validate_endpoint
-      io = Blob.new
-      raise ConfigurationError, "Failed to access application insights #{@@configuration[:notification_endpoint]}, due to error #{io.last_io_exception.inspect}" unless io.test_notification_endpoint( @@configuration[:storage_account_name_key][0][0] )
-    end
-
-    def self.validate_storage
-      io = Blob.new
-      @@configuration[:storage_account_name_key].each do |storage_account_name, storage_account_keys|
-        raise ConfigurationError, "Failed access azure storage account #{storage_account_name}, due to error #{io.last_io_exception.inspect}" unless io.test_storage( storage_account_name )
-      end
-    end
-
-
     def initialize ( channel = nil, id = nil , no_queue = false )
+      @configuration = Config.current
+      @logger = @configuration[:logger]
+      @storage_recovery = Storage_recovery.instance
+      @notification_recovery = Notification_recovery.instance
+      @max_tries = @configuration[:io_max_retries] + 1
+
       @uploaded_block_ids = [  ]
       @uploaded_block_numbers = [  ]
       @uploaded_bytesize = 0
       @uploaded_events_count = 0
-      @max_tries = @@io_max_retries + 1
       @sub_state = :none
 
       if channel
         @id = id
-        @intrumentation_key = channel.intrumentation_key
+        @instrumentation_key = channel.instrumentation_key
         @table_id = channel.table_id
         @blob_max_delay = channel.blob_max_delay
 
@@ -273,14 +105,13 @@ class LogStash::Outputs::Application_insights
 
               unless to_commit
                 @timer.set( block_to_upload.oldest_event_time + @blob_max_delay, nil ) {|object| @io_queue << :wakeup if 0 == @io_queue.length } if blob_empty?
-                to_commit = :commit if blob_full?
-                upload( block_to_upload, to_commit)
+                upload( block_to_upload )
                 block_to_upload = nil # release reference to resource for GC
-              else
-                commit unless @uploaded_block_ids.empty?
+                to_commit = :commit if blob_full?
               end
 
               if to_commit
+                commit unless @uploaded_block_ids.empty?
                 to_commit = nil
                 @uploaded_block_ids = [  ]
                 @timer.cancel
@@ -302,9 +133,9 @@ class LogStash::Outputs::Application_insights
 
     def blob_full? ( next_block = nil )
       if next_block
-        BLOB_MAX_BLOCKS < @uploaded_block_ids.length + 1 || @@blob_max_events < @uploaded_events_count + next_block.events_count || @@blob_max_bytesize < @uploaded_bytesize  + next_block.bytesize
+        BLOB_MAX_BLOCKS < @uploaded_block_ids.length + 1 || @configuration[:blob_max_events] < @uploaded_events_count + next_block.events_count || @configuration[:blob_max_bytesize] < @uploaded_bytesize  + next_block.bytesize
       else
-        BLOB_MAX_BLOCKS <= @uploaded_block_ids.length || @@blob_max_events <= @uploaded_events_count || @@blob_max_bytesize <= @uploaded_bytesize
+        BLOB_MAX_BLOCKS <= @uploaded_block_ids.length || @configuration[:blob_max_events] <= @uploaded_events_count || @configuration[:blob_max_bytesize] <= @uploaded_bytesize
       end
     end 
 
@@ -334,7 +165,7 @@ class LogStash::Outputs::Application_insights
     end
 
     def table_entity_to_tuple( options = {} )
-      [ options[:start_time.to_s] || Time.now.utc, options[:action.to_s], options[:intrumentation_key.to_s], options[:table_id.to_s], 
+      [ options[:start_time.to_s] || Time.now.utc, options[:action.to_s], options[:instrumentation_key.to_s], options[:table_id.to_s], 
         options[:storage_account_name.to_s], options[:container_name.to_s], options[:blob_name.to_s], 
         eval( options[:uploaded_block_ids.to_s] ), eval( options[:uploaded_block_numbers.to_s] ), 
         options[:uploaded_events_count.to_s] || 0, options[:uploaded_bytesize.to_s] || 0, options[:oldest_event_time.to_s] || Time.now.utc,
@@ -344,7 +175,7 @@ class LogStash::Outputs::Application_insights
     end
 
     def state_to_tuple
-      [ @start_time || Time.now.utc, @action, @intrumentation_key, @table_id, 
+      [ @start_time || Time.now.utc, @action, @instrumentation_key, @table_id, 
         @storage_account_name, @container_name, @blob_name, 
         @uploaded_block_ids, @uploaded_block_numbers, 
         @uploaded_events_count, @uploaded_bytesize, @oldest_event_time,
@@ -354,7 +185,7 @@ class LogStash::Outputs::Application_insights
     end
 
     def tuple_to_state ( tuple )
-      ( @start_time, @action, @intrumentation_key, @table_id, 
+      ( @start_time, @action, @instrumentation_key, @table_id, 
         @storage_account_name, @container_name, @blob_name, 
         @uploaded_block_ids, @uploaded_block_numbers, 
         @uploaded_events_count, @uploaded_bytesize, @oldest_event_time,
@@ -363,7 +194,7 @@ class LogStash::Outputs::Application_insights
     end
 
     def state_to_table_entity
-      { :start_time => @start_time, :intrumentation_key => @intrumentation_key, :table_id => @table_id, 
+      { :start_time => @start_time, :instrumentation_key => @instrumentation_key, :table_id => @table_id, 
         :storage_account_name => @storage_account_name, :container_name => @container_name, :blob_name => @blob_name, 
         :uploaded_block_ids => @uploaded_block_ids.to_s, :uploaded_block_numbers => @uploaded_block_numbers.to_s, 
         :uploaded_events_count => @uploaded_events_count, :uploaded_bytesize => @uploaded_bytesize, :oldest_event_time => @oldest_event_time,
@@ -372,42 +203,32 @@ class LogStash::Outputs::Application_insights
     end
 
 
-    def test_storage_recover
-      proc do |reason, e| @recovery = :ok if :container_exist == reason || :create_container == reason end
-    end
-
-
     def test_storage ( storage_account_name )
       @storage_account_name = storage_account_name
       @action = :test_storage
       @max_tries = 1
       @force_client = true # to enable get a client even if all storage_accounts marked dead
-      @recoverable = [ :invalid_storage_key ]
-      storage_io_block( test_storage_recover ) {
+      @recoverable = [ :invalid_storage_key, :container_exist, :create_container ]
+      storage_io_block {
         if @recovery.nil? || :invalid_storage_key == @recovery
-          container_name = "logstash-test-container"
           @info = "#{@action} #{@storage_account_name}"
-          @client.blobClient.create_container( container_name ) unless @@configuration[:disable_blob_upload]
+          @client.blobClient.create_container( @configuration[:test_storage_container] ) unless @configuration[:disable_blob_upload]
         end
       }
     end
 
-    def test_notification_endpoint_recover
-      proc do |reason, e| @recovery = :ok if :invalid_intrumentation_key == reason || :invalid_table_id == reason end
-    end
-
-    def test_notification_endpoint( storage_account_name )
+    def test_notification( storage_account_name )
       @storage_account_name = storage_account_name
-      @action = :test_notification_endpoint
+      @action = :test_notification
       @max_tries = 1
       @force_client = true # to enable get a client even if all storage_accounts marked dead
-      @recoverable = [  ]
-      success = storage_io_block( test_notification_endpoint_recover ) {
+      @recoverable = [ :invalid_instrumentation_key, :invalid_table_id ]
+      success = storage_io_block {
         if @recovery.nil?
           @container_name = "logstash-test-container"
           @blob_name = "logstash-test-blob"
           @table_id = GUID_NULL
-          @intrumentation_key = GUID_NULL
+          @instrumentation_key = GUID_NULL
           @info = "#{@action}"
           set_blob_sas_url
           payload = create_payload
@@ -419,24 +240,21 @@ class LogStash::Outputs::Application_insights
     end
 
 
-    def notify_recover
-      proc do |reason, e|
-        if :notify_failed_blob_not_accessible == reason
-          @sub_state = reason
-          @@failed_on_notify_retry_Qs[@storage_account_name] << state_to_tuple
-        elsif :invalid_intrumentation_key == reason || :invalid_table_id == reason
-          @sub_state = reason
-          Channels.instance.channel( @intrumentation_key, @table_id ).failed_on_notify_retry_Q << state_to_tuple
+    def notify_retry_later
+      if :notify_failed_blob_not_accessible == @recovery
+        @sub_state = @recovery
+        @storage_recovery.recover_later( state_to_tuple, :notify, @storage_account_name )
+      elsif :invalid_instrumentation_key == @recovery || :invalid_table_id == @recovery
+        @sub_state = @recovery
+        Channels.instance.channel( @instrumentation_key, @table_id ).failed_on_notify_retry_Q << state_to_tuple
 
+      else
+        if :notify_failed_blob_not_accessible == @sub_state
+          @storage_recovery.recover_later( state_to_tuple, :notify, @storage_account_name )
+        elsif :invalid_instrumentation_key == @sub_state || :invalid_table_id == @sub_state
+          Channels.instance.channel( @instrumentation_key, @table_id ).failed_on_notify_retry_Q << state_to_tuple
         else
-          @@endpoint_state_on = false
-          if :notify_failed_blob_not_accessible == @sub_state
-            @@failed_on_notify_retry_Qs[@storage_account_name] << state_to_tuple
-          elsif :invalid_intrumentation_key == @sub_state || :invalid_table_id == @sub_state
-            Channels.instance.channel( @intrumentation_key, @table_id ).failed_on_notify_retry_Q << state_to_tuple
-          else
-            @@failed_on_notification_endpoint_retry_Q << state_to_tuple
-          end
+          @notification_recovery.recover_later( state_to_tuple )
         end
       end
     end
@@ -446,32 +264,36 @@ class LogStash::Outputs::Application_insights
       @action = :notify
       @force_client = true # to enable get a client even if all storage_accounts marked dead
       @recoverable = [ :notify_failed_blob_not_accessible, :io_failure, :service_unavailable ]
-      success = storage_io_block( notify_recover ) {
+      success = storage_io_block {
         set_blob_sas_url
         payload = create_payload
-        @@logger.debug { "notification payload: #{payload}" }
+        @logger.debug { "notification payload: #{payload}" }
         @info = "#{@action.to_s} #{@storage_account_name}/#{@container_name}/#{@blob_name}, events: #{@uploaded_events_count}, size: #{@uploaded_bytesize}, blocks: #{@uploaded_block_numbers}, delay: #{Time.now.utc - @oldest_event_time}, blob_sas_url: #{@blob_sas_url}"
 
         # assume that exceptions can be raised due to this method:
-        post_notification( @client.notifyClient, payload ) unless @@configuration[:disable_notification]
+        post_notification( @client.notifyClient, payload ) unless @configuration[:disable_notification]
         @log_state = :notified
       }
-      log_to_table_update if success
+      if success
+        state_table_update
+      else
+        notify_retry_later
+      end
     end
 
     CREATE_EXIST_ERRORS = { :container => [ :create_container, :container_exist ], :table => [ :create_table, :table_exist ] }
     def create_exist_recovery( type, name = nil )
       prev_info = @info
       if CREATE_EXIST_ERRORS[type][0] == @recovery
-        name ||= ( :table == type ? @@state_table_name : @container_name )
+        name ||= ( :table == type ? @configuration[:state_table_name] : @container_name )
         @info = "create #{type} #{@storage_account_name}/#{name}"
 
         # assume that exceptions can be raised due to this method:
         yield name
-        @@logger.info { "Successed to #{@info}" }
+        @logger.info { "Successed to #{@info}" }
         @info = prev_info
       elsif CREATE_EXIST_ERRORS[type][1] == @recovery
-        @@logger.info { "Successed (already exist) to #{@info}" }
+        @logger.info { "Successed (already exist) to #{@info}" }
         @info = prev_info
       end
     end
@@ -485,39 +307,38 @@ class LogStash::Outputs::Application_insights
     end
 
     # return true on success
-    def log_to_table_insert
-      @action = :log_to_table_insert
+    def state_table_insert
+      @action = :state_table_insert
       @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :table_exist, :create_table, :table_busy, :entity_exist ]
       @info  = "#{@action} #{@log_state} #{@storage_account_name}/#{@container_name}/#{@blob_name}"
-      success =  storage_io_block( :uploading == @log_state ? proc do |reason, e| end : log_to_table_update_recover ) {
+      success =  storage_io_block {
         create_table_exist_recovery
         if :entity_exist == @recovery
           raise NotRecoverableError if :uploading == @log_state
         else
           entity_values = state_to_table_entity
-          entity_values[:PartitionKey] = "#{@@configuration[:azure_storage_blob_prefix]}-#{@log_state}"
-          entity_values[:RowKey] = @blob_name
-          @client.tableClient.insert_entity( @@state_table_name, entity_values )
+          entity_values[:PartitionKey] = "#{@configuration[:partition_key_prefix]}-#{@log_state}"
+          entity_values[:RowKey] = @blob_name.gsub("/","_")
+          @client.tableClient.insert_entity( @configuration[:state_table_name], entity_values )
         end
       }
+      @storage_recovery.recover_later( state_to_tuple, :state_table_update, @storage_account_name )  unless success || :uploading == @log_state
+      success
     end
 
-    def log_to_table_update_recover
-      proc do |reason, e| @@failed_on_log_to_table_retry_Qs[@storage_account_name] << state_to_tuple end
-    end
-
-    def log_to_table_update ( tuple = nil )
+    def state_table_update ( tuple = nil )
       tuple_to_state( tuple ) if tuple
       if :uploading == @log_state
-        log_to_table_delete
+        state_table_delete
       elsif :committed == @log_state
-        if log_to_table_insert && log_to_table_delete( nil, :uploading )
+        if state_table_insert && state_table_delete( nil, :uploading )
           State.instance.dec_pending_commits
           State.instance.inc_pending_notifications
-          @@failed_on_notification_endpoint_retry_Q << state_to_tuple
+          # this is not a recovery, it is actually enqueue to notify
+          @notification_recovery.enqueue( state_to_tuple )
         end
       elsif :notified == @log_state
-        if (!@@save_notified_blobs_records || log_to_table_insert) && log_to_table_delete( nil, :committed ) 
+        if (!@configuration[:save_notified_blobs_records] || state_table_insert) && state_table_delete( nil, :committed ) 
           State.instance.dec_pending_notifications
         end
       end
@@ -525,81 +346,81 @@ class LogStash::Outputs::Application_insights
 
 
     # retturn tru on success
-    def log_to_table_delete ( tuple = nil, state = nil )
+    def state_table_delete ( tuple = nil, state = nil )
       tuple_to_state( tuple ) if tuple
       state ||= @log_state
-      @action = :log_to_table_delete
+      @action = :state_table_delete
       @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :table_exist, :create_table, :table_busy, :create_resource ]
       @info  = "#{@action} #{state} #{@storage_account_name}/#{@container_name}/#{@blob_name}"
 
-      success =  storage_io_block( log_to_table_update_recover ) {
+      success =  storage_io_block {
         create_table_exist_recovery
         if :create_resource == @recovery
-          @@logger.info { "Note: delete entity failed, already deleted, #{@info}, state: #{state}, log_state: #{@log_state}" }
+          @logger.info { "Note: delete entity failed, already deleted, #{@info}, state: #{state}, log_state: #{@log_state}" }
         else
-          @client.tableClient.delete_entity( @@state_table_name, "#{@@configuration[:azure_storage_blob_prefix]}-#{state}", @blob_name )
+          @client.tableClient.delete_entity( @configuration[:state_table_name], "#{@configuration[:partition_key_prefix]}-#{state}", @blob_name.gsub( "/", "_" ) )
         end
       }
+      @storage_recovery.recover_later( state_to_tuple, :state_table_update, @storage_account_name ) unless success
+      success
     end
 
     # return entities
-    def log_to_table_query ( storage_account_name, filter , token )
+    def state_table_query ( storage_account_name, filter , token )
       @storage_account_name = storage_account_name
 
-      @action = :log_to_table_query
+      @action = :state_table_query
       @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :table_exist, :create_table, :table_busy ]
-      @info  = "#{@action} #{@storage_account_name}/#{@@state_table_name}"
+      @info  = "#{@action} #{@storage_account_name}/#{@configuration[:state_table_name]}"
 
       entities = nil
-      success =  storage_io_block( proc do |reason, e| end ) {
+      success =  storage_io_block {
         create_table_exist_recovery
         options = { :filter => filter }
         options[:continuation_token] = token if token
-        entities = @client.tableClient.query_entities( @@state_table_name, options )
+        entities = @client.tableClient.query_entities( @configuration[:state_table_name], options )
       }
       entities
     end
 
-    def commit_recover
-      proc do |reason, e| @@failed_on_commit_retry_Qs[@storage_account_name] << state_to_tuple end
-    end
-
     def commit ( tuple = nil )
       tuple_to_state( tuple ) if tuple
 
       unless @uploaded_block_ids.empty?
         @action = :commit
         @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable ]
-        success =  storage_io_block( commit_recover ) {
+        success =  storage_io_block {
           @info = "#{@action.to_s} #{@storage_account_name}/#{@container_name}/#{@blob_name}, events: #{@uploaded_events_count}, size: #{@uploaded_bytesize}, blocks: #{@uploaded_block_numbers}, delay: #{Time.now.utc - @oldest_event_time}"
           # assume that exceptions can be raised due to this method:
-          @client.blobClient.commit_blob_blocks( @container_name, @blob_name, @uploaded_block_ids ) unless @@configuration[:disable_blob_upload]
+          @client.blobClient.commit_blob_blocks( @container_name, @blob_name, @uploaded_block_ids ) unless @configuration[:disable_blob_upload]
           @log_state = :committed
         }
-        # next stage
-        log_to_table_update if success
+        if success
+          # next stage
+          state_table_update
+        else
+          @storage_recovery.recover_later( state_to_tuple, :commit, @storage_account_name )
+        end
       end
     end
 
 
-    def upload_recover
-      proc do |reason, e|
-        unless @uploaded_block_ids.empty?
-          info1 = "#{:commit} #{@storage_account_name}/#{@container_name}/#{@blob_name}, events: #{@uploaded_events_count}, size: #{@uploaded_bytesize}, blocks: #{@uploaded_block_numbers}, delay: #{Time.now.utc - @oldest_event_time}"
-          @@logger.error { "Pospone to #{info1} (; retry later, error: #{e.inspect}" }
-          @@failed_on_commit_retry_Qs[@storage_account_name] << state_to_tuple
-          @uploaded_block_ids = [  ]
-        end
-        unless :io_all_dead == reason
-          @recovery = :invalid_storage_account
-        else 
-          Channels.instance.channel( @intrumentation_key, @table_id ).failed_on_upload_retry_Q << @block_to_upload
-          @block_to_upload = nil
-        end
+    def upload_retry_later
+      unless @uploaded_block_ids.empty?
+        info1 = "#{:commit} #{@storage_account_name}/#{@container_name}/#{@blob_name}, events: #{@uploaded_events_count}, size: #{@uploaded_bytesize}, blocks: #{@uploaded_block_numbers}, delay: #{Time.now.utc - @oldest_event_time}"
+        @logger.error { "Pospone to #{info1} (; retry later, error: #{@last_io_exception.inspect}" }
+        @storage_recovery.recover_later( state_to_tuple, :commit, @storage_account_name )
+        @uploaded_block_ids = [  ]
+      end
+      unless :io_all_dead == @recovery
+        raise UploadRetryError
+      else 
+        Channels.instance.channel( @instrumentation_key, @table_id ).failed_on_upload_retry_Q << @block_to_upload
+        @block_to_upload = nil
       end
     end
 
-    def upload ( block, to_commit = nil )
+    def upload ( block )
       @storage_account_name = nil if @uploaded_block_ids.empty?
       @block_to_upload = block
       block = nil # remove reference for GC
@@ -615,7 +436,7 @@ class LogStash::Outputs::Application_insights
           # remove record of previous upload that failed
           if @storage_account_name
             exclude_storage_account_names << @storage_account_name
-            @@failed_on_log_to_table_retry_Qs[@storage_account_name] << state_to_tuple
+            @storage_recovery.recover_later( state_to_tuple, :state_table_update, @storage_account_name )
           end
           set_conatainer_and_blob_names
           @storage_account_name = Clients.instance.get_random_active_storage( exclude_storage_account_names )
@@ -623,7 +444,7 @@ class LogStash::Outputs::Application_insights
             upload_recover.call( :io_all_dead, nil )
             return false
           end
-          raise UploadRetryError unless log_to_table_insert
+          raise UploadRetryError unless state_table_insert
         end
 
         @action = :upload
@@ -631,12 +452,12 @@ class LogStash::Outputs::Application_insights
         @info = "#{@action} #{@storage_account_name}/#{@container_name}/#{@blob_name}, #{@block_info}, commitId: [\"#{100001 + @uploaded_block_ids.length}\"]"
         @recoverable = [ :invalid_storage_key, :invalid_storage_account, :io_failure, :service_unavailable, :container_exist, :create_container ]
 
-        success = storage_io_block( upload_recover ) {
+        success = storage_io_block {
           create_container_exist_recovery
           block_id = "#{100001 + @uploaded_block_ids.length}"
 
           # assume that exceptions can be raised due to this method:
-          @client.blobClient.put_blob_block( @container_name, @blob_name, block_id, @block_to_upload.bytes ) unless @@configuration[:disable_blob_upload]
+          @client.blobClient.put_blob_block( @container_name, @blob_name, block_id, @block_to_upload.bytes ) unless @configuration[:disable_blob_upload]
 
           # upload success
           first_block_in_blob = @uploaded_block_ids.empty?
@@ -655,8 +476,7 @@ class LogStash::Outputs::Application_insights
           Telemetry.instance.track_event("uploading", {:properties => state_to_table_entity})
         }
 
-        raise UploadRetryError if :invalid_storage_account == @recovery
-        commit if success && to_commit
+        upload_retry_later unless success
       rescue UploadRetryError
         @recovery = nil
         retry
@@ -670,7 +490,7 @@ class LogStash::Outputs::Application_insights
       @action = :list_blob_blocks
       @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :container_exist, :create_container, :create_blob ]
       list_blob_blocks = nil
-      success =  storage_io_block( proc do |reason, e| end ) {
+      success =  storage_io_block {
         @info = "#{@action} #{@storage_account_name}/#{@container_name}/#{@blob_name}"
 
         create_container_exist_recovery
@@ -705,15 +525,15 @@ class LogStash::Outputs::Application_insights
     private
 
 
-    def storage_io_block( recover_later_proc, valid_recovery = nil )
+    def storage_io_block
       @recovery = nil
       @try_count = 1
 
       begin
         @client ||= Client.new( @storage_account_name, @force_client )
         yield
-        disabled = :notify == @action ? @@configuration[:disable_notification] : @@configuration[:disable_blob_upload]
-        @@logger.info { "Successed to #{disabled ? 'DISABLED ' : ''}#{@info}" }
+        disabled = :notify == @action ? @configuration[:disable_notification] : @configuration[:disable_blob_upload]
+        @logger.info { "Successed to #{disabled ? 'DISABLED ' : ''}#{@info}" }
         true
 
       rescue TypeError
@@ -721,8 +541,13 @@ class LogStash::Outputs::Application_insights
 
       rescue StandardError => e
         @last_io_exception = e
-        @recovery = nil
-        retry if recover_retry?( e, recover_later_proc )
+        @recovery, reason = recover_retry?( e )
+        retry if @recovery || reason.nil?
+
+        puts " +++ recovery: #{@recovery}, reason: #{reason}"
+
+        @recovery = reason
+        @logger.error { "Failed to #{@info} ; retry later, error: #{e.inspect}" }
         false
 
       ensure
@@ -730,157 +555,173 @@ class LogStash::Outputs::Application_insights
       end
     end
 
-
-    def recover_retry? ( e, recover_later_proc )
-      # http error, probably server error
+    def error_to_sym ( e )
       if e.is_a?( Azure::Core::Http::HTTPError )
+        if 404 == e.status_code
+          if "ContainerNotFound" == e.type
+            :create_container
 
-        if 404 == e.status_code && "ContainerNotFound" == e.type
-          @recovery = :create_container
-
-        elsif 404 == e.status_code && "TableNotFound" == e.type
-          @recovery = :create_table
+          elsif "TableNotFound" == e.type
+            :create_table
 
-        elsif 404 == e.status_code && "BlobNotFound" == e.type
-          @recovery = :create_blob
+          elsif "BlobNotFound" == e.type
+            :create_blob
           
-        elsif 404 == e.status_code && "ResourceNotFound" == e.type
-          @recovery = :create_resource
+          elsif "ResourceNotFound" == e.type
+            :create_resource
 
-        elsif 409 == e.status_code && "ContainerAlreadyExists" == e.type
-          @recovery = :container_exist
+          else
+            :create_resource
+          end
 
-        elsif 409 == e.status_code && "BlobAlreadyExists" == e.type
-          @recovery = :blob_exist
+        elsif 409 == e.status_code 
+          if "ContainerAlreadyExists" == e.type
+            :container_exist
 
-        elsif 409 == e.status_code && "TableAlreadyExists" == e.type
-          @recovery = :table_exist
+          elsif "BlobAlreadyExists" == e.type
+            :blob_exist
 
-        elsif 409 == e.status_code && "TableBeingDeleted" == e.type
-          @recovery = :table_busy
+          elsif "TableAlreadyExists" == e.type
+            :table_exist
 
-        elsif 409 == e.status_code && "EntityAlreadyExists" == e.type
-          @recovery = :entity_exist
+          elsif "TableBeingDeleted" == e.type
+            :table_busy
 
-        elsif 403 == e.status_code && "AuthenticationFailed" == e.type
-          @recovery = :invalid_storage_key
+          elsif "EntityAlreadyExists" == e.type
+            :entity_exist
 
-        elsif 403 == e.status_code  && "Unknown" == e.type && e.description.include?("Blob does not exist or not accessible.")
-          @recovery = :notify_failed_blob_not_accessible
+          else
+            :http_unknown
+          end
+
+        elsif 403 == e.status_code 
+          if "AuthenticationFailed" == e.type
+            :invalid_storage_key
+
+          elsif "Unknown" == e.type && e.description.include?("Blob does not exist or not accessible.")
+            :notify_failed_blob_not_accessible
+
+          else
+            :access_denied
+          end
 
         elsif 400 == e.status_code  && "Unknown" == e.type && e.description.include?("Invalid instrumentation key")
-          @recovery = :invalid_intrumentation_key
+          :invalid_instrumentation_key
 
         elsif 500 == e.status_code  && "Unknown" == e.type && e.description.include?("Processing error")
-          @recovery = :notification_process_down
+          :notification_process_down
 
         elsif 503 == e.status_code
-          @recovery = :service_unavailable
-        elsif 404 == e.status_code
-          @recovery = :create_resource
-        elsif 403 == e.status_code
-          # todo, came from updating the log_table, how to hnadle this
-          @recovery = :access_denied
+          :service_unavailable
+
         else
-          puts "\n>>>> HTTP error - #{e.inspect} <<<<\n"
-          @recovery = :http_unknown
-          raise e if  @@configuration[:stop_on_unknown_io_errors]
+          :http_unknown
         end
 
       # communication error
       elsif e.is_a?( Faraday::ClientError )
-        @recovery = :io_failure
+        :io_failure
 
       # communication error
       elsif e.is_a?( IOError )
-        @recovery = :io_failure
+        :io_failure
 
       # all storage accounts are dead, couldn't get client (internal exception)
       elsif e.is_a?( StorageAccountsOffError )
-        @recovery = :io_all_dead
+        :io_all_dead
 
       # all storage accounts are dead, couldn't get client (internal exception)
       elsif e.is_a?( NotRecoverableError )
-        @recovery = :not_recoverable
+        :not_recoverable
 
       elsif e.is_a?( NameError ) && e.message.include?( "uninitialized constant Azure::Core::Auth::Signer::OpenSSL" )
-        sleep( 1 )
-        @recovery = :io_failure
+        :init_error
 
       elsif e.is_a?( NameError ) && e.message.include?( "uninitialized constant Azure::Storage::Auth::SharedAccessSignature" )
-        sleep( 1 )
-        @recovery = :io_failure
+        :init_error
 
       else
-        # UNKNOWN error - #<NameError: uninitialized constant Azure::Core::Auth::Signer::OpenSSL>
-        puts "\n>>>> UNKNOWN error - #{e.inspect} <<<<\n"
-        raise e
+        :unknown
+      end
+    end
 
+
+    def recover_retry? ( e )
+      recovery = error_to_sym( e )
+      if :init_error == recovery
+        @client = @client.dispose if @client
+        sleep( 1 )
+        recovery = nil
+
+      elsif :http_unknown == recovery || :unknown == recovery
+        puts "\n>>>> UNKNOWN error - #{e.inspect} <<<<\n"
+        raise e if  @configuration[:stop_on_unknown_io_errors]
       end
 
-      reason = @recovery
-      if @recovery && @recoverable.include?( @recovery )
-        case @recovery
-        when :container_exist, :table_exist, :entity_exist, :create_container, :create_table
-          # ignore log error
-          # @@logger.error { "Failed to #{@info} ;( recovery: continue, error: #{e.inspect}" }
+      return [recovery, recovery] unless  recovery && @recoverable.include?( recovery )
 
-        when :invalid_storage_key, :notify_failed_blob_not_accessible
-          if @client.switch_storage_account_key!
-            @@logger.error { "Failed to #{@info} ;( recovery: switched to secondary storage key, error: #{e.inspect}" }
-          else
-            @client = @client.dispose( :auth_to_storage_failed ) if @client && :invalid_storage_key == @recovery 
-            @recovery = nil
-          end
+      case recovery
+      when :container_exist, :table_exist, :entity_exist, :create_container, :create_table
+        # ignore log error
+        # @logger.error { "Failed to #{@info} ;( recovery: continue, error: #{e.inspect}" }
 
-        when :table_busy
+      when :invalid_storage_key, :notify_failed_blob_not_accessible
+        if @client.switch_storage_account_key!
+          @logger.error { "Failed to #{@info} ;( recovery: switched to secondary storage key, error: #{e.inspect}" }
+        else
+          @client = @client.dispose( :auth_to_storage_failed ) if @client && :invalid_storage_key == recovery 
+          return [nil, recovery]
+        end
+
+      when :table_busy
+        @client = @client.dispose if @client
+        sleep( @configuration[:io_retry_delay] )
+        @logger.error { "Failed to #{@info} ;( recovery: retry, error: #{e.inspect}" }
+
+      when :io_failure, :service_unavailable, :notification_process_down
+        if @try_count < @max_tries
           @client = @client.dispose if @client
-          sleep( @@io_retry_delay )
-          @@logger.error { "Failed to #{@info} ;( recovery: retry, error: #{e.inspect}" }
+          sleep( @configuration[:io_retry_delay] )
+          @logger.error { "Failed to #{@info} ;( recovery: retry, try #{@try_count} / #{@max_tries}, error: #{e.inspect}" }
+          @try_count += 1
+        else
+          if :io_failure == recovery || ( :service_unavailable == recovery && :notify != @action )
+            @client = @client.dispose( :io_to_storage_failed ) if @client
+          end
+          return [nil, recovery]
+        end
 
-        when :io_failure, :service_unavailable, :notification_process_down, :invalid_intrumentation_key, :invalid_table_id
+      when :invalid_instrumentation_key, :invalid_table_id
+        if :notify == @action # only for notify, not for test endpoint
           if @try_count < @max_tries
             @client = @client.dispose if @client
-            sleep( @@io_retry_delay )
-            @@logger.error { "Failed to #{@info} ;( recovery: retry, try #{@try_count} / #{@max_tries}, error: #{e.inspect}" }
+            sleep( @configuration[:io_retry_delay] )
+            @logger.error { "Failed to #{@info} ;( recovery: retry, try #{@try_count} / #{@max_tries}, error: #{e.inspect}" }
             @try_count += 1
           else
-            if :invalid_intrumentation_key == @recovery
-              Channels.instance.mark_invalid_intrumentation_key( @intrumentation_key )
-            elsif :invalid_table_id == @recovery
+            if :invalid_instrumentation_key == recovery
+              Channels.instance.mark_invalid_instrumentation_key( @instrumentation_key )
+            elsif :invalid_table_id == recovery
               Channels.instance.mark_invalid_table_id( @table_id )
-            elsif :io_failure == @recovery || ( :service_unavailable == @recovery && :notify != @action )
-              @client = @client.dispose( :io_to_storage_failed ) if @client
             end
-            @recovery = nil
+            return [nil, recovery]
           end
         end
-      else
-        @recovery = nil
       end
+      [recovery, recovery]
+    end
 
-      if @recovery
-        true
-      else
-        recover_later_proc.call( reason, e )
-        @@logger.error { "Failed to #{@info} ; retry later, error: #{e.inspect}" } unless :ok == @recovery
-        :ok == @recovery
-      end
 
-      # Blob service error codes - msdn.microsoft.com/en-us/library/azure/dd179439.aspx
-      # ConnectionFailed - problem with connection
-      # ParsingError - problem with request/response payload
-      # ResourceNotFound, SSLError, TimeoutError
-    end
 
     def set_conatainer_and_blob_names
       time_utc = Time.now.utc
       id = @id.to_s.rjust(4, "0")
       strtime = time_utc.strftime( "%F" )
-      @container_name = "#{@@configuration[:azure_storage_container_prefix]}-#{strtime}"
+      @container_name = "#{AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX}#{@configuration[:azure_storage_container_prefix]}-#{strtime}"
 
       strtime = time_utc.strftime( "%F-%H-%M-%S-%L" )
-      @blob_name = "#{@@configuration[:azure_storage_blob_prefix]}_ikey-#{@intrumentation_key}_table-#{@table_id}_id-#{id}_#{strtime}.#{@event_format_ext}"
+      # @blob_name = "#{@configuration[:azure_storage_blob_prefix]}_ikey-#{@instrumentation_key}_table-#{@table_id}_id-#{id}_#{strtime}.#{@event_format_ext}"
+      @blob_name = "#{AZURE_STORAGE_BLOB_LOGSTASH_PREFIX}#{@configuration[:azure_storage_blob_prefix]}/ikey-#{@instrumentation_key}/table-#{@table_id}/#{strtime}_#{id}.#{@event_format_ext}"
     end
 
 
@@ -892,30 +733,30 @@ class LogStash::Outputs::Application_insights
             :ver           => BASE_DATA_REQUIRED_VERSION,
             :blobSasUri    => @blob_sas_url.to_s,
             :sourceName    => @table_id,
-            :sourceVersion => @@configuration[:notification_version].to_s
+            :sourceVersion => @configuration[:notification_version].to_s
           }
         }, 
-        :ver  => @@configuration[:notification_version], 
+        :ver  => @configuration[:notification_version], 
         :name => REQUEST_NAME,
         :time => Time.now.utc.iso8601,
-        :iKey => @intrumentation_key
+        :iKey => @instrumentation_key
       }
       notification_hash.to_json
     end 
 
 
     def post_notification ( http_client, body )
-      request = Azure::Core::Http::HttpRequest.new( :post, @@configuration[:notification_endpoint], { :body => body, :client => http_client } )
+      request = Azure::Core::Http::HttpRequest.new( :post, @configuration[:application_insights_endpoint], { :body => body, :client => http_client } )
       request.headers['Content-Type'] = 'application/json; charset=utf-8'
       request.headers['Accept'] = 'application/json'
-      @@logger.debug { "send notification : \n    endpoint: #{@@configuration[:notification_endpoint]}\n    body : #{body}" }
+      @logger.debug { "send notification : \n    endpoint: #{@configuration[:application_insights_endpoint]}\n    body : #{body}" }
       response = request.call
     end 
 
 
     def set_blob_sas_url
       blob_url ="https://#{@storage_account_name}.blob.core.windows.net/#{@container_name}/#{@blob_name}"
-      options_and_constrains = {:permissions => "r", :resource => "b", :expiry => ( Time.now.utc + @@configuration[:blob_access_expiry_time] ).iso8601 }
+      options_and_constrains = {:permissions => "r", :resource => "b", :expiry => ( Time.now.utc + @configuration[:blob_access_expiry_time] ).iso8601 }
       @blob_sas_url = @client.storage_auth_sas.signed_uri( URI( blob_url ), options_and_constrains )
     end