logstash-output-application_insights 0.1.3

data/Rakefile

@@ -0,0 +1,22 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+
+require "logstash/devutils/rake"

data/lib/logstash/outputs/application_insights.rb

@@ -0,0 +1,393 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+
+require "logstash/outputs/base"
+require "logstash/namespace"
+require 'logstash-core/version'
+
+require "stud/interval"
+
+require "azure/storage"
+require 'azure/storage/core/auth/shared_access_signature'
+
+require 'azure/core/http/http_request'
+require 'azure/core/http/http_response'
+
+require 'rbconfig'
+require 'faraday'
+require 'openssl'
+require "time"
+require "json"
+require "uri"
+require "socket"
+
+require "thread"
+require "concurrent" # for atomic and thread safe operations
+require "logger"
+require "csv"
+
+require "application_insights"
+
+class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
+  require "logstash/outputs/application_insights/version"
+  require "logstash/outputs/application_insights/utils" 
+  require "logstash/outputs/application_insights/constants" 
+  require "logstash/outputs/application_insights/config" 
+  require "logstash/outputs/application_insights/blob" 
+  autoload :Block, "logstash/outputs/application_insights/block"
+  autoload :Storage_cleanup, "logstash/outputs/application_insights/storage_cleanup"
+
+  autoload :Clients, "logstash/outputs/application_insights/clients" 
+  autoload :Client, "logstash/outputs/application_insights/client" 
+
+  autoload :Channels, "logstash/outputs/application_insights/channels"
+  autoload :Channel, "logstash/outputs/application_insights/channel"
+  autoload :Sub_channel, "logstash/outputs/application_insights/sub_channel" 
+  autoload :State, "logstash/outputs/application_insights/state"
+  autoload :Flow_control, "logstash/outputs/application_insights/flow_control"
+  autoload :Shutdown, "logstash/outputs/application_insights/shutdown"
+  autoload :Telemetry, "logstash/outputs/application_insights/telemetry"
+
+  require "logstash/outputs/application_insights/exceptions" 
+  require "logstash/outputs/application_insights/timer" 
+  require "logstash/outputs/application_insights/multi_io_logger" 
+
+  config_name "application_insights"
+    
+  # default :codec, "json_lines" 
+  
+  # Array of pairs, storage_account_name and an array of acces_keys 
+  # examples [ account1, key1 ]
+  #          [ [account1, key1], [accoun2, key2], [account3, key3] ]
+  #          [ [account1, [key11, key12]], [account2, key2], [account3, [key3] ]
+  config :storage_account_name_key, :validate => :array
+
+  # prefix for the azure storage tables used by this Logstash instance
+  # it is recommeded that each Logstash prefix have a unique prefix, to avoid
+  # confusion and loss of tracking, although sharing tables won't damage proper execution
+  # if not set, the machine name is used (not alphanumeric characters are removed, and converted downcase), if machine name available
+  # string may contain only alphanumeric character, case sensitive, and must start with a letter 
+  config :azure_storage_table_prefix, :validate => :string
+
+  # prefix for the azure storage containers used by this Logstash instance
+  # it is recommeded that each Logstash prefix have a unique prefix, to avoid
+  # confusion and loss of tracking, although sharing containers won't damage proper execution
+  # if not set, the machine name is used (not alphanumeric characters are removed, and converted downcase), if machine name available
+  # string may contain only alphanumeric character and dash, double dash not allowed, case insesitive
+  config :azure_storage_container_prefix, :validate => :string
+
+  # prefix for the azure storage blobs used by this Logstash instance
+  # Each Logstash prefix MUST have a unique prefix, to avoid
+  # loss of data !!!
+  # if not set, the machine name is used (not alphanumeric characters are removed, and converted downcase), if machine name available
+  # string may include only characters that are allowed in any valid url
+  config :azure_storage_blob_prefix, :validate => :string
+  
+  # Default Application Insights Analytics intrumentation_key
+  # will be used only in case it is not specified as a table_id property in tables
+  # or as part of the event's fields or event's metadata fields
+  config :intrumentation_key, :validate => :string
+
+  # Default Application Insights Analytics table_id
+  # will be used only in case it is not specified as part o
+  # of the event's fields or event's metadata fields
+  config :table_id, :validate => :string
+
+  # specifies the list of the fields that will be filtered from the events, fields not specified will be ignored.
+  # if not specified all fileds in events will be filtered, the order is kept, and is used for csv serialization
+  config :table_columns, :validate => :array
+
+  # if set to true events fields are refered as case insensitive, default case sensitive 
+  config :case_insensitive_columns, :validate => :boolean
+
+  # A hash of table_ids, where each table_id points to a set of properties
+  # the properties are a hash, where the keys are are the properties
+  # current supported properties per table_id are:
+  # intrumentation_key, ext, table_columns, csv_default_value, csv_separator, blob_max_delay, event_separator, serialized_event_field
+  # intrumentation_key, Application Insights Analytics intrumentation_key, will be used in case not specified in any of the event's fields or events's metadata fileds
+  # serialized_event_field, specifies the field that may contain the full serialized event (either as json or csv), 
+  #             when specified, the ext property should be set either to csv or to json (json is the default)
+  #             if event. does not conatin the field, value will be created based on the fileds in the evnt, according to table_columns if configured, or all fileds in event
+  #             if event contains this filed, and ext is csv
+  #                  if value is a string, it will be used as is as the serialized event, without validating whether it is a csv string
+  #                  if value is an array, it will be serialized as an array of csv columns
+  #                  if value is a hash, it will be serialized based on table_columns to csv columns
+  #             if event contains this filed, and ext is json
+  #                  if value is a string, it will be used as is as the serialized event, without validating whether it is a json string
+  #                  if value is a hash, it will be serialized to json, if fileds_map exit, it will be based on filds_map
+  #                  if value is an array, it will be zipped with table_columns (if exist) and serialized to json
+  # ext, blob extension, the only valid values are either csv or json, 
+  #             should be set whenever the default json is not appropriate (.e, csv)
+  # blob_max_delay, maximum latency time, in seconds, since the time the event arrived till it should be commited in azure storage, and Application Insights is notified
+  # event_separator, specifies the string that is used as a separator between events in the blob
+  # table_columns, specifies the event fields that should be serialized, and their order (order is required for csv)
+  #          if csv serialization will be used for this table_id
+  #          each table_columns field is a hash with 3 keys: name, type, and default. Only name is mandatory
+  #               name - is the name of the event fleld that its value should be mapped to this columns
+  #               type - is the type of this field: "string", "hash", "array", "number", "json", "boolean", "float", "integer", "dynamic", "datetime", "object"
+  #               default - is the value to be used for this column, in case the field is missing in the event
+  # csv_separator, specifies the string that is used as a separator between columns, 
+  #                can be specified only together with table_columns
+  # csv_default_value, specifies the string that is used as the value in a csv record, in case the field does not exist in the event
+  #                can be specified only together with table_columns
+  #
+  # Example json table_id
+  #   tables => {"a679fbd2-702c-4c46-8548-80082c66ef28" => {"intrumentation_key" => "abee940b-e648-4242-b6b3-f2826667bf96", "blob_max_delay" => 60} }
+  # Example json table_id, input in serialized_event_field
+  #   {"ab6a3584-aef0-4a82-8725-2f2336e59f3e" => {"serialized_event_field" => "message". "ext" => "json"} }
+  # Example csv table_id, input in serialized_event_field
+  #   {"ab6a3584-aef0-4a82-8725-2f2336e59f3e" => {"serialized_event_field" => "csv_message". "ext" => "csv"} }
+  # Example csv table_id, input in event fields
+  #   {"ab6a3584-aef0-4a82-8725-2f2336e59f3e" => { "ext" => "csv", "table_columns" => [ {name => "Timestamp" type => datetime }, "Value", "Custom" ] } }
+  # Example csv table_id, input in event fields
+  #   {"ab6a3584-aef0-4a82-8725-2f2336e59f3e" => { "ext" => "json", "table_columns" => [ "Timestamp", "Value", "Custom" ] } }
+
+  config :tables, :validate => :hash, :default => {}
+
+  # Advanced, internal, should not be set, the default is 192 GB ( = 50,000 * 4 MB ) 
+  # azure storage maximum number of blocks per blob is 192 GB ( = 50,000 * 4 MB ) 
+  config :blob_max_bytesize, :validate => :number
+
+  # Specifies, maximum number of events in one blob,
+  # setting it low may reduce the latency, but reduce logstash performance
+  # setting it high may increase latency to maximum delay, but logstash will be more efficient, and load on network will be lower
+  config :blob_max_events, :validate => :number
+
+  # Specifies maximum latency time, in seconds, since the time the event arrived 
+  # till it is commited to azure storage, and Application Insights is notified
+  # The total latency may be higher, as this is not the full ingestion flow 
+  config :blob_max_delay, :validate => :number
+
+  # Specifies the blob serialziation to create. Default "json"
+  # currently 2 types are supported "csv" and "json""
+  config :blob_serialization, :validate => :string
+
+  # Interval of time between retries due to IO failures
+  config :io_retry_delay, :validate => :number
+
+  # Number of retries on IO failures, before giving up, and move to available options
+  config :io_max_retries, :validate => :number
+
+  # Specifies the retention time of the blob in the container after it is notified to Application Insighta Analytics
+  # Once the retention time expires, the blob is the deleted from container
+  config :blob_retention_time, :validate => :number
+
+  # Specifies the time Application Insights Analytics have access to the blob that are notifie
+  # Blob access is limited with SAS URL
+  config :blob_access_expiry_time, :validate => :number
+
+  # Advanced, internal, should not be set, the default is \r\n, 
+  # specifies the string that is used as a separator between events in the blob
+  config :event_separator, :validate => :string
+
+  # Advanced, internal, should not be set, the default is comma, 
+  # specifies the string that is used as a separator between columns in a csv record
+  config :csv_separator, :validate => :string
+
+  # specifies the string that is used as the value in a csv record, in case the field does not exist in the event, the default is ""
+  config :csv_default_value, :validate => :string
+
+  # specifies the log level. valid values are: DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN
+  config :logger_level, :validate => :string
+
+  # Specifies the list of targets for the log. may include files, devices, "stdout: and "stderr"
+  config :logger_files, :validate => :array
+
+  # Advanced, internal, should not be set, the default is AI, 
+  # Specifies the program name that will displayed in each log record
+  config :logger_progname, :validate => :string
+
+  # Specifies when file logs are shifted. valid values are either an integer or "daily", "weekly" or "monthly"
+  config :logger_shift_size
+
+  # Specifies the shift age of a log.
+  # Number of old files to keep, or frequency of rotation (daily, weekly or monthly)
+  config :logger_shift_age, :validate => :number
+
+  # Specifies a serialized event field name, that if exist in current event, its value as is will be taken as the serialized event. No Default
+  config :serialized_event_field, :validate => :string
+
+  # Specifies the time interval, between tests that check whether a stoarge account came back to life, 
+  # after it stoped responding 
+  config :resurrect_delay, :validate => :number
+
+  # specifies the high water mark for the flow control, that used to avoid out of memory crash
+  # once memory consumption reach reach the high water mark, the plugin will stop accepting events, till memory
+  # is below the low water mark
+  config :flow_control_suspend_bytes, :validate => :number
+
+  # specifies the low water mark for the flow control, that used to avoid out of memory crash
+  # once memory consumption reach the high water mark, the plugin will stop accepting events, till memory
+  # is below the low water mark
+  config :flow_control_resume_bytes, :validate => :number
+
+  # specifies the amount of time the flow control suspend receiving event, to allow GC, and flush of event to Azure storage
+  # before checking whether memory is below low water mark
+  config :flow_control_delay, :validate => :number
+
+  # File path of the CA file if having issue with SSL
+  config :ca_file, :validate => :string
+
+
+  # When set to true, telemetry about the plugin, won't be sent to Application Insights
+  config :disable_telemetry, :validate => :boolean
+
+  # When set to true, storage cleanup won't be done by the plugin (should be done by some other means or by another Logstash process with this flag enabled)
+  config :disable_cleanup, :validate => :boolean
+
+  # When set to true, not notified blobs are deleted, if not set they are copied to the orphan-blobs container
+  config :delete_not_notified_blobs, :validate => :boolean
+
+  # When set to true, notified blobs records are save in table, as long as blobs are retained in their containers
+  config :save_notified_blobs_records, :validate => :boolean
+
+  # Advanced, internal, should not be set, the default is false
+  # When set to true, notification is not sent to application insights, but behaves as if notified
+  config :disable_notification, :validate => :boolean
+
+  # Advanced, internal, should not be set, the default is false
+  # When set to true, events are not uploaded, and blob not commited, but behaves as if uploaded and notified
+  config :disable_blob_upload, :validate => :boolean
+
+  # Advanced, internal, should not be set, the default is false
+  # When set to true, process will stop if an unknown IO error is found
+  config :stop_on_unknown_io_errors, :validate => :boolean
+
+  # Advanced, internal, should not be set, the default is Application Insights production endpoint
+  # when set notification are sent to an alternative endpoint, used for internal testing
+  config :notification_endpoint, :validate => :string
+
+  # Advanced, internal, should not be set, the only current valid value is 1
+  config :notification_version, :validate => :number
+
+  # When set to true, access to application insights will be validated at initialization
+  # and if validation fail, logstash process will abort
+  config :validate_endpoint, :validate => :boolean, :default => true
+
+  # When set to true, access to azure storage for each of the configured accounts will be validated at initialization
+  # and if validation fail, logstash process will abort
+  config :validate_storage, :validate => :boolean, :default => true
+
+  public
+
+  def register
+
+    # set configuration
+    Config.validate_and_adjust_configuration( default_configuration )
+    configuration = Config.current
+
+    Multi_io_logger.config( configuration )
+
+    # be careful don't use here @logger, as it will override Logstash @logger, and may create starnge behaviour
+    @private_logger = configuration[:logger]
+
+    @private_logger.info { "configuration: #{configuration}" }
+
+    @telemetry = Telemetry.instance
+    configuration[:telemetry_channel] = @telemetry.telemetry_channel
+
+    Timer.config( configuration )
+    Blob.config( configuration )
+    Blob.validate_endpoint if @validate_endpoint
+    Blob.validate_storage if @validate_storage
+
+    @shutdown = Shutdown.instance
+    @channels = Channels.instance
+
+    @storage_cleanup = Storage_cleanup.start
+
+    @private_logger.info { "plugin registered" }
+
+    # @codec.on_event do |event, encoded_event|
+    #   @channels.receive( event, encoded_event )
+    # end
+
+    Telemetry.instance.track_event("register", {:properties => configuration})
+
+
+    return "ok\n"
+  end # def register
+  
+
+  def receive ( event )
+    # @codec.encode( event )
+    @channels.receive( event, nil )
+    return "ok\n"
+  end
+
+  def close
+    Telemetry.instance.track_event( "close" )
+    Telemetry.instance.flush
+    @shutdown.submit
+  end
+
+  private
+
+  # -----------------------------------------------
+
+
+  def list_blob_names
+    blob_names = Set.new []
+    loop do
+      continuation_token = NIL
+      entries = @azure_blob.list_blobs(@container, { :timeout => 10, :marker => continuation_token})
+      @@logger.debug { 'blob entries: #{entries}' }
+      entries.each do |entry|
+        @@logger.debug { 'blob entry name: #{entry.name}' }
+        blob_names << entry.name
+      end
+      continuation_token = entries.continuation_token
+      break if continuation_token.empty?
+    end
+    return blob_names
+  end # def list_blobs
+
+
+  def list_container_names
+    container_names = Set.new []
+    loop do
+      continuation_token = NIL
+      containers = @azure_blob.list_containers()
+      @@logger.debug { 'containers: #{containers}' }
+      containers.each do |container|
+        @@logger.debug { 'container entry name:' + container.name }
+        container_names << container.name
+        upload(container.name, "blob-append-" + container.name, "test - " + container.name)
+        blobs = @azure_blob.list_blobs(container.name)
+        blobs.each do |blob|
+          @@logger.debug { 'blob name: ' + blob.name }
+        end
+      end
+      continuation_token = containers.continuation_token
+      break if continuation_token.empty?
+    end
+    return container_names
+  end # def list_blobs
+
+  def create_container (container_name)
+    begin
+      @azure_blob.create_container(container_name)
+    rescue
+      @@logger.debug { $! }
+    end
+  end
+end
+

data/lib/logstash/outputs/application_insights/blob.rb

@@ -0,0 +1,923 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+class LogStash::Outputs::Application_insights
+  class Blob
+
+    attr_reader :intrumentation_key
+    attr_reader :table_id
+    attr_reader :storage_account_name
+    attr_reader :container_name
+    attr_reader :blob_name
+    attr_reader :uploaded_events_count
+    attr_reader :uploaded_bytesize
+    attr_reader :oldest_event_time
+
+    attr_reader :io_queue
+
+    attr_reader :last_io_exception
+
+    public
+
+    def self.config ( configuration )
+      @@configuration = configuration
+
+      @@logger = configuration[:logger]
+      @@io_retry_delay = configuration[:io_retry_delay]
+      @@io_max_retries = configuration[:io_max_retries]
+      @@blob_max_bytesize = configuration[:blob_max_bytesize]
+      @@blob_max_events = configuration[:blob_max_events]
+      @@state_table_name = "#{configuration[:azure_storage_table_prefix]}#{STATE_TABLE_NAME}"
+      @@save_notified_blobs_records = configuration[:save_notified_blobs_records]
+
+      @@closing = false
+
+      # queues, per storage_account_name, for failed blob commit, will continue to try resending
+      @@failed_on_commit_retry_Qs = {}
+      launch_storage_recovery_threads( @@failed_on_commit_retry_Qs, :commit, :io_failure )
+      launch_storage_recovery_table_threads( :uploading )
+
+      # queues, per storage_account_name, for failed notify, will continue to try resending
+      @@failed_on_notify_retry_Qs = {}
+      launch_storage_recovery_threads( @@failed_on_notify_retry_Qs, :notify, :notify_failed_blob_not_accessible )
+      launch_storage_recovery_table_threads( :committed )
+
+      # for failed to notify due to endpoint, will continue to try resending
+      launch_endpoint_recovery_thread
+
+      # queues, per storage_account_name, for failed to log to table, will continue to try resending
+      @@failed_on_log_to_table_retry_Qs = {}
+      launch_storage_recovery_threads( @@failed_on_log_to_table_retry_Qs, :log_to_table_update, :io_failure )
+
+    end
+
+    def self.close
+      @@closing = true
+    end
+
+    def self.stopped?
+      @@closing
+    end
+
+    def self.launch_endpoint_recovery_thread
+      @@failed_on_notification_endpoint_retry_Q = Queue.new
+      storage_recovery_thread( nil, @@failed_on_notification_endpoint_retry_Q, :notify, :io_failure )
+    end
+
+    def self.launch_storage_recovery_threads ( queues, method, failure_reason )
+      @@configuration[:storage_account_name_key].each do |storage_account_name, storage_account_keys|
+        queues[storage_account_name] = Queue.new
+        # a threads, per storage  account name
+        storage_recovery_thread( storage_account_name, queues[storage_account_name], method, failure_reason )
+      end
+    end
+
+    def self.launch_storage_recovery_table_threads ( state )
+      @@configuration[:storage_account_name_key].each do |storage_account_name, storage_account_keys|
+        recovery_table_thread( storage_account_name, state)
+      end
+    end
+
+    #return thread
+    def self.recovery_table_thread( storage_account_name, state )
+      Thread.new( storage_account_name, state ) do |storage_account_name, state|
+        
+        blob = Blob.new
+
+        committed_tuples = [  ]
+        uncommitted_tuples = [  ]
+        upload_empty_tuples = [  ]
+        token = nil
+        finished = false
+        filter = "#{:PartitionKey} eq '#{@@configuration[:azure_storage_blob_prefix]}-#{state}'"
+
+        # should exit thread after fetching data from table, and submit recovery, the loop is only for case of failure
+        until finished || stopped? do
+          entities = blob.log_to_table_query( storage_account_name, filter, token )
+          if entities
+            token = entities.continuation_token
+
+            if :committed == state
+              entities.each do |entity|
+                State.instance.inc_pending_notifications
+                tuple = blob.table_entity_to_tuple( entity.properties )
+                @@failed_on_notification_endpoint_retry_Q << tuple
+              end
+
+            elsif :uploading == state
+              # first tuples are collected, before send to queues, to make sure blob states don't change in between
+              entities.each do |entity|
+                typed_tuple = nil
+                until typed_tuple || stopped?
+                  typed_tuple = blob.update_commited_or_uncommited_list( entity.properties )
+                  Stud.stoppable_sleep(60, 1) { stopped? } unless typed_tuple
+                end
+                next if stopped?
+
+                if typed_tuple[:committed]
+                  committed_tuples << typed_tuple[:committed]
+                elsif typed_tuple[:uncommitted]
+                  uncommitted_tuples << typed_tuple[:uncommitted]
+                else
+                  upload_empty_tuples << typed_tuple[:upload_empty]
+                end
+              end
+            end
+
+            next if token
+            committed_tuples.each do |tuple|
+              State.instance.inc_pending_commits
+              @@failed_on_log_to_table_retry_Qs[storage_account_name] << tuple
+            end
+            uncommitted_tuples.each do |tuple|
+              State.instance.inc_pending_commits
+              @@failed_on_commit_retry_Qs[storage_account_name] << tuple
+            end
+            upload_empty_tuples.each do |tuple|
+              @@failed_on_log_to_table_retry_Qs[storage_account_name] << tuple
+            end
+            finished = true
+          else
+            Stud.stoppable_sleep(60, 1) { stopped? }
+          end
+        end
+        @@logger.info { "exit table recovery thread, storage: #{storage_account_name}, state: #{state}, entities: #{entities ? entities.length : nil}" }
+      end
+    end
+
+    def self.state_on? ( storage_account_name, blob, failure_reason )
+      if blob
+        if :io_failure == failure_reason
+          @@endpoint_state_on ||= blob.test_notification_endpoint( @@configuration[:storage_account_name_key][0][0] )
+        else
+          Clients.instance.storage_account_state_on?( storage_account_name )
+        end
+      elsif storage_account_name
+        Clients.instance.storage_account_state_on?( storage_account_name )
+      else
+        Clients.instance.storage_account_state_on?
+      end
+    end
+
+    def self.storage_recovery_thread( storage_account_name, queue, method, failure_reason )
+      # a threads, per storage  account name, that retries failed blob commits / notification / table updates
+      Thread.new( storage_account_name, queue, method, failure_reason ) do |storage_account_name, queue, method, failure_reason|
+        blob = Blob.new if :notify == method
+        semaphore = Mutex.new
+        action = {:method => method, :semaphore => semaphore, :counter => 0 }
+        loop do
+          tuple ||= queue.pop
+          until state_on?( storage_account_name, blob, failure_reason ) do sleep( 1 ) end
+
+          not_busy = nil
+          semaphore.synchronize {
+            not_busy = action[:counter] += 1 if 10 > action[:counter]
+          }
+          if not_busy
+            Thread.new( action, tuple ) do |action, tuple|
+              Blob.new.send( action[:method], tuple )
+              action[:semaphore].synchronize {
+                action[:counter] -= 1
+              }
+            end
+            tuple = nil # release for GC
+          else
+            Stud.stoppable_sleep(60, 1) { 10 > action[:counter] }
+            next
+          end 
+        end
+      end
+    end
+
+    def self.validate_endpoint
+      io = Blob.new
+      raise ConfigurationError, "Failed to access application insights #{@@configuration[:notification_endpoint]}, due to error #{io.last_io_exception.inspect}" unless io.test_notification_endpoint( @@configuration[:storage_account_name_key][0][0] )
+    end
+
+    def self.validate_storage
+      io = Blob.new
+      @@configuration[:storage_account_name_key].each do |storage_account_name, storage_account_keys|
+        raise ConfigurationError, "Failed access azure storage account #{storage_account_name}, due to error #{io.last_io_exception.inspect}" unless io.test_storage( storage_account_name )
+      end
+    end
+
+
+    def initialize ( channel = nil, id = nil , no_queue = false )
+      @uploaded_block_ids = [  ]
+      @uploaded_block_numbers = [  ]
+      @uploaded_bytesize = 0
+      @uploaded_events_count = 0
+      @max_tries = @@io_max_retries + 1
+      @sub_state = :none
+
+      if channel
+        @id = id
+        @intrumentation_key = channel.intrumentation_key
+        @table_id = channel.table_id
+        @blob_max_delay = channel.blob_max_delay
+
+        @event_format_ext = channel.event_format_ext
+
+        unless no_queue
+
+          @io_queue = Queue.new
+          @timer = Timer.new
+      
+          # create a thread that handles the IO of the blob
+          Thread.new do
+            next_block = nil
+            loop do
+              block_to_upload = nil # release reference to resource for GC
+              block_to_upload = next_block || @io_queue.pop
+              next_block = nil
+
+              if :trigger == @timer.state
+                next_block = block_to_upload unless :wakeup == block_to_upload
+                block_to_upload = :timeout
+                to_commit = :commit
+
+              elsif :close == block_to_upload
+                to_commit = :commit
+
+              # ignore :trigger as they are only to casue check timeout
+              elsif :wakeup == block_to_upload # ignore :wakeup
+                next
+
+              else
+                while @io_queue.length > 0
+                  next_block = @io_queue.pop
+                  next if :wakeup == next_block # ignore :wakeup
+                  break if :close == next_block
+                  break if blob_full?( next_block )
+                  break unless block_to_upload.concat( next_block )
+                  next_block = nil 
+                end
+              end
+
+              unless to_commit
+                @timer.set( block_to_upload.oldest_event_time + @blob_max_delay, nil ) {|object| @io_queue << :wakeup if 0 == @io_queue.length } if blob_empty?
+                to_commit = :commit if blob_full?
+                upload( block_to_upload, to_commit)
+                block_to_upload = nil # release reference to resource for GC
+              else
+                commit unless @uploaded_block_ids.empty?
+              end
+
+              if to_commit
+                to_commit = nil
+                @uploaded_block_ids = [  ]
+                @timer.cancel
+                break if :close == block_to_upload
+              end
+            end
+          end
+        end
+
+      end
+
+    end
+
+    # close blob. It will finish whatever was already on the queue, and if necessary commit
+    # called on shutdown
+    def close
+      @io_queue << :close
+    end
+
+    def blob_full? ( next_block = nil )
+      if next_block
+        BLOB_MAX_BLOCKS < @uploaded_block_ids.length + 1 || @@blob_max_events < @uploaded_events_count + next_block.events_count || @@blob_max_bytesize < @uploaded_bytesize  + next_block.bytesize
+      else
+        BLOB_MAX_BLOCKS <= @uploaded_block_ids.length || @@blob_max_events <= @uploaded_events_count || @@blob_max_bytesize <= @uploaded_bytesize
+      end
+    end 
+
+    def blob_empty?
+      @uploaded_block_ids.empty?
+    end
+
+    def queue_empty?
+      @io_queue.length == 0 if @io_queue
+    end
+
+
+    def queue_size
+      @io_queue.length
+    end
+
+    def clear_state
+      @action = nil
+      @storage_account_name = nil
+      @container_name = nil
+      @blob_name = nil
+      @uploaded_block_ids = [  ]
+      @uploaded_block_numbers = [  ]
+      @uploaded_events_count = 0
+      @uploaded_bytesize = 0
+      @oldest_event_time = nil
+    end
+
+    def table_entity_to_tuple( options = {} )
+      [ options[:start_time.to_s] || Time.now.utc, options[:action.to_s], options[:intrumentation_key.to_s], options[:table_id.to_s], 
+        options[:storage_account_name.to_s], options[:container_name.to_s], options[:blob_name.to_s], 
+        eval( options[:uploaded_block_ids.to_s] ), eval( options[:uploaded_block_numbers.to_s] ), 
+        options[:uploaded_events_count.to_s] || 0, options[:uploaded_bytesize.to_s] || 0, options[:oldest_event_time.to_s] || Time.now.utc,
+        options[:event_format_ext.to_s], options[:blob_max_delay.to_s] || 0,
+        options[:log_state.to_s].to_sym, (options[:sub_state.to_s] || :none).to_sym
+      ]
+    end
+
+    def state_to_tuple
+      [ @start_time || Time.now.utc, @action, @intrumentation_key, @table_id, 
+        @storage_account_name, @container_name, @blob_name, 
+        @uploaded_block_ids, @uploaded_block_numbers, 
+        @uploaded_events_count, @uploaded_bytesize, @oldest_event_time,
+        @event_format_ext, @blob_max_delay,
+        @log_state, @sub_state
+      ]
+    end
+
+    def tuple_to_state ( tuple )
+      ( @start_time, @action, @intrumentation_key, @table_id, 
+        @storage_account_name, @container_name, @blob_name, 
+        @uploaded_block_ids, @uploaded_block_numbers, 
+        @uploaded_events_count, @uploaded_bytesize, @oldest_event_time,
+        @event_format_ext, @blob_max_delay,
+        @log_state, @sub_state) = tuple
+    end
+
+    def state_to_table_entity
+      { :start_time => @start_time, :intrumentation_key => @intrumentation_key, :table_id => @table_id, 
+        :storage_account_name => @storage_account_name, :container_name => @container_name, :blob_name => @blob_name, 
+        :uploaded_block_ids => @uploaded_block_ids.to_s, :uploaded_block_numbers => @uploaded_block_numbers.to_s, 
+        :uploaded_events_count => @uploaded_events_count, :uploaded_bytesize => @uploaded_bytesize, :oldest_event_time => @oldest_event_time,
+        :log_state => @log_state, :sub_state => @sub_state
+      }
+    end
+
+
+    def test_storage_recover
+      proc do |reason, e| @recovery = :ok if :container_exist == reason || :create_container == reason end
+    end
+
+
+    def test_storage ( storage_account_name )
+      @storage_account_name = storage_account_name
+      @action = :test_storage
+      @max_tries = 1
+      @force_client = true # to enable get a client even if all storage_accounts marked dead
+      @recoverable = [ :invalid_storage_key ]
+      storage_io_block( test_storage_recover ) {
+        if @recovery.nil? || :invalid_storage_key == @recovery
+          container_name = "logstash-test-container"
+          @info = "#{@action} #{@storage_account_name}"
+          @client.blobClient.create_container( container_name ) unless @@configuration[:disable_blob_upload]
+        end
+      }
+    end
+
+    def test_notification_endpoint_recover
+      proc do |reason, e| @recovery = :ok if :invalid_intrumentation_key == reason || :invalid_table_id == reason end
+    end
+
+    def test_notification_endpoint( storage_account_name )
+      @storage_account_name = storage_account_name
+      @action = :test_notification_endpoint
+      @max_tries = 1
+      @force_client = true # to enable get a client even if all storage_accounts marked dead
+      @recoverable = [  ]
+      success = storage_io_block( test_notification_endpoint_recover ) {
+        if @recovery.nil?
+          @container_name = "logstash-test-container"
+          @blob_name = "logstash-test-blob"
+          @table_id = GUID_NULL
+          @intrumentation_key = GUID_NULL
+          @info = "#{@action}"
+          set_blob_sas_url
+          payload = create_payload
+          post_notification( @client.notifyClient, payload )
+        end
+      }
+      sleep( 30 ) unless success
+      success
+    end
+
+
+    def notify_recover
+      proc do |reason, e|
+        if :notify_failed_blob_not_accessible == reason
+          @sub_state = reason
+          @@failed_on_notify_retry_Qs[@storage_account_name] << state_to_tuple
+        elsif :invalid_intrumentation_key == reason || :invalid_table_id == reason
+          @sub_state = reason
+          Channels.instance.channel( @intrumentation_key, @table_id ).failed_on_notify_retry_Q << state_to_tuple
+
+        else
+          @@endpoint_state_on = false
+          if :notify_failed_blob_not_accessible == @sub_state
+            @@failed_on_notify_retry_Qs[@storage_account_name] << state_to_tuple
+          elsif :invalid_intrumentation_key == @sub_state || :invalid_table_id == @sub_state
+            Channels.instance.channel( @intrumentation_key, @table_id ).failed_on_notify_retry_Q << state_to_tuple
+          else
+            @@failed_on_notification_endpoint_retry_Q << state_to_tuple
+          end
+        end
+      end
+    end
+
+    def notify ( tuple = nil )
+      tuple_to_state( tuple ) if tuple
+      @action = :notify
+      @force_client = true # to enable get a client even if all storage_accounts marked dead
+      @recoverable = [ :notify_failed_blob_not_accessible, :io_failure, :service_unavailable ]
+      success = storage_io_block( notify_recover ) {
+        set_blob_sas_url
+        payload = create_payload
+        @@logger.debug { "notification payload: #{payload}" }
+        @info = "#{@action.to_s} #{@storage_account_name}/#{@container_name}/#{@blob_name}, events: #{@uploaded_events_count}, size: #{@uploaded_bytesize}, blocks: #{@uploaded_block_numbers}, delay: #{Time.now.utc - @oldest_event_time}, blob_sas_url: #{@blob_sas_url}"
+
+        # assume that exceptions can be raised due to this method:
+        post_notification( @client.notifyClient, payload ) unless @@configuration[:disable_notification]
+        @log_state = :notified
+      }
+      log_to_table_update if success
+    end
+
+    CREATE_EXIST_ERRORS = { :container => [ :create_container, :container_exist ], :table => [ :create_table, :table_exist ] }
+    def create_exist_recovery( type, name = nil )
+      prev_info = @info
+      if CREATE_EXIST_ERRORS[type][0] == @recovery
+        name ||= ( :table == type ? @@state_table_name : @container_name )
+        @info = "create #{type} #{@storage_account_name}/#{name}"
+
+        # assume that exceptions can be raised due to this method:
+        yield name
+        @@logger.info { "Successed to #{@info}" }
+        @info = prev_info
+      elsif CREATE_EXIST_ERRORS[type][1] == @recovery
+        @@logger.info { "Successed (already exist) to #{@info}" }
+        @info = prev_info
+      end
+    end
+
+    def create_table_exist_recovery
+      create_exist_recovery( :table ) { |name| @client.tableClient.create_table( name ) }
+    end
+
+    def create_container_exist_recovery
+      create_exist_recovery( :container ) { |name| @client.blobClient.create_container( name ) }
+    end
+
+    # return true on success
+    def log_to_table_insert
+      @action = :log_to_table_insert
+      @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :table_exist, :create_table, :table_busy, :entity_exist ]
+      @info  = "#{@action} #{@log_state} #{@storage_account_name}/#{@container_name}/#{@blob_name}"
+      success =  storage_io_block( :uploading == @log_state ? proc do |reason, e| end : log_to_table_update_recover ) {
+        create_table_exist_recovery
+        if :entity_exist == @recovery
+          raise NotRecoverableError if :uploading == @log_state
+        else
+          entity_values = state_to_table_entity
+          entity_values[:PartitionKey] = "#{@@configuration[:azure_storage_blob_prefix]}-#{@log_state}"
+          entity_values[:RowKey] = @blob_name
+          @client.tableClient.insert_entity( @@state_table_name, entity_values )
+        end
+      }
+    end
+
+    def log_to_table_update_recover
+      proc do |reason, e| @@failed_on_log_to_table_retry_Qs[@storage_account_name] << state_to_tuple end
+    end
+
+    def log_to_table_update ( tuple = nil )
+      tuple_to_state( tuple ) if tuple
+      if :uploading == @log_state
+        log_to_table_delete
+      elsif :committed == @log_state
+        if log_to_table_insert && log_to_table_delete( nil, :uploading )
+          State.instance.dec_pending_commits
+          State.instance.inc_pending_notifications
+          @@failed_on_notification_endpoint_retry_Q << state_to_tuple
+        end
+      elsif :notified == @log_state
+        if (!@@save_notified_blobs_records || log_to_table_insert) && log_to_table_delete( nil, :committed ) 
+          State.instance.dec_pending_notifications
+        end
+      end
+    end
+
+
+    # retturn tru on success
+    def log_to_table_delete ( tuple = nil, state = nil )
+      tuple_to_state( tuple ) if tuple
+      state ||= @log_state
+      @action = :log_to_table_delete
+      @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :table_exist, :create_table, :table_busy, :create_resource ]
+      @info  = "#{@action} #{state} #{@storage_account_name}/#{@container_name}/#{@blob_name}"
+
+      success =  storage_io_block( log_to_table_update_recover ) {
+        create_table_exist_recovery
+        if :create_resource == @recovery
+          @@logger.info { "Note: delete entity failed, already deleted, #{@info}, state: #{state}, log_state: #{@log_state}" }
+        else
+          @client.tableClient.delete_entity( @@state_table_name, "#{@@configuration[:azure_storage_blob_prefix]}-#{state}", @blob_name )
+        end
+      }
+    end
+
+    # return entities
+    def log_to_table_query ( storage_account_name, filter , token )
+      @storage_account_name = storage_account_name
+
+      @action = :log_to_table_query
+      @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :table_exist, :create_table, :table_busy ]
+      @info  = "#{@action} #{@storage_account_name}/#{@@state_table_name}"
+
+      entities = nil
+      success =  storage_io_block( proc do |reason, e| end ) {
+        create_table_exist_recovery
+        options = { :filter => filter }
+        options[:continuation_token] = token if token
+        entities = @client.tableClient.query_entities( @@state_table_name, options )
+      }
+      entities
+    end
+
+    def commit_recover
+      proc do |reason, e| @@failed_on_commit_retry_Qs[@storage_account_name] << state_to_tuple end
+    end
+
+    def commit ( tuple = nil )
+      tuple_to_state( tuple ) if tuple
+
+      unless @uploaded_block_ids.empty?
+        @action = :commit
+        @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable ]
+        success =  storage_io_block( commit_recover ) {
+          @info = "#{@action.to_s} #{@storage_account_name}/#{@container_name}/#{@blob_name}, events: #{@uploaded_events_count}, size: #{@uploaded_bytesize}, blocks: #{@uploaded_block_numbers}, delay: #{Time.now.utc - @oldest_event_time}"
+          # assume that exceptions can be raised due to this method:
+          @client.blobClient.commit_blob_blocks( @container_name, @blob_name, @uploaded_block_ids ) unless @@configuration[:disable_blob_upload]
+          @log_state = :committed
+        }
+        # next stage
+        log_to_table_update if success
+      end
+    end
+
+
+    def upload_recover
+      proc do |reason, e|
+        unless @uploaded_block_ids.empty?
+          info1 = "#{:commit} #{@storage_account_name}/#{@container_name}/#{@blob_name}, events: #{@uploaded_events_count}, size: #{@uploaded_bytesize}, blocks: #{@uploaded_block_numbers}, delay: #{Time.now.utc - @oldest_event_time}"
+          @@logger.error { "Pospone to #{info1} (; retry later, error: #{e.inspect}" }
+          @@failed_on_commit_retry_Qs[@storage_account_name] << state_to_tuple
+          @uploaded_block_ids = [  ]
+        end
+        unless :io_all_dead == reason
+          @recovery = :invalid_storage_account
+        else 
+          Channels.instance.channel( @intrumentation_key, @table_id ).failed_on_upload_retry_Q << @block_to_upload
+          @block_to_upload = nil
+        end
+      end
+    end
+
+    def upload ( block, to_commit = nil )
+      @storage_account_name = nil if @uploaded_block_ids.empty?
+      @block_to_upload = block
+      block = nil # remove reference for GC
+      exclude_storage_account_names = [  ]
+      begin
+        if @uploaded_block_ids.empty?
+          @log_state = :uploading
+          @uploaded_block_numbers = [  ]
+          @uploaded_bytesize = 0
+          @uploaded_events_count = 0
+          @oldest_event_time = nil
+
+          # remove record of previous upload that failed
+          if @storage_account_name
+            exclude_storage_account_names << @storage_account_name
+            @@failed_on_log_to_table_retry_Qs[@storage_account_name] << state_to_tuple
+          end
+          set_conatainer_and_blob_names
+          @storage_account_name = Clients.instance.get_random_active_storage( exclude_storage_account_names )
+          unless @storage_account_name
+            upload_recover.call( :io_all_dead, nil )
+            return false
+          end
+          raise UploadRetryError unless log_to_table_insert
+        end
+
+        @action = :upload
+        @block_info = "blocks: #{@block_to_upload.block_numbers}, events: #{@block_to_upload.events_count}, size: #{@block_to_upload.bytes.length}"
+        @info = "#{@action} #{@storage_account_name}/#{@container_name}/#{@blob_name}, #{@block_info}, commitId: [\"#{100001 + @uploaded_block_ids.length}\"]"
+        @recoverable = [ :invalid_storage_key, :invalid_storage_account, :io_failure, :service_unavailable, :container_exist, :create_container ]
+
+        success = storage_io_block( upload_recover ) {
+          create_container_exist_recovery
+          block_id = "#{100001 + @uploaded_block_ids.length}"
+
+          # assume that exceptions can be raised due to this method:
+          @client.blobClient.put_blob_block( @container_name, @blob_name, block_id, @block_to_upload.bytes ) unless @@configuration[:disable_blob_upload]
+
+          # upload success
+          first_block_in_blob = @uploaded_block_ids.empty?
+          @uploaded_block_ids << [ block_id ]
+          @uploaded_block_numbers.concat( @block_to_upload.block_numbers )
+          @uploaded_bytesize += @block_to_upload.bytes.length
+          @uploaded_events_count += @block_to_upload.events_count
+          @oldest_event_time ||= @block_to_upload.oldest_event_time
+
+          # release memory
+          bytesize = @block_to_upload.bytesize
+          @block_to_upload.dispose
+          @block_to_upload = nil
+          State.instance.inc_pending_commits if first_block_in_blob
+          State.instance.dec_upload_bytesize( bytesize )
+          Telemetry.instance.track_event("uploading", {:properties => state_to_table_entity})
+        }
+
+        raise UploadRetryError if :invalid_storage_account == @recovery
+        commit if success && to_commit
+      rescue UploadRetryError
+        @recovery = nil
+        retry
+      end
+    end
+
+    def update_commited_or_uncommited_list( table_entity )
+      tuple = table_entity_to_tuple( table_entity )
+
+      tuple_to_state( tuple )
+      @action = :list_blob_blocks
+      @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :container_exist, :create_container, :create_blob ]
+      list_blob_blocks = nil
+      success =  storage_io_block( proc do |reason, e| end ) {
+        @info = "#{@action} #{@storage_account_name}/#{@container_name}/#{@blob_name}"
+
+        create_container_exist_recovery
+        if :create_blob == @recovery
+          list_blob_blocks = { :uncommitted => [  ], :committed => [  ] }
+        else
+          list_blob_blocks = @client.blobClient.list_blob_blocks( @container_name, @blob_name, { :blocklist_type => :all } ) unless :create_blob == @recovery
+        end
+      }
+
+      if list_blob_blocks
+        blocks = ( list_blob_blocks[:uncommitted].empty? ? list_blob_blocks[:committed] : list_blob_blocks[:uncommitted] )
+        blocks.each do |block|
+          @uploaded_block_ids << [ block.name ]
+          @uploaded_bytesize += block.size
+        end
+        type = ( blocks.empty? || 0 == @uploaded_bytesize ? :upload_empty : blocks[0].type )
+
+        @log_state = :committed if :committed == type
+        { type => state_to_tuple }
+      else
+        nil
+      end
+    end
+
+
+    def << ( block )
+      @io_queue << block
+    end
+
+
+    private
+
+
+    def storage_io_block( recover_later_proc, valid_recovery = nil )
+      @recovery = nil
+      @try_count = 1
+
+      begin
+        @client ||= Client.new( @storage_account_name, @force_client )
+        yield
+        disabled = :notify == @action ? @@configuration[:disable_notification] : @@configuration[:disable_blob_upload]
+        @@logger.info { "Successed to #{disabled ? 'DISABLED ' : ''}#{@info}" }
+        true
+
+      rescue TypeError
+        raise
+
+      rescue StandardError => e
+        @last_io_exception = e
+        @recovery = nil
+        retry if recover_retry?( e, recover_later_proc )
+        false
+
+      ensure
+        @client = @client.dispose if @client
+      end
+    end
+
+
+    def recover_retry? ( e, recover_later_proc )
+      # http error, probably server error
+      if e.is_a?( Azure::Core::Http::HTTPError )
+
+        if 404 == e.status_code && "ContainerNotFound" == e.type
+          @recovery = :create_container
+
+        elsif 404 == e.status_code && "TableNotFound" == e.type
+          @recovery = :create_table
+
+        elsif 404 == e.status_code && "BlobNotFound" == e.type
+          @recovery = :create_blob
+          
+        elsif 404 == e.status_code && "ResourceNotFound" == e.type
+          @recovery = :create_resource
+
+        elsif 409 == e.status_code && "ContainerAlreadyExists" == e.type
+          @recovery = :container_exist
+
+        elsif 409 == e.status_code && "BlobAlreadyExists" == e.type
+          @recovery = :blob_exist
+
+        elsif 409 == e.status_code && "TableAlreadyExists" == e.type
+          @recovery = :table_exist
+
+        elsif 409 == e.status_code && "TableBeingDeleted" == e.type
+          @recovery = :table_busy
+
+        elsif 409 == e.status_code && "EntityAlreadyExists" == e.type
+          @recovery = :entity_exist
+
+        elsif 403 == e.status_code && "AuthenticationFailed" == e.type
+          @recovery = :invalid_storage_key
+
+        elsif 403 == e.status_code  && "Unknown" == e.type && e.description.include?("Blob does not exist or not accessible.")
+          @recovery = :notify_failed_blob_not_accessible
+
+        elsif 400 == e.status_code  && "Unknown" == e.type && e.description.include?("Invalid instrumentation key")
+          @recovery = :invalid_intrumentation_key
+
+        elsif 500 == e.status_code  && "Unknown" == e.type && e.description.include?("Processing error")
+          @recovery = :notification_process_down
+
+        elsif 503 == e.status_code
+          @recovery = :service_unavailable
+        elsif 404 == e.status_code
+          @recovery = :create_resource
+        elsif 403 == e.status_code
+          # todo, came from updating the log_table, how to hnadle this
+          @recovery = :access_denied
+        else
+          puts "\n>>>> HTTP error - #{e.inspect} <<<<\n"
+          @recovery = :http_unknown
+          raise e if  @@configuration[:stop_on_unknown_io_errors]
+        end
+
+      # communication error
+      elsif e.is_a?( Faraday::ClientError )
+        @recovery = :io_failure
+
+      # communication error
+      elsif e.is_a?( IOError )
+        @recovery = :io_failure
+
+      # all storage accounts are dead, couldn't get client (internal exception)
+      elsif e.is_a?( StorageAccountsOffError )
+        @recovery = :io_all_dead
+
+      # all storage accounts are dead, couldn't get client (internal exception)
+      elsif e.is_a?( NotRecoverableError )
+        @recovery = :not_recoverable
+
+      elsif e.is_a?( NameError ) && e.message.include?( "uninitialized constant Azure::Core::Auth::Signer::OpenSSL" )
+        sleep( 1 )
+        @recovery = :io_failure
+
+      elsif e.is_a?( NameError ) && e.message.include?( "uninitialized constant Azure::Storage::Auth::SharedAccessSignature" )
+        sleep( 1 )
+        @recovery = :io_failure
+
+      else
+        # UNKNOWN error - #<NameError: uninitialized constant Azure::Core::Auth::Signer::OpenSSL>
+        puts "\n>>>> UNKNOWN error - #{e.inspect} <<<<\n"
+        raise e
+
+      end
+
+      reason = @recovery
+      if @recovery && @recoverable.include?( @recovery )
+        case @recovery
+        when :container_exist, :table_exist, :entity_exist, :create_container, :create_table
+          # ignore log error
+          # @@logger.error { "Failed to #{@info} ;( recovery: continue, error: #{e.inspect}" }
+
+        when :invalid_storage_key, :notify_failed_blob_not_accessible
+          if @client.switch_storage_account_key!
+            @@logger.error { "Failed to #{@info} ;( recovery: switched to secondary storage key, error: #{e.inspect}" }
+          else
+            @client = @client.dispose( :auth_to_storage_failed ) if @client && :invalid_storage_key == @recovery 
+            @recovery = nil
+          end
+
+        when :table_busy
+          @client = @client.dispose if @client
+          sleep( @@io_retry_delay )
+          @@logger.error { "Failed to #{@info} ;( recovery: retry, error: #{e.inspect}" }
+
+        when :io_failure, :service_unavailable, :notification_process_down, :invalid_intrumentation_key, :invalid_table_id
+          if @try_count < @max_tries
+            @client = @client.dispose if @client
+            sleep( @@io_retry_delay )
+            @@logger.error { "Failed to #{@info} ;( recovery: retry, try #{@try_count} / #{@max_tries}, error: #{e.inspect}" }
+            @try_count += 1
+          else
+            if :invalid_intrumentation_key == @recovery
+              Channels.instance.mark_invalid_intrumentation_key( @intrumentation_key )
+            elsif :invalid_table_id == @recovery
+              Channels.instance.mark_invalid_table_id( @table_id )
+            elsif :io_failure == @recovery || ( :service_unavailable == @recovery && :notify != @action )
+              @client = @client.dispose( :io_to_storage_failed ) if @client
+            end
+            @recovery = nil
+          end
+        end
+      else
+        @recovery = nil
+      end
+
+      if @recovery
+        true
+      else
+        recover_later_proc.call( reason, e )
+        @@logger.error { "Failed to #{@info} ; retry later, error: #{e.inspect}" } unless :ok == @recovery
+        :ok == @recovery
+      end
+
+      # Blob service error codes - msdn.microsoft.com/en-us/library/azure/dd179439.aspx
+      # ConnectionFailed - problem with connection
+      # ParsingError - problem with request/response payload
+      # ResourceNotFound, SSLError, TimeoutError
+    end
+
+    def set_conatainer_and_blob_names
+      time_utc = Time.now.utc
+      id = @id.to_s.rjust(4, "0")
+      strtime = time_utc.strftime( "%F" )
+      @container_name = "#{@@configuration[:azure_storage_container_prefix]}-#{strtime}"
+
+      strtime = time_utc.strftime( "%F-%H-%M-%S-%L" )
+      @blob_name = "#{@@configuration[:azure_storage_blob_prefix]}_ikey-#{@intrumentation_key}_table-#{@table_id}_id-#{id}_#{strtime}.#{@event_format_ext}"
+    end
+
+
+    def create_payload
+      notification_hash = {
+        :data => {
+          :baseType => DATA_BASE_TYPE,
+          :baseData => {
+            :ver           => BASE_DATA_REQUIRED_VERSION,
+            :blobSasUri    => @blob_sas_url.to_s,
+            :sourceName    => @table_id,
+            :sourceVersion => @@configuration[:notification_version].to_s
+          }
+        }, 
+        :ver  => @@configuration[:notification_version], 
+        :name => REQUEST_NAME,
+        :time => Time.now.utc.iso8601,
+        :iKey => @intrumentation_key
+      }
+      notification_hash.to_json
+    end 
+
+
+    def post_notification ( http_client, body )
+      request = Azure::Core::Http::HttpRequest.new( :post, @@configuration[:notification_endpoint], { :body => body, :client => http_client } )
+      request.headers['Content-Type'] = 'application/json; charset=utf-8'
+      request.headers['Accept'] = 'application/json'
+      @@logger.debug { "send notification : \n    endpoint: #{@@configuration[:notification_endpoint]}\n    body : #{body}" }
+      response = request.call
+    end 
+
+
+    def set_blob_sas_url
+      blob_url ="https://#{@storage_account_name}.blob.core.windows.net/#{@container_name}/#{@blob_name}"
+      options_and_constrains = {:permissions => "r", :resource => "b", :expiry => ( Time.now.utc + @@configuration[:blob_access_expiry_time] ).iso8601 }
+      @blob_sas_url = @client.storage_auth_sas.signed_uri( URI( blob_url ), options_and_constrains )
+    end
+
+  end
+end