logstash-output-application_insights 0.1.3 → 0.1.4

data/lib/logstash/outputs/application_insights/channel.rb

@@ -22,7 +22,7 @@
 class LogStash::Outputs::Application_insights
   class Channel
 
-    attr_reader :intrumentation_key
+    attr_reader :instrumentation_key
     attr_reader :table_id
     attr_reader :failed_on_upload_retry_Q
     attr_reader :failed_on_notify_retry_Q
@@ -31,14 +31,14 @@ class LogStash::Outputs::Application_insights
 
     public
 
-    def initialize ( intrumentation_key, table_id )
+    def initialize ( instrumentation_key, table_id )
       @closing = false
       configuration = Config.current
 
       @logger = configuration[:logger]
 
-      @logger.debug { "Create a new channel, intrumentation_key / table_id : #{intrumentation_key} / #{table_id}" }
-      @intrumentation_key = intrumentation_key
+      @logger.debug { "Create a new channel, instrumentation_key / table_id : #{instrumentation_key} / #{table_id}" }
+      @instrumentation_key = instrumentation_key
       @table_id = table_id
       set_table_properties( configuration )
       @semaphore = Mutex.new

data/lib/logstash/outputs/application_insights/channels.rb

@@ -29,11 +29,11 @@ class LogStash::Outputs::Application_insights
 
       @logger = configuration[:logger]
 
-      @intrumentation_key_table_id_db = {}
+      @instrumentation_key_table_id_db = {}
       @channels = [  ]
       @create_semaphore = Mutex.new
 
-      @default_intrumentation_key = configuration[:intrumentation_key]
+      @default_instrumentation_key = configuration[:instrumentation_key]
       @default_table_id = configuration[:table_id]
       @tables = configuration[:tables]
 
@@ -46,29 +46,29 @@ class LogStash::Outputs::Application_insights
 
     def receive ( event, encoded_event )
       if LogStash::SHUTDOWN == event
-        @logger.info { "received a LogStash::SHUTDOWN event, start shutdown" }
+        @logger.info { "received a LogStash::SHUTDOWN event" }
 
       elsif LogStash::FLUSH == event
-        @logger.info { "received a LogStash::FLUSH event, start shutdown" }
-      end
-
-      table_id = event[METADATA_FIELD_TABLE_ID] || event[FIELD_TABLE_ID] || @default_table_id
-      intrumentation_key = event[METADATA_FIELD_INSTRUMENTATION_KEY] || event[FIELD_INSTRUMENTATION_KEY] || ( @tables[table_id][TABLE_PROPERTY_INSTRUMENTATION_KEY] if @tables[table_id] ) || @default_intrumentation_key
+        @logger.info { "received a LogStash::FLUSH event" }
+      else
+        table_id = event[METADATA_FIELD_TABLE_ID] || event[FIELD_TABLE_ID] || @default_table_id
+        instrumentation_key = event[METADATA_FIELD_INSTRUMENTATION_KEY] || event[FIELD_INSTRUMENTATION_KEY] || ( @tables[table_id][:instrumentation_key] if @tables[table_id] ) || @default_instrumentation_key
 
-      @flow_control.pass_or_wait
-      channel( intrumentation_key, table_id ) << event
+        @flow_control.pass_or_wait
+        channel( instrumentation_key, table_id ) << event
+      end
     end
 
 
-    def channel ( intrumentation_key, table_id )
+    def channel ( instrumentation_key, table_id )
       begin
-        dispatch_channel( intrumentation_key, table_id )
+        dispatch_channel( instrumentation_key, table_id )
 
       rescue NoChannelError
         begin
-          create_channel( intrumentation_key, table_id )
+          create_channel( instrumentation_key, table_id )
         rescue ChannelExistError # can happen due to race conditions
-          dispatch_channel( intrumentation_key, table_id )
+          dispatch_channel( instrumentation_key, table_id )
         end
       end
     end
@@ -89,13 +89,13 @@ class LogStash::Outputs::Application_insights
     private
 
     # return channel
-    def dispatch_channel ( intrumentation_key, table_id )
+    def dispatch_channel ( instrumentation_key, table_id )
       begin
-        channel = @intrumentation_key_table_id_db[intrumentation_key][table_id]
-        channel.intrumentation_key     # don't remove it, it is to emit an exception in case channel not created yet'
+        channel = @instrumentation_key_table_id_db[instrumentation_key][table_id]
+        channel.instrumentation_key     # don't remove it, it is to emit an exception in case channel not created yet'
         channel
       rescue => e
-        raise NoChannelError if @intrumentation_key_table_id_db[intrumentation_key].nil? || @intrumentation_key_table_id_db[intrumentation_key][table_id].nil?
+        raise NoChannelError if @instrumentation_key_table_id_db[instrumentation_key].nil? || @instrumentation_key_table_id_db[instrumentation_key][table_id].nil?
         @logger.error { "Channel dispatch failed - error: #{e.inspect}" }
         raise e
       end 
@@ -103,12 +103,12 @@ class LogStash::Outputs::Application_insights
 
 
     # return channel
-    def create_channel ( intrumentation_key, table_id )
+    def create_channel ( instrumentation_key, table_id )
       @create_semaphore.synchronize {      
-        raise ChannelExistError if @intrumentation_key_table_id_db[intrumentation_key] && @intrumentation_key_table_id_db[intrumentation_key][table_id]
-        @intrumentation_key_table_id_db[intrumentation_key] ||= {}
-        channel = Channel.new( intrumentation_key, table_id )
-        @intrumentation_key_table_id_db[intrumentation_key][table_id] = channel
+        raise ChannelExistError if @instrumentation_key_table_id_db[instrumentation_key] && @instrumentation_key_table_id_db[instrumentation_key][table_id]
+        @instrumentation_key_table_id_db[instrumentation_key] ||= {}
+        channel = Channel.new( instrumentation_key, table_id )
+        @instrumentation_key_table_id_db[instrumentation_key][table_id] = channel
         @channels << channel
         channel
       }
@@ -122,7 +122,7 @@ class LogStash::Outputs::Application_insights
       end
     end
 
-    def mark_invalid_intrumentation_key ( intrumentation_key )
+    def mark_invalid_instrumentation_key ( instrumentation_key )
       # TODO should go to lost and found container
     end
 

data/lib/logstash/outputs/application_insights/client.rb

@@ -98,12 +98,18 @@ class LogStash::Outputs::Application_insights
 
     def set_current_storage_account_client
       configuration = Config.current
-      alt_storage_access_key = @storage_account[:keys][@current_storage_account_key_index]
-      options = { :storage_account_name => @storage_account_name, :storage_access_key => alt_storage_access_key }
+      storage_access_key = @storage_account[:keys][@current_storage_account_key_index]
+
+      options = { 
+        :storage_account_name => @storage_account_name, 
+        :storage_access_key => storage_access_key,
+        :storage_blob_host => "https://#{@storage_account_name}.#{:blob}.#{configuration[:azure_storage_host_suffix]}",
+        :storage_table_host => "https://#{@storage_account_name}.#{:table}.#{configuration[:azure_storage_host_suffix]}"
+      }
       options[:ca_file] = configuration[:ca_file] unless configuration[:ca_file].empty?
-      @current_azure_storage_client = Azure::Storage::Client.new( options )
 
-      @current_azure_storage_auth_sas = Azure::Storage::Auth::SharedAccessSignature.new( @storage_account_name, alt_storage_access_key )
+      @current_azure_storage_client = Azure::Storage::Client.new( options )
+      @current_azure_storage_auth_sas = Azure::Storage::Auth::SharedAccessSignature.new( @storage_account_name, storage_access_key )
     end
 
   end

data/lib/logstash/outputs/application_insights/config.rb

@@ -79,26 +79,25 @@ class LogStash::Outputs::Application_insights
           configuration[config_name] = config_value
 
         when :azure_storage_container_prefix
-          azure_storage_container_prefix = validate_and_adjust( config_name, config_value, String )
-          unless azure_storage_container_prefix.empty?
-            len = 63 - "-#{AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX}-yyyy-mm-dd".length
+          unless config_value.empty?
+            azure_storage_container_prefix = validate_and_adjust( config_name, config_value, String )
+            len = 63 - "#{AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX}--yyyy-mm-dd".length
             validate_max( "azure_storage_container_prefix length", azure_storage_container_prefix.length, len )
-            azure_storage_container_prefix += "-"
+            azure_storage_container_prefix = azure_storage_container_prefix.downcase
+            container_name = "#{AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX}-#{azure_storage_container_prefix}-yyyy-mm-dd"
+            raise ConfigurationError, "#{config_name.to_s} must have only alphanumeric characters and dash, cannot start or end with a dash, and a dash cannot follow a dash" unless Utils.valid_container_name?( container_name )
+            configuration[config_name] = "-#{azure_storage_container_prefix}"
           end
-          azure_storage_container_prefix = azure_storage_container_prefix.downcase
-          container_name = "#{azure_storage_container_prefix}#{AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX}-yyyy-mm-dd"
-          raise ConfigurationError, "#{config_name.to_s} must have only alphanumeric and dash characters, cannot start or end with a dash, and a dash cannot follow a dash" unless Utils.valid_container_name?( container_name )
-          configuration[config_name] = azure_storage_container_prefix + AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX
-
-        when :azure_storage_azure_storage_table_prefix
-          azure_storage_table_prefix = validate_and_adjust( config_name, config_value, String )
-          unless azure_storage_table_prefix.empty?
-            len = 63 - "-#{AZURE_STORAGE_TABLE_LOGSTASH_PREFIX}yyyymmdd".length
+
+        when :azure_storage_table_prefix
+          unless config_value.empty?
+            azure_storage_table_prefix = validate_and_adjust( config_name, config_value, String )
+            len = 63 - "#{AZURE_STORAGE_TABLE_LOGSTASH_PREFIX}yyyymmdd".length
             validate_max( "azure_storage_table_prefix length", azure_storage_table_prefix.length, len )
+            table_name = "#{AZURE_STORAGE_TABLE_LOGSTASH_PREFIX}#{azure_storage_table_prefix}yyyymmdd"
+            raise ConfigurationError, "#{config_name} must have only alphanumeric" unless Utils.valid_table_name?( table_name )
+            configuration[config_name] = azure_storage_table_prefix
           end
-          table_name = "#{azure_storage_table_prefix}#{AZURE_STORAGE_TABLE_LOGSTASH_PREFIX}yyyymmdd"
-          raise ConfigurationError, "#{config_name} must have only alphanumeric" unless Utils.valid_table_name?( table_name )
-          configuration[config_name] = azure_storage_table_prefix + AZURE_STORAGE_TABLE_LOGSTASH_PREFIX
 
         when :ca_file
           config_value = validate_and_adjust( config_name, config_value, String )
@@ -107,17 +106,21 @@ class LogStash::Outputs::Application_insights
           end
           configuration[config_name] = validate_and_adjust( config_name, config_value, String )
 
+        when :azure_storage_host_suffix
+          config_value = validate_and_adjust( config_name, config_value, String )
+          unless config_value.empty?
+            raise ConfigurationError, "#{config_name} must have a valid host DNS address" unless Utils.dns_address?( config_value )
+          end
+          configuration[config_name] = validate_and_adjust( config_name, config_value, String )
+
         when :azure_storage_blob_prefix
-          azure_storage_blob_prefix = validate_and_adjust( config_name, config_value, String )
-          unless azure_storage_blob_prefix.empty?
-            len = 1024 - "-#{AZURE_STORAGE_BLOB_LOGSTASH_PREFIX}_ikey-#{INSTRUMENTATION_KEY_TEMPLATE}_table-#{TABLE_ID_TEMPLATE}_yyyy-mm-dd-HH-MM-SS-LLL".length
+          unless config_value.empty?
+            azure_storage_blob_prefix = validate_and_adjust( config_name, config_value, String )
+            len = 1024 - "#{AZURE_STORAGE_BLOB_LOGSTASH_PREFIX}//ikey-#{INSTRUMENTATION_KEY_TEMPLATE}/table-#{TABLE_ID_TEMPLATE}/yyyy-mm-dd-HH-MM-SS-LLL_0000.json".length
             validate_max( "azure_storage_blob_prefix length", azure_storage_blob_prefix.length, len )
-            azure_storage_blob_prefix += "-"
+            raise ConfigurationError, "#{config_name.to_s} doesn't meet url format" unless Utils.url?( "http://storage/container/#{azure_storage_blob_prefix}_ikey-#{INSTRUMENTATION_KEY_TEMPLATE}_table-#{TABLE_ID_TEMPLATE}.json" )
+            configuration[config_name] = "/#{azure_storage_blob_prefix}"
           end
-          azure_storage_blob_prefix += AZURE_STORAGE_BLOB_LOGSTASH_PREFIX
-
-          raise ConfigurationError, "#{config_name.to_s} doesn't meet url format" unless Utils.url?( "http://storage/container/#{azure_storage_blob_prefix}_ikey-#{INSTRUMENTATION_KEY_TEMPLATE}_table-#{TABLE_ID_TEMPLATE}.json" )
-          configuration[config_name] = azure_storage_blob_prefix
 
         when :table_id
           configuration[config_name] = validate_and_adjust_guid( config_name, config_value )
@@ -200,6 +203,10 @@ class LogStash::Outputs::Application_insights
         end
       }
       validate_and_adjust_table_properties!( configuration, configuration )
+
+      configuration[:state_table_name] = "#{AZURE_STORAGE_TABLE_LOGSTASH_PREFIX}#{configuration[:azure_storage_table_prefix]}#{STATE_TABLE_NAME}"
+      configuration[:test_storage_container] = "#{AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX}#{configuration[:azure_storage_container_prefix]}-#{STORAGE_TEST_CONTAINER_NAME}"
+      configuration[:partition_key_prefix] = configuration[:azure_storage_blob_prefix].gsub( "/", "" )
       @@configuration = configuration
     end
 
@@ -220,8 +227,8 @@ class LogStash::Outputs::Application_insights
 
         case property_name.downcase
 
-        when :intrumentation_key
-          properties[:intrumentation_key] = validate_and_adjust_guid( info, property_value )
+        when :instrumentation_key
+          properties[:instrumentation_key] = validate_and_adjust_guid( info, property_value )
         when :blob_serialization
           property_value = property_value.downcase
           raise ConfigurationError, "#{info}, can be set to only one of the following values: #{VALID_EXT_EVENT_FORMAT}" unless VALID_EXT_EVENT_FORMAT.include?( property_value )
@@ -281,7 +288,7 @@ class LogStash::Outputs::Application_insights
     def self.validate_and_adjust_ext ( property, ext, prefix )
       ext = validate_and_adjust( property, ext, String )
       raise ConfigurationError, "#{property.to_s} must be a valid extension string, have only alphanumeric, dash and underline characters" unless Utils.ext?( ext )
-      len = 1024 - "#{prefix}-#{AZURE_STORAGE_BLOB_LOGSTASH_PREFIX}_ikey-#{INSTRUMENTATION_KEY_TEMPLATE}_table-#{TABLE_ID_TEMPLATE}_yyyy-mm-dd-HH-MM-SS-LLL".length
+      len = 1024 - "#{AZURE_STORAGE_BLOB_LOGSTASH_PREFIX}#{prefix}/ikey-#{INSTRUMENTATION_KEY_TEMPLATE}/table-#{TABLE_ID_TEMPLATE}/yyyy-mm-dd-HH-MM-SS-LLL_0000".length
       raise ConfigurationError, "#{property.to_s} length cannot be more than #{len} characters" unless ext.length <= len
       ext
     end

data/lib/logstash/outputs/application_insights/constants.rb

@@ -26,7 +26,8 @@ class LogStash::Outputs::Application_insights
       :notification_version => @notification_version || DEFAULT_NOTIFICATION_VERSION,
       :event_separator => @event_separator || DEFAULT_EVENT_SEPARATOR,
 
-      :notification_endpoint => @notification_endpoint || DEFAULT_NOTIFICATION_ENDPOINT,
+      :azure_storage_host_suffix => @azure_storage_host_suffix || DEFAULT_AZURE_STORAGE_HOST_SUFFIX, 
+      :application_insights_endpoint => @application_insights_endpoint || DEFAULT_APPLICATION_INSIGHTS_ENDPOINT,
       :azure_storage_blob_prefix => @azure_storage_blob_prefix || DEFAULT_AZURE_STORAGE_BLOB_PREFIX || Utils.to_storage_name( Socket.gethostname.strip ) || "",
       :azure_storage_container_prefix => @azure_storage_container_prefix || DEFAULT_AZURE_STORAGE_CONTAINER_PREFIX || Utils.to_storage_name( Socket.gethostname.strip ) || "",
       :azure_storage_table_prefix => @azure_storage_table_prefix || DEFAULT_AZURE_STORAGE_TABLE_PREFIX || Utils.to_storage_name( Socket.gethostname.strip ) || "",
@@ -43,6 +44,9 @@ class LogStash::Outputs::Application_insights
       :blob_retention_time => @blob_retention_time || DEFAULT_BLOB_RETENTION_TIME,
       :blob_access_expiry_time => @blob_access_expiry_time || DEFAULT_BLOB_ACCESS_EXPIRY_TIME,
 
+      :validate_notification => @validate_notification || DEFAULT_VALIDATE_NOTIFICATION,
+      :validate_storage => @validate_storage || DEFAULT_VALIDATE_STORAGE,
+
       :resurrect_delay => @resurrect_delay || DEFAULT_STORAGE_RESURRECT_DELAY,
       :io_retry_delay => @io_retry_delay || DEFAULT_IO_RETRY_DELAY,
       :io_max_retries => @io_max_retries || DEFAULT_IO_MAX_RETRIES,
@@ -61,7 +65,7 @@ class LogStash::Outputs::Application_insights
 
       :tables => @tables || {  },
       :table_id => @table_id || DEFAULT_TABLE_ID,
-      :intrumentation_key => @intrumentation_key || DEFAULT_INSTRUMENTATION_KEY,
+      :instrumentation_key => @instrumentation_key || DEFAULT_INSTRUMENTATION_KEY,
       :table_columns => @table_columns,
       :case_insensitive_columns => @case_insensitive_columns || DEFAULT_CASE_INSENSITIVE,
       :serialized_event_field => @serialized_event_field,
@@ -75,7 +79,8 @@ class LogStash::Outputs::Application_insights
 
   BOOLEAN_PROPERTIES = [  :disable_notification, :disable_blob_upload, 
                           :stop_on_unknown_io_errors, :disable_telemetry, 
-                          :disable_cleanup, :delete_not_notified_blobs, 
+                          :disable_cleanup, :delete_not_notified_blobs,
+                          :validate_notification, :validate_storage,
                           :save_notified_blobs_records, :case_insensitive_columns,
                           :table_columns, :serialized_event_field ]
 
@@ -135,12 +140,13 @@ class LogStash::Outputs::Application_insights
   MIN_FLOW_CONTROL_DELAY = 0.1                            # in seconds, 1 seconds, can be less than 1 seconds, like 0.5, 0.1
   MAX_FLOW_CONTROL_DELAY = 0                              # in seconds, 1 seconds, can be less than 1 seconds, like 0.5, 0.1
 
-  METADATA_FIELD_INSTRUMENTATION_KEY = "[@metadata]intrumentation_key"
+  METADATA_FIELD_INSTRUMENTATION_KEY = "[@metadata]instrumentation_key"
   METADATA_FIELD_TABLE_ID = "[@metadata]table_id"
-  FIELD_INSTRUMENTATION_KEY = "intrumentation_key"
+  FIELD_INSTRUMENTATION_KEY = "instrumentation_key"
   FIELD_TABLE_ID = "table_id"
 
   STATE_TABLE_NAME = "BlobsState"
+  STORAGE_TEST_CONTAINER_NAME = "test-container"
 
   AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX = "logstash" # lower case only, dash allowed
   AZURE_STORAGE_BLOB_LOGSTASH_PREFIX = "logstash"
@@ -181,7 +187,8 @@ class LogStash::Outputs::Application_insights
   DEFAULT_BLOB_RETENTION_TIME = 60 * 60 * 24 * 7      # in seconds - one week
   DEFAULT_BLOB_ACCESS_EXPIRY_TIME = 60 * 60 * 24 * 1  # in seconds - one day
   DEFAULT_STORAGE_RESURRECT_DELAY = 10
-  DEFAULT_NOTIFICATION_ENDPOINT = "https://dc.services.visualstudio.com/v2/track"
+  DEFAULT_APPLICATION_INSIGHTS_ENDPOINT = "https://dc.services.visualstudio.com/v2/track"
+  DEFAULT_AZURE_STORAGE_HOST_SUFFIX = "core.windows.net"
   DEFAULT_NOTIFICATION_VERSION = 1
   DEFAULT_DISABLE_NOTIFICATION = false
   DEFAULT_DISABLE_BLOB_UPLOAD = false
@@ -193,6 +200,9 @@ class LogStash::Outputs::Application_insights
   
   DEFAULT_CASE_INSENSITIVE = false
 
+  DEFAULT_VALIDATE_NOTIFICATION = false
+  DEFAULT_VALIDATE_STORAGE = false
+
   DEFAULT_LOGGER_FILES = [ "logstash-output-application-insights.log" ]
   DEFAULT_LOG_LEVEL = "INFO"
   DEFAULT_LOGGER_PROGNAME = "AI"

data/lib/logstash/outputs/application_insights/notification_recovery.rb

@@ -0,0 +1,101 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+class LogStash::Outputs::Application_insights
+  class Notification_recovery
+
+    public
+
+    def initialize
+      configuration = Config.current
+      @logger = configuration[:logger]
+      @storage_account_name_key = configuration[:storage_account_name_key]
+      @queue = Queue.new
+
+      @closing = nil
+      @thread = nil
+    end
+
+    def start
+      @thread = recovery_thread
+    end
+
+    def recover_later ( tuple )
+      @notification_state_on = false
+      @queue << tuple
+    end
+
+    def enqueue ( tuple )
+      @queue << tuple
+    end
+
+    def close
+      @closing = true
+      # @thread.join
+    end
+
+    private
+
+    def stopped?
+      @closing
+    end
+
+    def init_queues ( storage_account_name_key, queues )
+      storage_account_name_key.each do |storage_account_name, storage_account_keys|
+        queues.each_key do |action|
+          queues[action][storage_account_name] = Queue.new
+        end
+      end
+    end
+
+    def recovery_thread
+      Thread.new do
+        blob = Blob.new
+        counter = Concurrent::AtomicFixnum.new(0)
+
+        loop do
+          tuple = @queue.pop
+          Stud.stoppable_sleep(Float::INFINITY, 1) { state_on?( blob ) && 10 > counter.value }
+
+          counter.increment
+          Thread.new( counter, tuple ) do |counter, tuple|
+            Blob.new.send( :notify, tuple )
+            counter.decrement
+          end
+          tuple = nil # release for GC
+        end
+      end
+    end
+
+    def state_on? ( blob )
+      @notification_state_on ||= blob.test_notification( @storage_account_name_key[0][0] )
+    end
+
+    public
+
+    @@instance = Notification_recovery.new
+
+    def self.instance
+      @@instance
+    end
+
+    private_class_method :new
+  end
+end

data/lib/logstash/outputs/application_insights/shutdown.rb

@@ -48,7 +48,10 @@ class LogStash::Outputs::Application_insights
       end
       display_msg( "all events were uploaded to Azure storage" )
 
+      # close all blobs activity
       Blob.close
+
+      # close all channels activity
       @channels.close
 
       # wait for all uploads to commit

data/lib/logstash/outputs/application_insights/shutdown_recovery.rb

@@ -0,0 +1,140 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+class LogStash::Outputs::Application_insights
+  class Shutdown_recovery
+
+    public
+
+    def initialize
+      configuration = Config.current
+      @logger = configuration[:logger]
+      @storage_account_name_key = configuration[:storage_account_name_key]
+      @partition_key_prefix =configuration[:azure_storage_blob_prefix].gsub( "/", "" )
+
+      @storage_recovery = Storage_recovery.instance
+      @notification_recovery = Notification_recovery.instance
+
+      @closing = nil
+      @threads = []
+    end
+
+    def start
+      @storage_account_name_key.each do |storage_account_name, storage_account_keys|
+        @threads << recovery_thread( storage_account_name, :uploading)
+        @threads << recovery_thread( storage_account_name, :committed)
+      end
+    end
+
+    def close
+      @closing = true
+      @threads.each do |thread|
+        thread.join
+      end
+    end
+
+    private
+
+    def stopped?
+      @closing
+    end
+
+    def recovery_thread( storage_account_name, state )
+      Thread.new( storage_account_name, state ) do |storage_account_name, state|
+        
+        blob = Blob.new
+
+        committed_tuples = [  ]
+        uncommitted_tuples = [  ]
+        upload_empty_tuples = [  ]
+        token = nil
+        finished = false
+        filter = "#{:PartitionKey} eq '#{@partition_key_prefix}-#{state}'"
+
+        # should exit thread after fetching data from table, and submit recovery, the loop is only for case of failure
+        until finished || stopped? do
+          entities = blob.state_table_query( storage_account_name, filter, token )
+          if entities
+            token = entities.continuation_token
+
+            if :committed == state
+              entities.each do |entity|
+                State.instance.inc_pending_notifications
+                tuple = blob.table_entity_to_tuple( entity.properties )
+                @notification_recovery.enqueue( tuple )
+              end
+
+            elsif :uploading == state
+              # first tuples are collected, before send to queues, to make sure blob states don't change in between
+              entities.each do |entity|
+                typed_tuple = nil
+                until typed_tuple || stopped?
+                  typed_tuple = blob.update_commited_or_uncommited_list( entity.properties )
+                  Stud.stoppable_sleep(60, 1) { stopped? } unless typed_tuple
+                end
+
+                next if stopped?
+
+                if typed_tuple[:committed]
+                  committed_tuples << typed_tuple[:committed]
+                elsif typed_tuple[:uncommitted]
+                  uncommitted_tuples << typed_tuple[:uncommitted]
+                else
+                  upload_empty_tuples << typed_tuple[:upload_empty]
+                end
+              end
+            end
+
+            next if token
+
+            committed_tuples.each do |tuple|
+              State.instance.inc_pending_commits
+              @storage_recovery.recover_later( tuple, :state_table_update, storage_account_name )
+            end
+
+            uncommitted_tuples.each do |tuple|
+              State.instance.inc_pending_commits
+              @storage_recovery.recover_later( tuple, :commit, storage_account_name )
+            end
+
+            upload_empty_tuples.each do |tuple|
+              @storage_recovery.recover_later( tuple, :state_table_update, storage_account_name )
+            end
+
+            finished = true
+          else
+            Stud.stoppable_sleep(60, 1) { stopped? }
+          end
+        end
+        @logger.info { "exit table recovery thread, storage: #{storage_account_name}, state: #{state}, entities: #{entities ? entities.length : nil}" }
+      end
+    end
+
+    public
+
+    @@instance = Shutdown_recovery.new
+
+    def self.instance
+      @@instance
+    end
+
+    private_class_method :new
+  end
+end