logstash-output-application_insights 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12c164429fdbd439168dab5277c28185ce2e4985
-  data.tar.gz: 44048dcef57a0bbe542eb8a3b68811f968eb8352
+  metadata.gz: eeff77711bd05c02e3efc1c3a0a7ab7dc40fb109
+  data.tar.gz: d4e3a91792cccfd42befb748d1dfc0dea6181735
 SHA512:
-  metadata.gz: 8388e00a758efb2fb30215ace69730d28690b573c66ee901be7dd635f16660b5a2236ef0f1bb4c0665924b54e8a3150ea868ce8c1771f42409af3cd5ed251ec3
-  data.tar.gz: cc27c111592a9bd9aca93fb8f697219eb58698ae7691222b43ff98313ce1cd365263feb9fd19fbfbcaa07cca98a9a114c2e0eb3b1a25319b151d7f789f515880
+  metadata.gz: bff4c4074ed237dc768c9c69d555eee4241651d240dc69786431f7f24d7ab67487a01c7f209cc51b8d210561bb348f4c2f32c3b183ebf9c7d52bb0f2eb2c7039
+  data.tar.gz: 18a245f8fd5ecdff45c0efdb6fc73b236d9f89996d711ed8cce63fb9bd2a09e66a0731ec2cea5a8d0a23fb5b4cd02d58ae2cf2fe112b3d14aebd0ca9297759ba

data/README.md

@@ -48,13 +48,16 @@ input {
   }
 }
 filter {
-  # some filters here
+    # some filters here
 }
 output {
   application_insights {
     instrumentation_key => "5a6714a3-ec7b-4999-ab96-232f1da92059"
     table_id => "c24394e1-f077-420e-8a25-ef6fdf045938"
     storage_account_name_key => [ "my-storage-account", "pfrYTwPgKyYNfKBY2QdF+v5sbgx8/eAQp+FFkGpPBnkMDE1k+ZNK3r3qIPqqw8UsOIUqaF3dXBdPDouGJuxNXQ==" ]
+      #
+      # if you want to allow Microsoft get telemtry data about this process please set it to true
+    enable_telemetry_to_microsoft => false
   }
 }
 ```
@@ -341,11 +344,12 @@ example:
 ca_file => "/path/to/cafile.crt"
 ```
 
-### disable_telemetry
-When set to true, telemetry about the plugin, won't be sent to Application Insights. Deafult false
+### enable_telemetry_to_microsoft
+When set to true, telemetry about the plugin, will't be sent to Microsoft. Deafult false
+**Only if you want to allow Microsoft get telemtry data about this process set it to true**
 example:
 ```ruby
-disable_telemetry => true
+enable_telemetry_to_microsoft => true
 ```
 
 ### disable_cleanup

data/lib/logstash/outputs/application_insights.rb

@@ -253,8 +253,8 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
   config :ca_file, :validate => :string
 
 
-  # When set to true, telemetry about the plugin, won't be sent to Application Insights
-  config :disable_telemetry, :validate => :boolean
+  # When set to true, telemetry about the plugin, will't be sent to Microsoft. Deafult false
+  config :enable_telemetry_to_microsoft, :validate => :boolean
 
   # When set to true, storage cleanup won't be done by the plugin (should be done by some other means or by another Logstash process with this flag enabled)
   config :disable_cleanup, :validate => :boolean
@@ -349,7 +349,7 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
     #   @channels.receive( event, encoded_event )
     # end
 
-    @telemetry.track_event("register", {:properties => configuration})
+    @telemetry.track_event { { :name => "register", :properties => configuration } }
 
 
     return "ok\n"
@@ -363,7 +363,7 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
   end
 
   def close
-    @telemetry.track_event( "close" )
+    @telemetry.track_event { { :name => "close" } }
     @telemetry.flush
     @shutdown_recovery.close
     @storage_recovery.close

data/lib/logstash/outputs/application_insights/blob.rb

@@ -259,6 +259,7 @@ class LogStash::Outputs::Application_insights
       end
     end
 
+    # must return whether notification was successful or failed
     def notify ( tuple = nil )
       tuple_to_state( tuple ) if tuple
       @action = :notify
@@ -275,10 +276,12 @@ class LogStash::Outputs::Application_insights
         @log_state = :notified
       }
       if success
+        Telemetry.instance.track_event { { :name => "notified", :properties => state_to_table_entity } }
         state_table_update
       else
         notify_retry_later
       end
+      success
     end
 
     CREATE_EXIST_ERRORS = { :container => [ :create_container, :container_exist ], :table => [ :create_table, :table_exist ] }
@@ -473,7 +476,6 @@ class LogStash::Outputs::Application_insights
           @block_to_upload = nil
           State.instance.inc_pending_commits if first_block_in_blob
           State.instance.dec_upload_bytesize( bytesize )
-          Telemetry.instance.track_event("uploading", {:properties => state_to_table_entity})
         }
 
         upload_retry_later unless success

data/lib/logstash/outputs/application_insights/channel.rb

@@ -46,7 +46,6 @@ class LogStash::Outputs::Application_insights
       @failed_on_notify_retry_Q = Queue.new
       @workers_channel = {  }
       @active_blobs = [ Blob.new( self, 1 ) ]
-      @state = State.instance
 
       launch_upload_recovery_thread
       launch_notify_recovery_thread
@@ -137,20 +136,29 @@ class LogStash::Outputs::Application_insights
     end
 
 
-    # thread that failed to notify due to Application Isights error, such as wrong key or wrong schema
+    # thread that failed to notify due to Application Insights error, such as wrong key or wrong schema
     def launch_notify_recovery_thread
       #recovery thread
       Thread.new do
         loop do
-          tuple ||= @failed_on_notify_retry_Q.pop
+          tuple = @failed_on_notify_retry_Q.pop
           begin
             Stud.stoppable_sleep( 60 ) { stopped? }
           end until Clients.instance.storage_account_state_on? || stopped?
+
           if  stopped?
+            @state ||= State.instance
             @state.dec_pending_notifications
+            @shutdown ||= Shutdown.instance
+            @shutdown.display_msg("!!! notification won't recover in this session due to shutdown")
           else
-            Blob.new.notify( tuple )
+            success = Blob.new.notify( tuple )
+            while success && @failed_on_notify_retry_Q.length > 0
+              tuple = @failed_on_notify_retry_Q.pop
+              success = Blob.new.notify( tuple )
+            end
           end
+          tuple = nil # release for GC
         end
       end
     end

data/lib/logstash/outputs/application_insights/constants.rb

@@ -37,7 +37,7 @@ class LogStash::Outputs::Application_insights
       :stop_on_unknown_io_errors => @stop_on_unknown_io_errors || DEFAULT_STOP_ON_UNKNOWN_IO_ERRORS,
       :delete_not_notified_blobs => @delete_not_notified_blobs || DEFAULT_DELETE_NOT_NOTIFIED_BLOBS,
       :save_notified_blobs_records => @save_notified_blobs_records || DEFAULT_SAVE_NOTIFIED_BLOBS_RECORDS,
-      :disable_telemetry => @disable_telemetry || DEFAULT_DISABLE_TELEMETRY,
+      :enable_telemetry_to_microsoft => @enable_telemetry_to_microsoft || DEFAULT_ENABLE_TELEMETRY_TO_MICROSOFT,
       :disable_cleanup => @disable_cleanup || DEFAULT_DISABLE_CLEANUP,
       :blob_max_bytesize => @blob_max_bytesize || DEFAULT_BLOB_MAX_BYTESIZE,
       :blob_max_events => @blob_max_events || DEFAULT_BLOB_MAX_EVENTS,
@@ -78,7 +78,7 @@ class LogStash::Outputs::Application_insights
   end
 
   BOOLEAN_PROPERTIES = [  :disable_notification, :disable_blob_upload, 
-                          :stop_on_unknown_io_errors, :disable_telemetry, 
+                          :stop_on_unknown_io_errors, :enable_telemetry_to_microsoft, 
                           :disable_cleanup, :delete_not_notified_blobs,
                           :validate_notification, :validate_storage,
                           :save_notified_blobs_records, :case_insensitive_columns,
@@ -193,7 +193,7 @@ class LogStash::Outputs::Application_insights
   DEFAULT_DISABLE_NOTIFICATION = false
   DEFAULT_DISABLE_BLOB_UPLOAD = false
   DEFAULT_STOP_ON_UNKNOWN_IO_ERRORS = false
-  DEFAULT_DISABLE_TELEMETRY = false
+  DEFAULT_ENABLE_TELEMETRY_TO_MICROSOFT = false
   DEFAULT_DISABLE_CLEANUP = false
   DEFAULT_DELETE_NOT_NOTIFIED_BLOBS = false
   DEFAULT_SAVE_NOTIFIED_BLOBS_RECORDS = false

data/lib/logstash/outputs/application_insights/notification_recovery.rb

@@ -39,7 +39,14 @@ class LogStash::Outputs::Application_insights
 
     def recover_later ( tuple )
       @notification_state_on = false
-      @queue << tuple
+      if stopped?
+        @state ||= State.instance
+        @state.dec_pending_notifications
+        @shutdown ||= Shutdown.instance
+        @shutdown.display_msg("!!! notification won't recover in this session due to shutdown")
+      else
+        @queue << tuple
+      end
     end
 
     def enqueue ( tuple )
@@ -57,14 +64,6 @@ class LogStash::Outputs::Application_insights
       @closing
     end
 
-    def init_queues ( storage_account_name_key, queues )
-      storage_account_name_key.each do |storage_account_name, storage_account_keys|
-        queues.each_key do |action|
-          queues[action][storage_account_name] = Queue.new
-        end
-      end
-    end
-
     def recovery_thread
       Thread.new do
         blob = Blob.new
@@ -72,12 +71,16 @@ class LogStash::Outputs::Application_insights
 
         loop do
           tuple = @queue.pop
-          Stud.stoppable_sleep(Float::INFINITY, 1) { state_on?( blob ) && 10 > counter.value }
-
-          counter.increment
-          Thread.new( counter, tuple ) do |counter, tuple|
-            Blob.new.send( :notify, tuple )
-            counter.decrement
+          Stud.stoppable_sleep(Float::INFINITY, 1) { ( state_on?( blob ) || stopped? ) && 10 > counter.value }
+
+          if stopped? && !state_on?( blob )
+            recover_later( tuple )
+          else
+            counter.increment
+            Thread.new( counter, tuple ) do |counter, tuple|
+              Blob.new.notify( tuple )
+              counter.decrement
+            end
           end
           tuple = nil # release for GC
         end

data/lib/logstash/outputs/application_insights/shutdown.rb

@@ -79,8 +79,6 @@ class LogStash::Outputs::Application_insights
       display_msg( "all events were notified to Application Insights Analytics" )
     end
 
-    private
-
     def display_msg ( msg )
         puts "+++ #{@logger_progname} #{msg}"
         # @logger.info { "#{msg}" }

data/lib/logstash/outputs/application_insights/storage_recovery.rb

@@ -43,8 +43,17 @@ class LogStash::Outputs::Application_insights
       end
     end
 
-    def recover_later ( tuple, action, storage_account_name )
-      @queues[action][storage_account_name] << tuple
+    def recover_later ( tuple, action = nil, storage_account_name = nil )
+      if stopped?
+        if :commit == action
+          @state ||= State.instance
+          @state.dec_pending_commits
+          @shutdown ||= Shutdown.instance
+          @shutdown.display_msg("!!! commit won't recover in this session due to shutdown")
+        end
+      else
+        @queues[action][storage_account_name] << tuple
+      end
     end
 
     def close
@@ -75,12 +84,16 @@ class LogStash::Outputs::Application_insights
         queue = @queues[action][storage_account_name]
         loop do
           tuple = queue.pop
-          Stud.stoppable_sleep(Float::INFINITY, 1) { state_on?( storage_account_name ) && 10 > counter.value }
-
-          counter.increment
-          Thread.new( action, counter, tuple ) do |action, counter, tuple|
-            Blob.new.send( action, tuple )
-            counter.decrement
+          Stud.stoppable_sleep(Float::INFINITY, 1) { ( state_on?( storage_account_name )  || stopped? ) && 10 > counter.value }
+
+          if stopped? && !state_on?( storage_account_name )
+            recover_later( tuple )
+          else
+            counter.increment
+            Thread.new( action, counter, tuple ) do |action, counter, tuple|
+              Blob.new.send( action, tuple )
+              counter.decrement
+            end
           end
           tuple = nil # release for GC
         end

data/lib/logstash/outputs/application_insights/telemetry.rb

@@ -30,8 +30,8 @@ class LogStash::Outputs::Application_insights
 
     def initialize
       configuration = Config.current
-      @disable_telemetry = configuration[:disable_telemetry]
-      unless @disable_telemetry
+      @enable_telemetry_to_microsoft = configuration[:enable_telemetry_to_microsoft]
+      if @enable_telemetry_to_microsoft
         @telemetry_channel = create_async_channel( LOGSTASH_TELEMETRY_INSTRUMENTATION_KEY )
         set_async_channel_properties( @telemetry_channel )
         set_channel_context( @telemetry_channel )
@@ -69,16 +69,33 @@ class LogStash::Outputs::Application_insights
       # tc.context.user.id = 'santa@northpole.net'
     end
 
-    def track_event ( name, options={} )
-       @telemetry_channel.track_event( name, options ) unless @disable_telemetry
+    def track_event
+      if @enable_telemetry_to_microsoft
+        options = yield
+        name = options.delete( :name )
+        @telemetry_channel.track_event( name, options )
+      end
     end
 
-    def track_metric ( name, value, options={} )
-       @telemetry_channel.track_metric( name, value, options ) unless @disable_telemetry
+    def track_metric
+      if @enable_telemetry_to_microsoft
+        options = yield
+        name = options.delete( :name )
+        value = options.delete( :value )
+        @telemetry_channel.track_metric( name, value, options )
+      end
     end
 
-    def track_request (id, start_time, duration, response_code, success, options = {} )
-      @telemetry_channel.track_request( id, start_time, duration, response_code, success, options ) unless @disable_telemetry
+    def track_request
+      if @enable_telemetry_to_microsoft
+        options = yield
+        id = options.delete( :id )
+        start_time = options.delete( :start_time )
+        duration = options.delete( :duration )
+        response_code = options.delete( :response_code )
+        success = options.delete( :success )
+        @telemetry_channel.track_request( id, start_time, duration, response_code, success, options )
+      end
     end
 
     def flush

data/lib/logstash/outputs/application_insights/version.rb

@@ -20,5 +20,5 @@
 # ----------------------------------------------------------------------------------
 
 # class LogStash::Outputs::Application_insights
-  APPLICATION_INSIGHTS_VERSION ||= "0.1.4"
+  APPLICATION_INSIGHTS_VERSION ||= "0.1.5"
 # end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-application_insights
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Microsoft Corporation
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-13 00:00:00.000000000 Z
+date: 2016-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement