logstash-output-application_insights 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eeff77711bd05c02e3efc1c3a0a7ab7dc40fb109
-  data.tar.gz: d4e3a91792cccfd42befb748d1dfc0dea6181735
+  metadata.gz: 800b0ad22dc672b293d6f6aba51437e0544e919e
+  data.tar.gz: 45f3ef6fe212653bb90e92855a3f28ce8b9ee863
 SHA512:
-  metadata.gz: bff4c4074ed237dc768c9c69d555eee4241651d240dc69786431f7f24d7ab67487a01c7f209cc51b8d210561bb348f4c2f32c3b183ebf9c7d52bb0f2eb2c7039
-  data.tar.gz: 18a245f8fd5ecdff45c0efdb6fc73b236d9f89996d711ed8cce63fb9bd2a09e66a0731ec2cea5a8d0a23fb5b4cd02d58ae2cf2fe112b3d14aebd0ca9297759ba
+  metadata.gz: 21ed44eca0a8e7fc4ddc0a67e1c08bfef8e98d4531e2b06d73e034009b263058125fa74279c197059535238f7ef9481e0d79cc88070cfc55ba0c6be8fdb3dfd2
+  data.tar.gz: 381a07cccf8dcd5d96995229bc0cc87327670fd740f961c5fce1f6a867fc03ea080b562e8b0c171e0f9a00398b348b1a94783272c39e35facee08d4e46eec5db

data/lib/logstash/outputs/application_insights.rb

@@ -59,6 +59,8 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
   autoload :Notification_recovery, "logstash/outputs/application_insights/notification_recovery"
   autoload :Validate_storage, "logstash/outputs/application_insights/validate_storage"
   autoload :Validate_notification, "logstash/outputs/application_insights/validate_notification"
+  autoload :Test_storage, "logstash/outputs/application_insights/test_storage"
+  autoload :Test_notification, "logstash/outputs/application_insights/test_notification"
   
 
   autoload :Clients, "logstash/outputs/application_insights/clients" 
@@ -304,19 +306,24 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
     # set configuration
     Config.validate_and_adjust_configuration( default_configuration )
     configuration = Config.current
+    masked_configuration = Config.masked_current
 
     Multi_io_logger.config( configuration )
 
     # be careful don't use here @logger, as it will override Logstash @logger, and may create starnge behaviour
     @private_logger = configuration[:logger]
 
-    @private_logger.info { "configuration: #{configuration}" }
+    @private_logger.info { "configuration: #{masked_configuration}" }
 
     @telemetry = Telemetry.instance
     configuration[:telemetry_channel] = @telemetry.telemetry_channel
 
     Timer.config( configuration )
 
+    @notification_recovery = Notification_recovery.instance
+    @storage_recovery = Storage_recovery.instance
+    @shutdown_recovery = Shutdown_recovery.instance
+
     if @validate_notification
       status = Validate_notification.new.validate
       raise ConfigurationError, "Failed to access application insights at #{configuration[:application_insights_endpoint]}, due to error #{status[:error].inspect}" unless status[:success]
@@ -329,18 +336,16 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
       end
     end
 
-    @notification_recovery = Notification_recovery.instance
     @notification_recovery.start
-
-    @storage_recovery = Storage_recovery.instance
     @storage_recovery.start
-
-    @shutdown_recovery = Shutdown_recovery.instance
     @shutdown_recovery.start
 
     @shutdown = Shutdown.instance
     @channels = Channels.instance
 
+    @shutdown.start
+    @channels.start
+
     @storage_cleanup = Storage_cleanup.start
 
     @private_logger.info { "plugin registered" }
@@ -349,7 +354,7 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
     #   @channels.receive( event, encoded_event )
     # end
 
-    @telemetry.track_event { { :name => "register", :properties => configuration } }
+    @telemetry.track_event { { :name => "register", :properties => masked_configuration } }
 
 
     return "ok\n"

data/lib/logstash/outputs/application_insights/blob.rb

@@ -203,43 +203,6 @@ class LogStash::Outputs::Application_insights
     end
 
 
-    def test_storage ( storage_account_name )
-      @storage_account_name = storage_account_name
-      @action = :test_storage
-      @max_tries = 1
-      @force_client = true # to enable get a client even if all storage_accounts marked dead
-      @recoverable = [ :invalid_storage_key, :container_exist, :create_container ]
-      storage_io_block {
-        if @recovery.nil? || :invalid_storage_key == @recovery
-          @info = "#{@action} #{@storage_account_name}"
-          @client.blobClient.create_container( @configuration[:test_storage_container] ) unless @configuration[:disable_blob_upload]
-        end
-      }
-    end
-
-    def test_notification( storage_account_name )
-      @storage_account_name = storage_account_name
-      @action = :test_notification
-      @max_tries = 1
-      @force_client = true # to enable get a client even if all storage_accounts marked dead
-      @recoverable = [ :invalid_instrumentation_key, :invalid_table_id ]
-      success = storage_io_block {
-        if @recovery.nil?
-          @container_name = "logstash-test-container"
-          @blob_name = "logstash-test-blob"
-          @table_id = GUID_NULL
-          @instrumentation_key = GUID_NULL
-          @info = "#{@action}"
-          set_blob_sas_url
-          payload = create_payload
-          post_notification( @client.notifyClient, payload )
-        end
-      }
-      sleep( 30 ) unless success
-      success
-    end
-
-
     def notify_retry_later
       if :notify_failed_blob_not_accessible == @recovery
         @sub_state = @recovery

data/lib/logstash/outputs/application_insights/channels.rb

@@ -36,9 +36,10 @@ class LogStash::Outputs::Application_insights
       @default_instrumentation_key = configuration[:instrumentation_key]
       @default_table_id = configuration[:table_id]
       @tables = configuration[:tables]
+    end
 
+    def start
       @flow_control = Flow_control.instance
-
       # launch tread that forward events from channels to azure storage
       periodic_forward_events
     end
@@ -133,6 +134,7 @@ class LogStash::Outputs::Application_insights
     public
 
     @@instance = Channels.new
+
     def self.instance
       @@instance
     end

data/lib/logstash/outputs/application_insights/clients.rb

@@ -78,9 +78,10 @@ class LogStash::Outputs::Application_insights
             @current_storage_account_names = current_storage_account_names
 
             Thread.new( storage_account_name ) do |account_name|
+              test_storage = Test_storage.new( account_name )
               loop do
                 sleep( @resurrect_delay )
-                if Blob.new.test_storage( account_name )
+                if test_storage.submit
                   @state_semaphore.synchronize {
                     storage_account = @storage_accounts[account_name]
                     storage_account[:off_reason] = [  ]

data/lib/logstash/outputs/application_insights/config.rb

@@ -25,11 +25,16 @@ class LogStash::Outputs::Application_insights
     public
 
     @@configuration = {}
+    @@masked_configuration = {}
 
     def self.current
       @@configuration
     end
 
+    def self.masked_current
+      @@masked_configuration
+    end
+
     def self.validate_and_adjust_configuration ( configuration )
       configuration.each_pair { |config_name, config_value|
         raise ConfigurationError, "#{config_name.to_s} must be defined" unless config_value || BOOLEAN_PROPERTIES.include?( config_name )
@@ -46,7 +51,7 @@ class LogStash::Outputs::Application_insights
 
           i = 0
           logger_files.map! do |file_name|
-            file_name = validate_and_adjust( "#{config_name.to_s}[#{i}]", file_name, String )
+            file_name = validate_and_adjust( "#{config_name}[#{i}]", file_name, String )
             i += 1
             if "stdout" == file_name.downcase
               file_name = :stdout
@@ -55,7 +60,11 @@ class LogStash::Outputs::Application_insights
               file_name = :stderr
               file = STDERR
             else
-              file = ::File.open( file_name, "a+" )
+              begin
+                file = ::File.open( file_name, "a+" )
+              rescue => e
+                raise ConfigurationError, "#{config_name}[#{i}] cannot open file #{file_name}, due to error #{e.inspect}"
+              end
             end
             [ file_name, file ]
           end
@@ -158,26 +167,26 @@ class LogStash::Outputs::Application_insights
         when :storage_account_name_key
           config_value = validate_and_adjust( config_name, config_value, Array )
           if config_value.empty?
-            raise ConfigurationError, "#{config_name.to_s} is empty, at least one storage account name should be defined" unless ENV['AZURE_STORAGE_ACCOUNT']
-            raise ConfigurationError, "#{config_name.to_s} is empty, at least one storage account access key should be defined" unless ENV['AZURE_STORAGE_ACCESS_KEY']
+            raise ConfigurationError, "#{config_name} is empty, at least one storage account name should be defined" unless ENV['AZURE_STORAGE_ACCOUNT']
+            raise ConfigurationError, "#{config_name} is empty, at least one storage account access key should be defined" unless ENV['AZURE_STORAGE_ACCESS_KEY']
             config_value = [ ENV['AZURE_STORAGE_ACCOUNT'], ENV['AZURE_STORAGE_ACCESS_KEY'] ]
           end
 
           storage_account_name_key = validate_and_adjust( config_name, config_value, Array, :disallow_empty )
           unless storage_account_name_key[0].is_a?( Array )
-            raise ConfigurationError, "#{config_name.to_s} property is empty, should contain at least one pair: account_name, key" unless 2 == storage_account_name_key.length
+            raise ConfigurationError, "#{config_name} property is empty, should contain at least one pair: account_name, key" unless 2 == storage_account_name_key.length
             storage_account_name_key = [ [ storage_account_name_key[0], storage_account_name_key[1] ]]
           end
 
           index = 0
           storage_account_name_key.map! { |pair|
-            pair = validate_and_adjust( "#{config_name.to_s}[#{index}]", pair, Array, :disallow_empty )
-            raise ConfigurationError, "#{config_name.to_s}[#{index}] must have two items" unless 2 == pair.length
+            pair = validate_and_adjust( "#{config_name}[#{index}]", pair, Array, :disallow_empty )
+            raise ConfigurationError, "#{config_name}[#{index}] must have two items" unless 2 == pair.length
 
             ( name, keys ) = pair
-            name = validate_and_adjust( "#{config_name.to_s}[#{index}]:name", name, String, :disallow_empty )
-            raise ConfigurationError, "#{config_name.to_s}[#{index}]:name must between 3 to 24 characters" unless (name.length >= 3) && (name.length <= 24)
-            raise ConfigurationError, "##{config_name.to_s}[#{index}]:name bad format, must have only alphanumeric characters" unless Utils.alphanumeric?( name )
+            name = validate_and_adjust( "#{config_name}[#{index}]:name", name, String, :disallow_empty )
+            raise ConfigurationError, "#{config_name}[#{index}]:name must between 3 to 24 characters" unless (name.length >= 3) && (name.length <= 24)
+            raise ConfigurationError, "##{config_name}[#{index}]:name bad format, must have only alphanumeric characters" unless Utils.alphanumeric?( name )
 
             keys = [ keys ] if keys.is_a?( String )
             keys = validate_and_adjust( "#{config_name}[#{index}]:keys", keys, Array, :disallow_empty )
@@ -207,9 +216,23 @@ class LogStash::Outputs::Application_insights
       configuration[:state_table_name] = "#{AZURE_STORAGE_TABLE_LOGSTASH_PREFIX}#{configuration[:azure_storage_table_prefix]}#{STATE_TABLE_NAME}"
       configuration[:test_storage_container] = "#{AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX}#{configuration[:azure_storage_container_prefix]}-#{STORAGE_TEST_CONTAINER_NAME}"
       configuration[:partition_key_prefix] = configuration[:azure_storage_blob_prefix].gsub( "/", "" )
+
+      @@masked_configuration = mask_configuration( configuration )
+
       @@configuration = configuration
     end
 
+    def self.mask_configuration ( configuration )
+      masked_configuration = configuration.dup
+      storage_account_name_key = masked_configuration[:storage_account_name_key]
+      masked_storage_account_name_key = storage_account_name_key.map { |pair|
+        ( name, keys ) = pair
+        masked_keys = keys.map { |key| "*****************" }
+        [ name, masked_keys ]
+      }
+      masked_configuration[:storage_account_name_key] = masked_storage_account_name_key
+      masked_configuration
+    end
 
     def self.symbolize_table_properties ( properties )
       new_properties = {}

data/lib/logstash/outputs/application_insights/notification_recovery.rb

@@ -25,8 +25,7 @@ class LogStash::Outputs::Application_insights
 
     def initialize
       configuration = Config.current
-      @logger = configuration[:logger]
-      @storage_account_name_key = configuration[:storage_account_name_key]
+      @resurrect_delay = configuration[:resurrect_delay]
       @queue = Queue.new
 
       @closing = nil
@@ -34,6 +33,7 @@ class LogStash::Outputs::Application_insights
     end
 
     def start
+      @test_notification = Test_notification.new
       @thread = recovery_thread
     end
 
@@ -88,7 +88,11 @@ class LogStash::Outputs::Application_insights
     end
 
     def state_on? ( blob )
-      @notification_state_on ||= blob.test_notification( @storage_account_name_key[0][0] )
+      return @notification_state_on if @notification_state_on
+      @notification_state_on = @test_notification.submit
+      return @notification_state_on if @notification_state_on
+      sleep( @resurrect_delay )
+      @notification_state_on
     end
 
     public

data/lib/logstash/outputs/application_insights/shutdown.rb

@@ -30,6 +30,9 @@ class LogStash::Outputs::Application_insights
       @logger_progname = configuration[:logger_progname]
       @logger = configuration[:logger]
 
+    end
+
+    def start
       @state = State.instance
       @channels = Channels.instance
     end

data/lib/logstash/outputs/application_insights/shutdown_recovery.rb

@@ -29,14 +29,13 @@ class LogStash::Outputs::Application_insights
       @storage_account_name_key = configuration[:storage_account_name_key]
       @partition_key_prefix =configuration[:azure_storage_blob_prefix].gsub( "/", "" )
 
-      @storage_recovery = Storage_recovery.instance
-      @notification_recovery = Notification_recovery.instance
-
       @closing = nil
       @threads = []
     end
 
     def start
+      @storage_recovery = Storage_recovery.instance
+      @notification_recovery = Notification_recovery.instance
       @storage_account_name_key.each do |storage_account_name, storage_account_keys|
         @threads << recovery_thread( storage_account_name, :uploading)
         @threads << recovery_thread( storage_account_name, :committed)

data/lib/logstash/outputs/application_insights/storage_cleanup.rb

@@ -43,11 +43,9 @@ class LogStash::Outputs::Application_insights
       # super first parameter must be nil. blob first parameter is channel, otherwise it will pass storage_account_name as channel
       super( nil )
       @storage_account_name = storage_account_name
-      configuration = Config.current
-      @logger = configuration[:logger]
-      @azure_storage_container_prefix = configuration[:azure_storage_container_prefix]
-      @retention_time = configuration[:blob_retention_time] + 24 * 60 * 60
-      @not_notified_container = "#{configuration[:azure_storage_container_prefix]}-#{AZURE_STORAGE_ORPHAN_BLOBS_CONTAINER_NAME}"
+      @azure_storage_container_prefix = @configuration[:azure_storage_container_prefix]
+      @retention_time = @configuration[:blob_retention_time] + 24 * 60 * 60
+      @not_notified_container = "#{@configuration[:azure_storage_container_prefix]}-#{AZURE_STORAGE_ORPHAN_BLOBS_CONTAINER_NAME}"
       # launch tread that cleans the storage
       periodic_storage_cleanup
     end

data/lib/logstash/outputs/application_insights/telemetry.rb

@@ -99,7 +99,9 @@ class LogStash::Outputs::Application_insights
     end
 
     def flush
-      @telemetry_channel.flush
+      if @enable_telemetry_to_microsoft
+        @telemetry_channel.flush
+      end
     end
 
     public

data/lib/logstash/outputs/application_insights/test_notification.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+class LogStash::Outputs::Application_insights
+  class Test_notification < Blob
+
+
+    def initialize
+      # super first parameter must be nil. blob first parameter is channel, otherwise it will pass storage_account_name as channel
+      super( nil )
+      @storage_account_name = @configuration[:storage_account_name_key][0][0]
+      @action = :test_notification
+      @info = "#{@action}"
+      @recoverable = [ :invalid_instrumentation_key, :invalid_table_id ]
+      @force_client = true # to enable get a client even if all storage_accounts marked dead
+      @container_name = "logstash-test-container"
+      @blob_name = "logstash-test-blob"
+      @table_id = GUID_NULL
+      @instrumentation_key = GUID_NULL
+    end
+
+    def submit
+      @max_tries = 1
+      storage_io_block {
+        if @recovery.nil?
+          set_blob_sas_url
+          payload = create_payload
+          post_notification( @client.notifyClient, payload )
+        end
+      }
+    end
+  end
+end

data/lib/logstash/outputs/application_insights/test_storage.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+class LogStash::Outputs::Application_insights
+  class Test_storage < Blob
+
+
+    def initialize ( storage_account_name )
+      # super first parameter must be nil. blob first parameter is channel, otherwise it will pass storage_account_name as channel
+      super( nil )
+      @storage_account_name = storage_account_name
+      @action = :test_storage
+      @info = "#{@action} #{@storage_account_name}"
+      @recoverable = [ :invalid_storage_key, :container_exist, :create_container ]
+      @force_client = true # to enable get a client even if all storage_accounts marked dead
+    end
+
+    def submit
+      @max_tries = 1
+      storage_io_block {
+        if @recovery.nil? || :invalid_storage_key == @recovery
+          @client.blobClient.create_container( @configuration[:test_storage_container] ) unless @configuration[:disable_blob_upload]
+        end
+      }
+    end
+  end
+end

data/lib/logstash/outputs/application_insights/validate_notification.rb

@@ -20,18 +20,16 @@
 # ----------------------------------------------------------------------------------
 
 class LogStash::Outputs::Application_insights
-  class Validate_notification < Blob
+  class Validate_notification
 
     public
 
     def initialize
-      # super first parameter must be nil. blob first parameter is channel, otherwise it will pass storage_account_name as channel
-      super( nil )
-      @storage_account_name_key = @configuration[:storage_account_name_key]
+      @test_notification = Test_notification.new
     end
 
     def validate
-      {:success => test_notification( @storage_account_name_key[0][0] ), :error => @last_io_exception }
+      {:success => @test_notification.submit, :error => @test_notification.last_io_exception }
     end
   end
 end

data/lib/logstash/outputs/application_insights/validate_storage.rb

@@ -20,20 +20,20 @@
 # ----------------------------------------------------------------------------------
 
 class LogStash::Outputs::Application_insights
-  class Validate_storage < Blob
+  class Validate_storage
 
     public
 
     def initialize
-      # super first parameter must be nil. blob first parameter is channel, otherwise it will pass storage_account_name as channel
-      super( nil )
-      @storage_account_name_key = @configuration[:storage_account_name_key]
+      configuration = Config.current
+      @storage_account_name_key = configuration[:storage_account_name_key]
     end
 
     def validate
       result = {}
       @storage_account_name_key.each do |storage_account_name, storage_account_keys|
-        result[storage_account_name] = {:success => test_storage( storage_account_name ), :error => @last_io_exception }
+        test_storage = Test_storage.new( storage_account_name )
+        result[storage_account_name] = {:success => test_storage.submit, :error => test_storage.last_io_exception }
       end
       result
     end

data/lib/logstash/outputs/application_insights/version.rb

@@ -20,5 +20,5 @@
 # ----------------------------------------------------------------------------------
 
 # class LogStash::Outputs::Application_insights
-  APPLICATION_INSIGHTS_VERSION ||= "0.1.5"
+  APPLICATION_INSIGHTS_VERSION ||= "0.1.6"
 # end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-application_insights
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Microsoft Corporation
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-14 00:00:00.000000000 Z
+date: 2016-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -119,6 +119,8 @@ files:
 - lib/logstash/outputs/application_insights/storage_recovery.rb
 - lib/logstash/outputs/application_insights/sub_channel.rb
 - lib/logstash/outputs/application_insights/telemetry.rb
+- lib/logstash/outputs/application_insights/test_notification.rb
+- lib/logstash/outputs/application_insights/test_storage.rb
 - lib/logstash/outputs/application_insights/timer.rb
 - lib/logstash/outputs/application_insights/utils.rb
 - lib/logstash/outputs/application_insights/validate_notification.rb