logstash-output-application_insights 0.1.3 → 0.1.4

data/lib/logstash/outputs/application_insights/storage_cleanup.rb

@@ -90,7 +90,7 @@ class LogStash::Outputs::Application_insights
       @info  = "#{@action} #{@storage_account_name}"
 
       containers = nil
-      success =  storage_io_block( proc do |reason, e| end ) {
+      success =  storage_io_block {
         options = { :metadata => true }
         options[:marker] = token if token
         options[:prefix] = azure_storage_container_prefix if azure_storage_container_prefix
@@ -115,7 +115,7 @@ class LogStash::Outputs::Application_insights
       continuation_token = nil
       filter = "#{:container_name} eq '#{container_name}' and #{:log_state} ne '#{:notified}'"
       begin
-        entities = log_to_table_query( @storage_account_name, filter , continuation_token )
+        entities = state_table_query( @storage_account_name, filter , continuation_token )
         return nil unless entities
         token = entities.continuation_token
         entities.each do |entity|
@@ -140,7 +140,7 @@ class LogStash::Outputs::Application_insights
       @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :blob_exit, :create_container, :container_exist ]
       @info  = "#{@action} #{@storage_account_name}/#{container_name}/#{blob_name}"
       tuple = nil
-      success =  storage_io_block( proc do |reason, e| end ) {
+      success =  storage_io_block {
         create_exist_recovery( :container, @not_notified_container ) { |name| @client.blobClient.create_container( name ) }
         if :blob_exit == @recovery
           tuple = ["", :pending]
@@ -157,7 +157,7 @@ class LogStash::Outputs::Application_insights
       @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :create_resource, :create_container, :container_exist ]
       @info  = "#{@action} #{@storage_account_name}/#{@not_notified_container}/#{blob_name}"
       status = nil
-      success =  storage_io_block( proc do |reason, e| end ) {
+      success =  storage_io_block {
         create_exist_recovery( :container, @not_notified_container ) { |name| @client.blobClient.create_container( name ) }
         if :create_resource == @recovery
           status = :not_started
@@ -177,11 +177,11 @@ class LogStash::Outputs::Application_insights
       continuation_token = nil
       filter = "#{:container_name} eq '#{container_name}'"
       begin
-        entities = log_to_table_query( @storage_account_name, filter , continuation_token )
+        entities = state_table_query( @storage_account_name, filter , continuation_token )
         return nil unless entities
         token = entities.continuation_token
         entities.each do |entity|
-          return nil unless log_to_table_delete( table_entity_to_tuple( entity.properties ) )
+          return nil unless state_table_delete( table_entity_to_tuple( entity.properties ) )
         end
       end while continuation_token
       true
@@ -194,7 +194,7 @@ class LogStash::Outputs::Application_insights
       @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :create_container ]
       @info  = "#{@action} #{@storage_account_name}/#{container_name}"
 
-      success =  storage_io_block( proc do |reason, e| end ) {
+      success =  storage_io_block {
         # delete container, if not found, skip
         containers = @client.blobClient.delete_container( container_name )  unless :create_container == @recovery
       }

data/lib/logstash/outputs/application_insights/storage_recovery.rb

@@ -0,0 +1,104 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+class LogStash::Outputs::Application_insights
+  class Storage_recovery
+
+    public
+
+    def initialize
+      configuration = Config.current
+      @logger = configuration[:logger]
+      @storage_account_name_key = configuration[:storage_account_name_key]
+      @queues = { :commit => {}, :notify => {}, :state_table_update => {} }
+      init_queues( @storage_account_name_key, @queues )
+
+      @closing = nil
+      @threads = []
+    end
+
+    def start
+      @storage_account_name_key.each do |storage_account_name, storage_account_keys|
+        # a threads, per storage  account name
+        @queues.each_key do |action|
+          @threads << recovery_thread( storage_account_name, action )
+        end
+      end
+    end
+
+    def recover_later ( tuple, action, storage_account_name )
+      @queues[action][storage_account_name] << tuple
+    end
+
+    def close
+      @closing = true
+      # @threads.each do |thread|
+      #   thread.join
+      # end
+    end
+
+    private
+
+    def stopped?
+      @closing
+    end
+
+    def init_queues ( storage_account_name_key, queues )
+      storage_account_name_key.each do |storage_account_name, storage_account_keys|
+        queues.each_key do |action|
+          queues[action][storage_account_name] = Queue.new
+        end
+      end
+    end
+
+    def recovery_thread( storage_account_name, action )
+      # a threads, per storage  account name, that retries actions on storage
+      Thread.new( storage_account_name, action ) do |storage_account_name, action|
+        counter = Concurrent::AtomicFixnum.new(0)
+        queue = @queues[action][storage_account_name]
+        loop do
+          tuple = queue.pop
+          Stud.stoppable_sleep(Float::INFINITY, 1) { state_on?( storage_account_name ) && 10 > counter.value }
+
+          counter.increment
+          Thread.new( action, counter, tuple ) do |action, counter, tuple|
+            Blob.new.send( action, tuple )
+            counter.decrement
+          end
+          tuple = nil # release for GC
+        end
+      end
+    end
+
+    def state_on? ( storage_account_name )
+      Clients.instance.storage_account_state_on?( storage_account_name )
+    end
+
+    public
+
+    @@instance = Storage_recovery.new
+
+    def self.instance
+      @@instance
+    end
+
+    private_class_method :new
+  end
+end

data/lib/logstash/outputs/application_insights/utils.rb

@@ -111,6 +111,22 @@ class LogStash::Outputs::Application_insights
       s =~ /\A(?<hostname>([A-Za-z0-9\.\-]+)|\[[0-9A-Fa-f\:]+\])(:(?<port>\d+))?\z/
     end
  
+    def self.ip4_address? ( s )
+      s =~ /\A(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\z/
+    end
+
+    def self.ip6_address? ( s )
+      s =~ /\A(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\z/
+    end
+
+    def self.dns_address? ( s )
+      s =~ /\A(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])\z/
+    end
+
+    def self.hosts_address? ( s )
+      ip4_address?( s ) || ip6_address?( s ) || dns_address?( s )
+    end
+
     def self.to_storage_name ( s )
       s.nil? ? nil : s.downcase.gsub(/[^0-9a-z]/i, '')
     end
@@ -134,6 +150,7 @@ class LogStash::Outputs::Application_insights
       end
       new_hash
     end
+
   end
 
 end

data/lib/logstash/outputs/application_insights/validate_notification.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+
+class LogStash::Outputs::Application_insights
+  class Validate_notification < Blob
+
+    public
+
+    def initialize
+      # super first parameter must be nil. blob first parameter is channel, otherwise it will pass storage_account_name as channel
+      super( nil )
+      @storage_account_name_key = @configuration[:storage_account_name_key]
+    end
+
+    def validate
+      {:success => test_notification( @storage_account_name_key[0][0] ), :error => @last_io_exception }
+    end
+  end
+end

data/lib/logstash/outputs/application_insights/validate_storage.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+
+class LogStash::Outputs::Application_insights
+  class Validate_storage < Blob
+
+    public
+
+    def initialize
+      # super first parameter must be nil. blob first parameter is channel, otherwise it will pass storage_account_name as channel
+      super( nil )
+      @storage_account_name_key = @configuration[:storage_account_name_key]
+    end
+
+    def validate
+      result = {}
+      @storage_account_name_key.each do |storage_account_name, storage_account_keys|
+        result[storage_account_name] = {:success => test_storage( storage_account_name ), :error => @last_io_exception }
+      end
+      result
+    end
+  end
+end

data/lib/logstash/outputs/application_insights/version.rb

@@ -20,5 +20,5 @@
 # ----------------------------------------------------------------------------------
 
 # class LogStash::Outputs::Application_insights
-  APPLICATION_INSIGHTS_VERSION ||= "0.1.3"
+  APPLICATION_INSIGHTS_VERSION ||= "0.1.4"
 # end

data/logstash-output-application-insights.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |s|
   s.version       = APPLICATION_INSIGHTS_VERSION
   s.licenses      = ['Apache License (2.0)']
   s.summary       = "Microsoft Application Insights openschema Logstash output plugin. 'Output events to Application Insights"
-  s.description   = "Outputs events to Microsoft Application Insights Analytics. This gem is a Logstash output plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.description   = "Outputs events to Microsoft Application Insights Analytics open schema tables. This gem is a Logstash output plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors       = ["Microsoft Corporation"]
   s.email         = "info@microsoft.com"
   s.homepage      = "https://github.com/Microsoft/logstash-output-application-insights"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-application_insights
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Microsoft Corporation
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-22 00:00:00.000000000 Z
+date: 2016-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -86,7 +86,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Outputs events to Microsoft Application Insights Analytics. This gem is a Logstash output plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: Outputs events to Microsoft Application Insights Analytics open schema tables. This gem is a Logstash output plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@microsoft.com
 executables: []
 extensions: []
@@ -111,13 +111,18 @@ files:
 - lib/logstash/outputs/application_insights/exceptions.rb
 - lib/logstash/outputs/application_insights/flow_control.rb
 - lib/logstash/outputs/application_insights/multi_io_logger.rb
+- lib/logstash/outputs/application_insights/notification_recovery.rb
 - lib/logstash/outputs/application_insights/shutdown.rb
+- lib/logstash/outputs/application_insights/shutdown_recovery.rb
 - lib/logstash/outputs/application_insights/state.rb
 - lib/logstash/outputs/application_insights/storage_cleanup.rb
+- lib/logstash/outputs/application_insights/storage_recovery.rb
 - lib/logstash/outputs/application_insights/sub_channel.rb
 - lib/logstash/outputs/application_insights/telemetry.rb
 - lib/logstash/outputs/application_insights/timer.rb
 - lib/logstash/outputs/application_insights/utils.rb
+- lib/logstash/outputs/application_insights/validate_notification.rb
+- lib/logstash/outputs/application_insights/validate_storage.rb
 - lib/logstash/outputs/application_insights/version.rb
 - logstash-output-application-insights.gemspec
 - spec/outputs/application_insights_spec.rb