logstash-output-application_insights 0.1.6 → 0.2.0

data/lib/logstash/outputs/application_insights/constants.rb

@@ -73,6 +73,7 @@ class LogStash::Outputs::Application_insights
       :blob_serialization => @blob_serialization || DEFAULT_BLOB_SERIALIZATION,
       :csv_separator => @csv_separator || DEFAULT_CSV_SEPARATOR,
       :csv_default_value => @csv_default_value || DEFAULT_CSV_DEFAULT_VALUE,
+      :disable_compression => @disable_compression || DEFAULT_DISABLE_COMPRESSION,
 
     }
   end
@@ -82,6 +83,7 @@ class LogStash::Outputs::Application_insights
                           :disable_cleanup, :delete_not_notified_blobs,
                           :validate_notification, :validate_storage,
                           :save_notified_blobs_records, :case_insensitive_columns,
+                          :disable_compression,
                           :table_columns, :serialized_event_field ]
 
   GUID_NULL =                    "00000000-0000-0000-0000-000000000000"
@@ -140,6 +142,9 @@ class LogStash::Outputs::Application_insights
   MIN_FLOW_CONTROL_DELAY = 0.1                            # in seconds, 1 seconds, can be less than 1 seconds, like 0.5, 0.1
   MAX_FLOW_CONTROL_DELAY = 0                              # in seconds, 1 seconds, can be less than 1 seconds, like 0.5, 0.1
 
+  MAX_CHANNEL_UPLOAD_PIPES = 40
+  CHANNEL_THRESHOLD_TO_ADD_UPLOAD_PIPE = 3                # not relevant for file upload mode
+
   METADATA_FIELD_INSTRUMENTATION_KEY = "[@metadata]instrumentation_key"
   METADATA_FIELD_TABLE_ID = "[@metadata]table_id"
   FIELD_INSTRUMENTATION_KEY = "instrumentation_key"
@@ -151,6 +156,7 @@ class LogStash::Outputs::Application_insights
   AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX = "logstash" # lower case only, dash allowed
   AZURE_STORAGE_BLOB_LOGSTASH_PREFIX = "logstash"
   AZURE_STORAGE_TABLE_LOGSTASH_PREFIX = "Logstash" # case sensitive, no dash
+  LOCAL_FS_FILE_PREFIX = "application_insights"
 
   AZURE_STORAGE_ORPHAN_BLOBS_CONTAINER_NAME = "orphan-blobs"
   AZURE_STORAGE_NOTIFIED_BLOBS_TABLE_NAME = "orphan-blobs"
@@ -174,14 +180,11 @@ class LogStash::Outputs::Application_insights
   DEFAULT_AZURE_STORAGE_BLOB_PREFIX = nil
   DEFAULT_AZURE_STORAGE_CONTAINER_PREFIX = nil
   DEFAULT_AZURE_STORAGE_TABLE_PREFIX = nil
-  DEFAULT_JSON_EXT = EXT_EVENT_FORMAT_JSON
-  DEFAULT_CSV_EXT = EXT_EVENT_FORMAT_CSV
-
 
   DEFAULT_BLOB_SERIALIZATION = EXT_EVENT_FORMAT_JSON
 
-  DEFAULT_BLOB_MAX_BYTESIZE = 1 * 1024 * 1024 * 1024
-  DEFAULT_BLOB_MAX_EVENTS = 256 * 1024                 # 256 Kilo events
+  DEFAULT_BLOB_MAX_BYTESIZE = 4 * 1024 * 1024 * 1024    # 4 Giga bytes
+  DEFAULT_BLOB_MAX_EVENTS = 1000 * 1000                 # 1,000,000
 
   DEFAULT_BLOB_MAX_DELAY = 60                         # in seconds
   DEFAULT_BLOB_RETENTION_TIME = 60 * 60 * 24 * 7      # in seconds - one week
@@ -197,6 +200,7 @@ class LogStash::Outputs::Application_insights
   DEFAULT_DISABLE_CLEANUP = false
   DEFAULT_DELETE_NOT_NOTIFIED_BLOBS = false
   DEFAULT_SAVE_NOTIFIED_BLOBS_RECORDS = false
+  DEFAULT_DISABLE_COMPRESSION = false
   
   DEFAULT_CASE_INSENSITIVE = false
 

data/lib/logstash/outputs/application_insights/context.rb

@@ -0,0 +1,97 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+
+class LogStash::Outputs::Application_insights
+  class Context
+
+    def initialize ( tuple )
+      if tuple
+        tuple_to_context( tuple )
+      else
+        clear_context
+      end
+    end
+
+    def clear_context
+      @start_time = Time.now.utc
+      @action = nil
+      @instrumentation_key = nil
+      @table_id = nil
+      @storage_account_name = nil
+      @container_name = nil
+      @blob_name = nil
+      @uploaded_block_ids = [  ]
+      @uploaded_block_numbers = [  ]
+      @uploaded_events_count = 0
+      @uploaded_bytesize = 0
+      @oldest_event_time = nil
+      @event_format = nil
+      @blob_max_delay = nil
+      @log_state = nil
+      @sub_state = :none
+      @file_size = 0
+    end
+
+    def table_entity_to_tuple( options = {} )
+      [ options[:start_time.to_s] || Time.now.utc, options[:action.to_s], options[:instrumentation_key.to_s], options[:table_id.to_s],
+        options[:storage_account_name.to_s], options[:container_name.to_s], options[:blob_name.to_s],
+        eval( options[:uploaded_block_ids.to_s] ), eval( options[:uploaded_block_numbers.to_s] ),
+        options[:uploaded_events_count.to_s] || 0, options[:uploaded_bytesize.to_s] || 0, options[:oldest_event_time.to_s] || Time.now.utc,
+        options[:event_format.to_s], options[:blob_max_delay.to_s] || 0,
+        options[:log_state.to_s].to_sym, (options[:sub_state.to_s] || :none).to_sym, options[:file_size.to_s] || 0
+      ]
+    end
+
+    def table_entity_to_context ( table_entity )
+      tuple_to_context( table_entity_to_tuple( table_entity ) )
+    end
+
+
+    def context_to_tuple
+      [ @start_time || Time.now.utc, @action, @instrumentation_key, @table_id,
+        @storage_account_name, @container_name, @blob_name,
+        @uploaded_block_ids, @uploaded_block_numbers,
+        @uploaded_events_count, @uploaded_bytesize, @oldest_event_time,
+        @event_format, @blob_max_delay,
+        @log_state, @sub_state, @file_size
+      ]
+    end
+
+    def tuple_to_context ( tuple )
+      ( @start_time, @action, @instrumentation_key, @table_id,
+        @storage_account_name, @container_name, @blob_name,
+        @uploaded_block_ids, @uploaded_block_numbers,
+        @uploaded_events_count, @uploaded_bytesize, @oldest_event_time,
+        @event_format, @blob_max_delay,
+        @log_state, @sub_state, @file_size ) = tuple
+    end
+
+    def context_to_table_entity
+      { :start_time => @start_time, :instrumentation_key => @instrumentation_key, :table_id => @table_id,
+        :storage_account_name => @storage_account_name, :container_name => @container_name, :blob_name => @blob_name,
+        :uploaded_block_ids => @uploaded_block_ids.to_s, :uploaded_block_numbers => @uploaded_block_numbers.to_s,
+        :uploaded_events_count => @uploaded_events_count, :uploaded_bytesize => @uploaded_bytesize, :oldest_event_time => @oldest_event_time,
+        :log_state => @log_state, :sub_state => @sub_state, :file_size => @file_size
+      }
+    end
+
+  end
+end

data/lib/logstash/outputs/application_insights/local_file.rb

@@ -0,0 +1,113 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+
+class LogStash::Outputs::Application_insights
+  class Local_file
+
+    attr_reader :bytesize
+    attr_reader :events_count
+    attr_reader :oldest_event_time
+
+    attr_reader :file_size
+
+    public
+
+    def initialize ( filename, is_gzip_file )
+      @file_name = filename
+      @writer = write_file = File.new( @file_name, File::RDWR|File::CREAT )
+      @writer = Zlib::GzipWriter.new( write_file ) if is_gzip_file
+      @read_file = nil
+      @bytesize = 0
+      @events_count = 0
+      @first_block_number = nil
+      @next_block_number = nil
+
+    end
+
+    def seal
+      if @writer
+        @writer.close
+        @writer = nil
+      end
+    end
+
+
+    def close_read
+      @read_file.close if @read_file
+      @read_file = nil
+    end
+
+
+    def get_next_block
+      block = Block.new
+      block.done_time = @done_time
+      block.oldest_event_time = @oldest_event_time
+
+      unless @read_file
+        @read_file = File.new( @file_name, File::RDWR )
+        @file_size =  @read_file.size
+        @blocks_num = ( @file_size + BLOB_BLOCK_MAX_BYTESIZE - 1 ) / BLOB_BLOCK_MAX_BYTESIZE
+        @events_per_block = @events_count / @blocks_num
+        block.events_count = @events_per_block + ( @events_count % @blocks_num )
+        @next_block_number = @first_block_number ||= Block.generate_block_number
+        block.block_numbers = [ @first_block_number ]
+      else
+        block.block_numbers = [ @next_block_number ]
+        block.events_count = @events_per_block
+      end
+      @next_block_number += 1
+      block.bytes = @read_file.read(BLOB_BLOCK_MAX_BYTESIZE)
+      return nil if block.bytes.nil? || 0 == block.bytes.length
+      block.bytesize = block.bytes.length
+      State.instance.inc_upload_bytesize( block.bytesize )
+      block
+    end
+
+
+    def << ( block )
+      @bytesize += block.bytesize
+      @events_count += block.events_count
+
+      @writer.write( block.bytes )
+      State.instance.dec_upload_bytesize( block.bytesize )
+
+      @oldest_event_time = block.oldest_event_time if @oldest_event_time.nil? || block.oldest_event_time < @oldest_event_time
+      @done_time = block.done_time if @done_time.nil? || block.done_time > @done_time
+    end
+
+
+    def dispose
+      @bytesize = nil
+      @events_count = nil
+      @done_time = nil
+      @oldest_event_time = nil
+      seal
+      close_read
+      File.delete( @file_name ) if @file_name
+      @file_name = nil
+    end
+
+
+    private
+
+  end
+
+end

data/lib/logstash/outputs/application_insights/notification.rb

@@ -0,0 +1,116 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+class LogStash::Outputs::Application_insights
+  class Notification < Blob
+
+    def initialize ( tuple )
+      # super first parameter must be nil. blob first parameter is channel, otherwise it will pass storage_account_name as channel
+      super( tuple )
+    end
+
+    # must return whether notification was successful or failed
+    def notify
+      @action = :notify
+      @recoverable = [ :notify_failed_blob_not_accessible, :io_failure, :service_unavailable, :notification_process_down ]
+      @force_client = true # to enable get a client even if all storage_accounts marked dead
+      @info = "#{@action} #{@storage_account_name}/#{@container_name}/#{@blob_name}, events: #{@uploaded_events_count}, size: #{@uploaded_bytesize}, blocks: #{@uploaded_block_numbers}, delay: #{Time.now.utc - @oldest_event_time}"
+      success = storage_io_block { submit }
+      if success
+        Telemetry.instance.track_event { { :name => "notified", :properties => context_to_table_entity } }
+        state_table_update
+      else
+        notify_retry_later
+      end
+      success
+    end
+
+    private
+
+    def submit
+        set_blob_sas_url
+        @logger.debug { "blob_sas_url: #{@blob_sas_url}" }
+        payload = create_payload
+        @logger.debug { "notification payload: #{payload}" }
+
+        # assume that exceptions can be raised due to this method:
+        post_notification( @client.notifyClient, payload ) unless @configuration[:disable_notification]
+
+        @log_state = :notified
+    end
+
+
+    def set_blob_sas_url
+      blob_url ="https://#{@storage_account_name}.blob.#{@configuration[:azure_storage_host_suffix]}/#{@container_name}/#{@blob_name}"
+      options_and_constrains = {:permissions => "r", :resource => "b", :expiry => ( Time.now.utc + @configuration[:blob_access_expiry_time] ).iso8601 }
+      @blob_sas_url = @client.storage_auth_sas.signed_uri( URI( blob_url ), options_and_constrains )
+    end
+
+
+    def create_payload
+      notification_hash = {
+        :data => {
+          :baseType => DATA_BASE_TYPE,
+          :baseData => {
+            :ver           => BASE_DATA_REQUIRED_VERSION,
+            :blobSasUri    => @blob_sas_url.to_s,
+            :sourceName    => @table_id,
+            :sourceVersion => @configuration[:notification_version].to_s
+          }
+        }, 
+        :ver  => @configuration[:notification_version], 
+        :name => REQUEST_NAME,
+        :time => Time.now.utc.iso8601,
+        :iKey => @instrumentation_key
+      }
+      notification_hash.to_json
+    end 
+
+
+    def post_notification ( http_client, body )
+      request = Azure::Core::Http::HttpRequest.new( :post, @configuration[:application_insights_endpoint], { :body => body, :client => http_client } )
+      request.headers['Content-Type'] = 'application/json; charset=utf-8'
+      request.headers['Accept'] = 'application/json'
+      @logger.debug { "send notification : \n    endpoint: #{@configuration[:application_insights_endpoint]}\n    body : #{body}" }
+      response = request.call
+    end 
+
+
+    def notify_retry_later
+      if :notify_failed_blob_not_accessible == @recovery
+        @sub_state = @recovery
+        @storage_recovery.recover_later( context_to_tuple, :notify, @storage_account_name )
+
+      elsif :invalid_instrumentation_key == @recovery || :invalid_table_id == @recovery
+        @sub_state = @recovery
+        Channels.instance.channel( @instrumentation_key, @table_id ).recover_later_notification( context_to_tuple )
+
+      else
+        if :notify_failed_blob_not_accessible == @sub_state
+          @storage_recovery.recover_later( context_to_tuple, :notify, @storage_account_name )
+        elsif :invalid_instrumentation_key == @sub_state || :invalid_table_id == @sub_state
+          Channels.instance.channel( @instrumentation_key, @table_id ).recover_later_notification( context_to_tuple )
+        else
+          @notification_recovery.recover_later( context_to_tuple )
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/application_insights/notification_recovery.rb

@@ -66,19 +66,18 @@ class LogStash::Outputs::Application_insights
 
     def recovery_thread
       Thread.new do
-        blob = Blob.new
         counter = Concurrent::AtomicFixnum.new(0)
 
         loop do
           tuple = @queue.pop
-          Stud.stoppable_sleep(Float::INFINITY, 1) { ( state_on?( blob ) || stopped? ) && 10 > counter.value }
+          Stud.stoppable_sleep(Float::INFINITY, 1) { ( state_on? || stopped? ) && 10 > counter.value }
 
-          if stopped? && !state_on?( blob )
+          if stopped? && !state_on?
             recover_later( tuple )
           else
             counter.increment
             Thread.new( counter, tuple ) do |counter, tuple|
-              Blob.new.notify( tuple )
+              Notification.new( tuple ).notify
               counter.decrement
             end
           end
@@ -87,9 +86,9 @@ class LogStash::Outputs::Application_insights
       end
     end
 
-    def state_on? ( blob )
+    def state_on?
       return @notification_state_on if @notification_state_on
-      @notification_state_on = @test_notification.submit
+      @notification_state_on = @test_notification.test
       return @notification_state_on if @notification_state_on
       sleep( @resurrect_delay )
       @notification_state_on

data/lib/logstash/outputs/application_insights/shutdown_recovery.rb

@@ -27,7 +27,7 @@ class LogStash::Outputs::Application_insights
       configuration = Config.current
       @logger = configuration[:logger]
       @storage_account_name_key = configuration[:storage_account_name_key]
-      @partition_key_prefix =configuration[:azure_storage_blob_prefix].gsub( "/", "" )
+      @partition_key_prefix =configuration[:partition_key_prefix]
 
       @closing = nil
       @threads = []
@@ -85,7 +85,8 @@ class LogStash::Outputs::Application_insights
               entities.each do |entity|
                 typed_tuple = nil
                 until typed_tuple || stopped?
-                  typed_tuple = blob.update_commited_or_uncommited_list( entity.properties )
+                  blob.table_entity_to_context( entity.properties )
+                  typed_tuple = blob.update_commited_or_uncommited_list
                   Stud.stoppable_sleep(60, 1) { stopped? } unless typed_tuple
                 end
 

data/lib/logstash/outputs/application_insights/state_table.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+
+# ----------------------------------------------------------------------------------
+# Logstash Output Application Insights
+#
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+#
+# Licensed under the Apache License, Version 2.0 (the License); 
+# you may not use this file except in compliance with the License. 
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 
+#
+# Unless required by applicable law or agreed to in writing, software 
+# distributed under the License is distributed on an "AS IS" BASIS, 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+#
+# See the Apache Version 2.0 License for specific language governing 
+# permissions and limitations under the License.
+# ----------------------------------------------------------------------------------
+class LogStash::Outputs::Application_insights
+  class State_table < Blob
+
+    def initialize( tuple )
+      # super first parameter must be nil. blob first parameter is channel, otherwise it will pass storage_account_name as channel
+      super( nil )
+      tuple_to_state( tuple ) if tuple
+    end
+
+    # return true on success
+    def state_table_insert
+      @action = :state_table_insert
+      @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :table_exist, :create_table, :table_busy, :entity_exist ]
+      @info  = "#{@action} #{@log_state} #{@storage_account_name}/#{@container_name}/#{@blob_name}"
+      success =  storage_io_block {
+        create_table_exist_recovery
+        if :entity_exist == @recovery
+          raise NotRecoverableError if :uploading == @log_state
+        else
+          entity_values = state_to_table_entity
+          entity_values[:PartitionKey] = "#{@configuration[:partition_key_prefix]}-#{@log_state}"
+          entity_values[:RowKey] = @blob_name.gsub("/","_")
+          @client.tableClient.insert_entity( @configuration[:state_table_name], entity_values )
+        end
+      }
+      @storage_recovery.recover_later( state_to_tuple, :state_table_update, @storage_account_name )  unless success || :uploading == @log_state
+      success
+    end
+
+
+    def state_table_update
+      if :uploading == @log_state
+        state_table_delete
+      elsif :committed == @log_state
+        if state_table_insert && state_table_delete( :uploading )
+          State.instance.dec_pending_commits
+          State.instance.inc_pending_notifications
+          # this is not a recovery, it is actually enqueue to notify
+          @notification_recovery.enqueue( state_to_tuple )
+        end
+      elsif :notified == @log_state
+        if (!@configuration[:save_notified_blobs_records] || state_table_insert) && state_table_delete( :committed ) 
+          State.instance.dec_pending_notifications
+        end
+      end
+    end
+
+
+    # return tru on success
+    def state_table_delete ( state = nil )
+      state ||= @log_state
+      @action = :state_table_delete
+      @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :table_exist, :create_table, :table_busy, :create_resource ]
+      @info  = "#{@action} #{state} #{@storage_account_name}/#{@container_name}/#{@blob_name}"
+
+      success =  storage_io_block {
+        create_table_exist_recovery
+        if :create_resource == @recovery
+          @logger.info { "Note: delete entity failed, already deleted, #{@info}, state: #{state}, log_state: #{@log_state}" }
+        else
+          @client.tableClient.delete_entity( @configuration[:state_table_name], "#{@configuration[:partition_key_prefix]}-#{state}", @blob_name.gsub( "/", "_" ) )
+        end
+      }
+      @storage_recovery.recover_later( state_to_tuple, :state_table_update, @storage_account_name ) unless success
+      success
+    end
+
+
+    # return entities
+    def state_table_query ( storage_account_name, filter , token )
+      @storage_account_name = storage_account_name
+
+      @action = :state_table_query
+      @recoverable = [ :invalid_storage_key, :io_failure, :service_unavailable, :table_exist, :create_table, :table_busy ]
+      @info  = "#{@action} #{@storage_account_name}/#{@configuration[:state_table_name]}"
+
+      entities = nil
+      success =  storage_io_block {
+        create_table_exist_recovery
+        options = { :filter => filter }
+        options[:continuation_token] = token if token
+        entities = @client.tableClient.query_entities( @configuration[:state_table_name], options )
+      }
+      entities
+    end
+
+  end
+end