logstash-output-application_insights 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bfd7fa91c562aaf482965ae4b0243dc64529f053
-  data.tar.gz: 605bd30348073847a7eb42cc2d492f8c8b088fcb
+  metadata.gz: 771238e72676111249e1ff68b5d04dd9c3fdad43
+  data.tar.gz: d5d25c0a1e678c27e9b034f1a63f5925cdd0ddfc
 SHA512:
-  metadata.gz: 019136969939d09b9fc152155f226c118aef4328909a94bd168ca76b212f69b73f4eb52b5c1aa5e953e3c1601be14712cc8fdf370506d60ecaa4e20252b0ed65
-  data.tar.gz: e9b0cddd450f56f5afd3379418e71494d7d88ded7f365d8bcf6f4add8233e933b7367aa1fb542ed38ec80d75963debccbf9cb30a90ed244bbe216aca2de03252
+  metadata.gz: d66c4886bb6aa1b08f0840b4090cb5d8874ca12af83de60b3bad38dfe7fbcc668215f2bc57b84ecc8ba075e26bbdcd94fefd6fa807990378823683432fe5135c
+  data.tar.gz: bfd12398aaf0c5a82d0256b71b9207af816bac5715ea4812079c04ebce46a73b392f0befae7b5c51d0215fbe1a779c6cf625e460e33be131d8d3a2dbd101d324

data/lib/logstash/outputs/application_insights/block.rb

@@ -40,6 +40,13 @@ class LogStash::Outputs::Application_insights
       @@semaphore.synchronize { @@Block_number = ( @@Block_number + 1 ) % 1000000 }
     end
 
+    def self.generate_block_numbers ( count )
+      @@semaphore.synchronize { 
+        firstNumber = ( @@Block_number + 1 ) % 1000000 
+        @@Block_number = ( @@Block_number + count ) % 1000000
+        firstNumber
+      }
+    end
 
 
     def initialize ( event_separator = "" )

data/lib/logstash/outputs/application_insights/channel.rb

@@ -129,8 +129,6 @@ class LogStash::Outputs::Application_insights
     end
 
     def recover_later_file_upload( file_to_upload )
-      # start the file from the begining
-      file_to_upload.close_read
       @failed_on_file_upload_retry_Q << file_to_upload
     end
 

data/lib/logstash/outputs/application_insights/client.rb

@@ -56,13 +56,17 @@ class LogStash::Outputs::Application_insights
     def blobClient
       raise UnexpectedBranchError, "client already disposed" unless @tuple
       @last_client_type = :blobClient
-      @current_azure_storage_client.blobClient
+      # breaking change after azure-storage 0.10.1 
+      # @current_azure_storage_client.blobClient
+      @current_azure_storage_client.blob_client
     end
 
     def tableClient
       raise UnexpectedBranchError, "client already disposed" unless @tuple
       @last_client_type = :blobClient
-      @current_azure_storage_client.tableClient
+      # breaking change after azure-storage 0.10.1 
+      # @current_azure_storage_client.tableClient
+      @current_azure_storage_client.table_client
     end
 
     def notifyClient
@@ -109,7 +113,9 @@ class LogStash::Outputs::Application_insights
       options[:ca_file] = configuration[:ca_file] unless configuration[:ca_file].empty?
 
       @current_azure_storage_client = Azure::Storage::Client.new( options )
-      @current_azure_storage_auth_sas = Azure::Storage::Auth::SharedAccessSignature.new( @storage_account_name, storage_access_key )
+      # breaking change after azure-storage 0.10.1 
+      # @current_azure_storage_auth_sas = Azure::Storage::Auth::SharedAccessSignature.new( @storage_account_name, storage_access_key )
+      @current_azure_storage_auth_sas = Azure::Storage::Core::Auth::SharedAccessSignature.new( @storage_account_name, storage_access_key )
     end
 
   end

data/lib/logstash/outputs/application_insights/exceptions.rb

@@ -43,6 +43,9 @@ class LogStash::Outputs::Application_insights
   class NotRecoverableError < StandardError
   end
 
+  class AssertError < StandardError
+  end
+
   # exception that cause process to end
   # LogStash::ConfigurationError, "ssl_truststore_location must be set when SSL is enabled"
   # class ConfigurationError < StandardError

data/lib/logstash/outputs/application_insights/local_file.rb

@@ -32,52 +32,53 @@ class LogStash::Outputs::Application_insights
 
     def initialize ( filename, is_gzip_file )
       @file_name = filename
-      @writer = write_file = File.new( @file_name, File::RDWR|File::CREAT )
+      @writer = write_file = File.new( @file_name, "wb" )
       @writer = Zlib::GzipWriter.new( write_file ) if is_gzip_file
       @read_file = nil
       @bytesize = 0
       @events_count = 0
       @first_block_number = nil
       @next_block_number = nil
-
+      @next_event_count = nil
     end
 
     def seal
-      if @writer
-        @writer.close
-        @writer = nil
-      end
+      @writer.close if @writer
+      @writer = nil
     end
 
-
     def close_read
       @read_file.close if @read_file
       @read_file = nil
     end
 
+    def open_read
+      @read_file = File.new( @file_name, "rb" ) # File.new( @file_name, File::RDWR )
+      @file_size =  @read_file.size
+      @blocks_num = ( @file_size + BLOB_BLOCK_MAX_BYTESIZE - 1 ) / BLOB_BLOCK_MAX_BYTESIZE
+      @events_per_block = @events_count / @blocks_num
+
+      @next_event_count = @events_per_block + ( @events_count % @blocks_num )
+      @first_block_number ||= Block.generate_block_numbers( @blocks_num )
+      @next_block_number = @first_block_number
+    end
 
     def get_next_block
       block = Block.new
-      block.done_time = @done_time
-      block.oldest_event_time = @oldest_event_time
-
-      unless @read_file
-        @read_file = File.new( @file_name, File::RDWR )
-        @file_size =  @read_file.size
-        @blocks_num = ( @file_size + BLOB_BLOCK_MAX_BYTESIZE - 1 ) / BLOB_BLOCK_MAX_BYTESIZE
-        @events_per_block = @events_count / @blocks_num
-        block.events_count = @events_per_block + ( @events_count % @blocks_num )
-        @next_block_number = @first_block_number ||= Block.generate_block_number
-        block.block_numbers = [ @first_block_number ]
-      else
-        block.block_numbers = [ @next_block_number ]
-        block.events_count = @events_per_block
-      end
-      @next_block_number += 1
       block.bytes = @read_file.read(BLOB_BLOCK_MAX_BYTESIZE)
       return nil if block.bytes.nil? || 0 == block.bytes.length
+
       block.bytesize = block.bytes.length
       State.instance.inc_upload_bytesize( block.bytesize )
+
+      block.done_time = @done_time
+      block.oldest_event_time = @oldest_event_time
+      block.block_numbers = [ @next_block_number ]
+      block.events_count = @next_event_count
+
+      @next_event_count = @events_per_block
+      @next_block_number += 1
+
       block
     end
 

data/lib/logstash/outputs/application_insights/notification.rb

@@ -60,7 +60,10 @@ class LogStash::Outputs::Application_insights
     def set_blob_sas_url
       blob_url ="https://#{@storage_account_name}.blob.#{@configuration[:azure_storage_host_suffix]}/#{@container_name}/#{@blob_name}"
       options_and_constrains = {:permissions => "r", :resource => "b", :expiry => ( Time.now.utc + @configuration[:blob_access_expiry_time] ).iso8601 }
-      @blob_sas_url = @client.storage_auth_sas.signed_uri( URI( blob_url ), options_and_constrains )
+      # breaking change after azure-storage 0.10.1 
+      # @blob_sas_url = @client.storage_auth_sas.signed_uri( URI( blob_url ), options_and_constrains )
+      use_account_sas = false
+      @blob_sas_url = @client.storage_auth_sas.signed_uri( URI( blob_url ), use_account_sas, options_and_constrains )
     end
 
 

data/lib/logstash/outputs/application_insights/upload_pipe.rb

@@ -107,20 +107,25 @@ class LogStash::Outputs::Application_insights
 
           break if :close == file_to_upload
 
-          @file_size = nil
+          file_to_upload.open_read
+          @file_size = file_to_upload.file_size
+
           while block = file_to_upload.get_next_block
-            @file_size ||= file_to_upload.file_size
             unless upload( block )
+              # start the file from the begining
+              file_to_upload.close_read
               @channel.recover_later_file_upload( file_to_upload )
               file_to_upload = nil
-              @uploaded_block_ids = [  ]
               break
             end
           end
-          file_to_upload.dispose if file_to_upload
-          file_to_upload = nil
 
-          commit unless @uploaded_block_ids.empty?
+          if file_to_upload
+            commit unless @uploaded_block_ids.empty?
+            file_to_upload.dispose
+            file_to_upload = nil
+          end
+
           @uploaded_block_ids = [  ]
         end
       end
@@ -187,7 +192,11 @@ class LogStash::Outputs::Application_insights
         if @file_pipe
           # remove "loading" record from state table of all previous blocks uploaded , we will try the whole file on an alternative storage
           @storage_recovery.recover_later( context_to_tuple, :state_table_update, @storage_account_name )
+          # memory is decrmeneted because the retry is done from the begining of the file
+          bytesize = @block_to_upload.bytesize
+          @block_to_upload.dispose
           @block_to_upload = nil
+          State.instance.dec_upload_bytesize( bytesize )
           return
         else
           info1 = "#{:commit} #{@storage_account_name}/#{@container_name}/#{@blob_name}, events: #{@uploaded_events_count}, size: #{@uploaded_bytesize}, blocks: #{@uploaded_block_numbers}, delay: #{Time.now.utc - @oldest_event_time}"

data/lib/logstash/outputs/application_insights/version.rb

@@ -20,5 +20,5 @@
 # ----------------------------------------------------------------------------------
 
 # class LogStash::Outputs::Application_insights
-  APPLICATION_INSIGHTS_VERSION ||= "0.2.1"
+  APPLICATION_INSIGHTS_VERSION ||= "0.2.2"
 # end

data/logstash-output-application-insights.gemspec

@@ -43,8 +43,13 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 6.0.0"
-  s.add_runtime_dependency "azure-storage", "0.10.1.preview"
-  s.add_runtime_dependency "azure-core", "0.1.2"
-  s.add_runtime_dependency "application_insights", ">= 0.5.3"
+  # s.add_runtime_dependency "azure-storage", "0.10.1.preview"
+  # s.add_runtime_dependency "azure-storage", "0.11.3.preview"
+  s.add_runtime_dependency "azure-storage", "~> 0.11"
+  # s.add_runtime_dependency "azure-core", "0.1.2"
+  # s.add_runtime_dependency "azure-core", "0.1.5"
+  s.add_runtime_dependency "azure-core", "~> 0.1"
+  # s.add_runtime_dependency "application_insights", "0.5.3"
+  s.add_runtime_dependency "application_insights", "~> 0.5"
   s.add_development_dependency "logstash-devutils"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-application_insights
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Microsoft Corporation
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-13 00:00:00.000000000 Z
+date: 2016-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -33,45 +33,45 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.10.1.preview
+        version: '0.11'
   name: azure-storage
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.10.1.preview
+        version: '0.11'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: '0.1'
   name: azure-core
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.3
+        version: '0.5'
   name: application_insights
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.3
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: