logstash-output-application_insights 0.2.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: bfd7fa91c562aaf482965ae4b0243dc64529f053
4
- data.tar.gz: 605bd30348073847a7eb42cc2d492f8c8b088fcb
3
+ metadata.gz: 771238e72676111249e1ff68b5d04dd9c3fdad43
4
+ data.tar.gz: d5d25c0a1e678c27e9b034f1a63f5925cdd0ddfc
5
5
  SHA512:
6
- metadata.gz: 019136969939d09b9fc152155f226c118aef4328909a94bd168ca76b212f69b73f4eb52b5c1aa5e953e3c1601be14712cc8fdf370506d60ecaa4e20252b0ed65
7
- data.tar.gz: e9b0cddd450f56f5afd3379418e71494d7d88ded7f365d8bcf6f4add8233e933b7367aa1fb542ed38ec80d75963debccbf9cb30a90ed244bbe216aca2de03252
6
+ metadata.gz: d66c4886bb6aa1b08f0840b4090cb5d8874ca12af83de60b3bad38dfe7fbcc668215f2bc57b84ecc8ba075e26bbdcd94fefd6fa807990378823683432fe5135c
7
+ data.tar.gz: bfd12398aaf0c5a82d0256b71b9207af816bac5715ea4812079c04ebce46a73b392f0befae7b5c51d0215fbe1a779c6cf625e460e33be131d8d3a2dbd101d324
@@ -40,6 +40,13 @@ class LogStash::Outputs::Application_insights
40
40
  @@semaphore.synchronize { @@Block_number = ( @@Block_number + 1 ) % 1000000 }
41
41
  end
42
42
 
43
+ def self.generate_block_numbers ( count )
44
+ @@semaphore.synchronize {
45
+ firstNumber = ( @@Block_number + 1 ) % 1000000
46
+ @@Block_number = ( @@Block_number + count ) % 1000000
47
+ firstNumber
48
+ }
49
+ end
43
50
 
44
51
 
45
52
  def initialize ( event_separator = "" )
@@ -129,8 +129,6 @@ class LogStash::Outputs::Application_insights
129
129
  end
130
130
 
131
131
  def recover_later_file_upload( file_to_upload )
132
- # start the file from the begining
133
- file_to_upload.close_read
134
132
  @failed_on_file_upload_retry_Q << file_to_upload
135
133
  end
136
134
 
@@ -56,13 +56,17 @@ class LogStash::Outputs::Application_insights
56
56
  def blobClient
57
57
  raise UnexpectedBranchError, "client already disposed" unless @tuple
58
58
  @last_client_type = :blobClient
59
- @current_azure_storage_client.blobClient
59
+ # breaking change after azure-storage 0.10.1
60
+ # @current_azure_storage_client.blobClient
61
+ @current_azure_storage_client.blob_client
60
62
  end
61
63
 
62
64
  def tableClient
63
65
  raise UnexpectedBranchError, "client already disposed" unless @tuple
64
66
  @last_client_type = :blobClient
65
- @current_azure_storage_client.tableClient
67
+ # breaking change after azure-storage 0.10.1
68
+ # @current_azure_storage_client.tableClient
69
+ @current_azure_storage_client.table_client
66
70
  end
67
71
 
68
72
  def notifyClient
@@ -109,7 +113,9 @@ class LogStash::Outputs::Application_insights
109
113
  options[:ca_file] = configuration[:ca_file] unless configuration[:ca_file].empty?
110
114
 
111
115
  @current_azure_storage_client = Azure::Storage::Client.new( options )
112
- @current_azure_storage_auth_sas = Azure::Storage::Auth::SharedAccessSignature.new( @storage_account_name, storage_access_key )
116
+ # breaking change after azure-storage 0.10.1
117
+ # @current_azure_storage_auth_sas = Azure::Storage::Auth::SharedAccessSignature.new( @storage_account_name, storage_access_key )
118
+ @current_azure_storage_auth_sas = Azure::Storage::Core::Auth::SharedAccessSignature.new( @storage_account_name, storage_access_key )
113
119
  end
114
120
 
115
121
  end
@@ -43,6 +43,9 @@ class LogStash::Outputs::Application_insights
43
43
  class NotRecoverableError < StandardError
44
44
  end
45
45
 
46
+ class AssertError < StandardError
47
+ end
48
+
46
49
  # exception that cause process to end
47
50
  # LogStash::ConfigurationError, "ssl_truststore_location must be set when SSL is enabled"
48
51
  # class ConfigurationError < StandardError
@@ -32,52 +32,53 @@ class LogStash::Outputs::Application_insights
32
32
 
33
33
  def initialize ( filename, is_gzip_file )
34
34
  @file_name = filename
35
- @writer = write_file = File.new( @file_name, File::RDWR|File::CREAT )
35
+ @writer = write_file = File.new( @file_name, "wb" )
36
36
  @writer = Zlib::GzipWriter.new( write_file ) if is_gzip_file
37
37
  @read_file = nil
38
38
  @bytesize = 0
39
39
  @events_count = 0
40
40
  @first_block_number = nil
41
41
  @next_block_number = nil
42
-
42
+ @next_event_count = nil
43
43
  end
44
44
 
45
45
  def seal
46
- if @writer
47
- @writer.close
48
- @writer = nil
49
- end
46
+ @writer.close if @writer
47
+ @writer = nil
50
48
  end
51
49
 
52
-
53
50
  def close_read
54
51
  @read_file.close if @read_file
55
52
  @read_file = nil
56
53
  end
57
54
 
55
+ def open_read
56
+ @read_file = File.new( @file_name, "rb" ) # File.new( @file_name, File::RDWR )
57
+ @file_size = @read_file.size
58
+ @blocks_num = ( @file_size + BLOB_BLOCK_MAX_BYTESIZE - 1 ) / BLOB_BLOCK_MAX_BYTESIZE
59
+ @events_per_block = @events_count / @blocks_num
60
+
61
+ @next_event_count = @events_per_block + ( @events_count % @blocks_num )
62
+ @first_block_number ||= Block.generate_block_numbers( @blocks_num )
63
+ @next_block_number = @first_block_number
64
+ end
58
65
 
59
66
  def get_next_block
60
67
  block = Block.new
61
- block.done_time = @done_time
62
- block.oldest_event_time = @oldest_event_time
63
-
64
- unless @read_file
65
- @read_file = File.new( @file_name, File::RDWR )
66
- @file_size = @read_file.size
67
- @blocks_num = ( @file_size + BLOB_BLOCK_MAX_BYTESIZE - 1 ) / BLOB_BLOCK_MAX_BYTESIZE
68
- @events_per_block = @events_count / @blocks_num
69
- block.events_count = @events_per_block + ( @events_count % @blocks_num )
70
- @next_block_number = @first_block_number ||= Block.generate_block_number
71
- block.block_numbers = [ @first_block_number ]
72
- else
73
- block.block_numbers = [ @next_block_number ]
74
- block.events_count = @events_per_block
75
- end
76
- @next_block_number += 1
77
68
  block.bytes = @read_file.read(BLOB_BLOCK_MAX_BYTESIZE)
78
69
  return nil if block.bytes.nil? || 0 == block.bytes.length
70
+
79
71
  block.bytesize = block.bytes.length
80
72
  State.instance.inc_upload_bytesize( block.bytesize )
73
+
74
+ block.done_time = @done_time
75
+ block.oldest_event_time = @oldest_event_time
76
+ block.block_numbers = [ @next_block_number ]
77
+ block.events_count = @next_event_count
78
+
79
+ @next_event_count = @events_per_block
80
+ @next_block_number += 1
81
+
81
82
  block
82
83
  end
83
84
 
@@ -60,7 +60,10 @@ class LogStash::Outputs::Application_insights
60
60
  def set_blob_sas_url
61
61
  blob_url ="https://#{@storage_account_name}.blob.#{@configuration[:azure_storage_host_suffix]}/#{@container_name}/#{@blob_name}"
62
62
  options_and_constrains = {:permissions => "r", :resource => "b", :expiry => ( Time.now.utc + @configuration[:blob_access_expiry_time] ).iso8601 }
63
- @blob_sas_url = @client.storage_auth_sas.signed_uri( URI( blob_url ), options_and_constrains )
63
+ # breaking change after azure-storage 0.10.1
64
+ # @blob_sas_url = @client.storage_auth_sas.signed_uri( URI( blob_url ), options_and_constrains )
65
+ use_account_sas = false
66
+ @blob_sas_url = @client.storage_auth_sas.signed_uri( URI( blob_url ), use_account_sas, options_and_constrains )
64
67
  end
65
68
 
66
69
 
@@ -107,20 +107,25 @@ class LogStash::Outputs::Application_insights
107
107
 
108
108
  break if :close == file_to_upload
109
109
 
110
- @file_size = nil
110
+ file_to_upload.open_read
111
+ @file_size = file_to_upload.file_size
112
+
111
113
  while block = file_to_upload.get_next_block
112
- @file_size ||= file_to_upload.file_size
113
114
  unless upload( block )
115
+ # start the file from the begining
116
+ file_to_upload.close_read
114
117
  @channel.recover_later_file_upload( file_to_upload )
115
118
  file_to_upload = nil
116
- @uploaded_block_ids = [ ]
117
119
  break
118
120
  end
119
121
  end
120
- file_to_upload.dispose if file_to_upload
121
- file_to_upload = nil
122
122
 
123
- commit unless @uploaded_block_ids.empty?
123
+ if file_to_upload
124
+ commit unless @uploaded_block_ids.empty?
125
+ file_to_upload.dispose
126
+ file_to_upload = nil
127
+ end
128
+
124
129
  @uploaded_block_ids = [ ]
125
130
  end
126
131
  end
@@ -187,7 +192,11 @@ class LogStash::Outputs::Application_insights
187
192
  if @file_pipe
188
193
  # remove "loading" record from state table of all previous blocks uploaded , we will try the whole file on an alternative storage
189
194
  @storage_recovery.recover_later( context_to_tuple, :state_table_update, @storage_account_name )
195
+ # memory is decrmeneted because the retry is done from the begining of the file
196
+ bytesize = @block_to_upload.bytesize
197
+ @block_to_upload.dispose
190
198
  @block_to_upload = nil
199
+ State.instance.dec_upload_bytesize( bytesize )
191
200
  return
192
201
  else
193
202
  info1 = "#{:commit} #{@storage_account_name}/#{@container_name}/#{@blob_name}, events: #{@uploaded_events_count}, size: #{@uploaded_bytesize}, blocks: #{@uploaded_block_numbers}, delay: #{Time.now.utc - @oldest_event_time}"
@@ -20,5 +20,5 @@
20
20
  # ----------------------------------------------------------------------------------
21
21
 
22
22
  # class LogStash::Outputs::Application_insights
23
- APPLICATION_INSIGHTS_VERSION ||= "0.2.1"
23
+ APPLICATION_INSIGHTS_VERSION ||= "0.2.2"
24
24
  # end
@@ -43,8 +43,13 @@ Gem::Specification.new do |s|
43
43
 
44
44
  # Gem dependencies
45
45
  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 6.0.0"
46
- s.add_runtime_dependency "azure-storage", "0.10.1.preview"
47
- s.add_runtime_dependency "azure-core", "0.1.2"
48
- s.add_runtime_dependency "application_insights", ">= 0.5.3"
46
+ # s.add_runtime_dependency "azure-storage", "0.10.1.preview"
47
+ # s.add_runtime_dependency "azure-storage", "0.11.3.preview"
48
+ s.add_runtime_dependency "azure-storage", "~> 0.11"
49
+ # s.add_runtime_dependency "azure-core", "0.1.2"
50
+ # s.add_runtime_dependency "azure-core", "0.1.5"
51
+ s.add_runtime_dependency "azure-core", "~> 0.1"
52
+ # s.add_runtime_dependency "application_insights", "0.5.3"
53
+ s.add_runtime_dependency "application_insights", "~> 0.5"
49
54
  s.add_development_dependency "logstash-devutils"
50
55
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-application_insights
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Microsoft Corporation
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-11-13 00:00:00.000000000 Z
11
+ date: 2016-11-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -33,45 +33,45 @@ dependencies:
33
33
  - !ruby/object:Gem::Dependency
34
34
  requirement: !ruby/object:Gem::Requirement
35
35
  requirements:
36
- - - '='
36
+ - - "~>"
37
37
  - !ruby/object:Gem::Version
38
- version: 0.10.1.preview
38
+ version: '0.11'
39
39
  name: azure-storage
40
40
  prerelease: false
41
41
  type: :runtime
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
- - - '='
44
+ - - "~>"
45
45
  - !ruby/object:Gem::Version
46
- version: 0.10.1.preview
46
+ version: '0.11'
47
47
  - !ruby/object:Gem::Dependency
48
48
  requirement: !ruby/object:Gem::Requirement
49
49
  requirements:
50
- - - '='
50
+ - - "~>"
51
51
  - !ruby/object:Gem::Version
52
- version: 0.1.2
52
+ version: '0.1'
53
53
  name: azure-core
54
54
  prerelease: false
55
55
  type: :runtime
56
56
  version_requirements: !ruby/object:Gem::Requirement
57
57
  requirements:
58
- - - '='
58
+ - - "~>"
59
59
  - !ruby/object:Gem::Version
60
- version: 0.1.2
60
+ version: '0.1'
61
61
  - !ruby/object:Gem::Dependency
62
62
  requirement: !ruby/object:Gem::Requirement
63
63
  requirements:
64
- - - ">="
64
+ - - "~>"
65
65
  - !ruby/object:Gem::Version
66
- version: 0.5.3
66
+ version: '0.5'
67
67
  name: application_insights
68
68
  prerelease: false
69
69
  type: :runtime
70
70
  version_requirements: !ruby/object:Gem::Requirement
71
71
  requirements:
72
- - - ">="
72
+ - - "~>"
73
73
  - !ruby/object:Gem::Version
74
- version: 0.5.3
74
+ version: '0.5'
75
75
  - !ruby/object:Gem::Dependency
76
76
  requirement: !ruby/object:Gem::Requirement
77
77
  requirements: