logstash-output-application_insights 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash/outputs/application_insights/block.rb +7 -0
- data/lib/logstash/outputs/application_insights/channel.rb +0 -2
- data/lib/logstash/outputs/application_insights/client.rb +9 -3
- data/lib/logstash/outputs/application_insights/exceptions.rb +3 -0
- data/lib/logstash/outputs/application_insights/local_file.rb +24 -23
- data/lib/logstash/outputs/application_insights/notification.rb +4 -1
- data/lib/logstash/outputs/application_insights/upload_pipe.rb +15 -6
- data/lib/logstash/outputs/application_insights/version.rb +1 -1
- data/logstash-output-application-insights.gemspec +8 -3
- metadata +14 -14
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 771238e72676111249e1ff68b5d04dd9c3fdad43
|
4
|
+
data.tar.gz: d5d25c0a1e678c27e9b034f1a63f5925cdd0ddfc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d66c4886bb6aa1b08f0840b4090cb5d8874ca12af83de60b3bad38dfe7fbcc668215f2bc57b84ecc8ba075e26bbdcd94fefd6fa807990378823683432fe5135c
|
7
|
+
data.tar.gz: bfd12398aaf0c5a82d0256b71b9207af816bac5715ea4812079c04ebce46a73b392f0befae7b5c51d0215fbe1a779c6cf625e460e33be131d8d3a2dbd101d324
|
@@ -40,6 +40,13 @@ class LogStash::Outputs::Application_insights
|
|
40
40
|
@@semaphore.synchronize { @@Block_number = ( @@Block_number + 1 ) % 1000000 }
|
41
41
|
end
|
42
42
|
|
43
|
+
def self.generate_block_numbers ( count )
|
44
|
+
@@semaphore.synchronize {
|
45
|
+
firstNumber = ( @@Block_number + 1 ) % 1000000
|
46
|
+
@@Block_number = ( @@Block_number + count ) % 1000000
|
47
|
+
firstNumber
|
48
|
+
}
|
49
|
+
end
|
43
50
|
|
44
51
|
|
45
52
|
def initialize ( event_separator = "" )
|
@@ -56,13 +56,17 @@ class LogStash::Outputs::Application_insights
|
|
56
56
|
def blobClient
|
57
57
|
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
58
58
|
@last_client_type = :blobClient
|
59
|
-
|
59
|
+
# breaking change after azure-storage 0.10.1
|
60
|
+
# @current_azure_storage_client.blobClient
|
61
|
+
@current_azure_storage_client.blob_client
|
60
62
|
end
|
61
63
|
|
62
64
|
def tableClient
|
63
65
|
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
64
66
|
@last_client_type = :blobClient
|
65
|
-
|
67
|
+
# breaking change after azure-storage 0.10.1
|
68
|
+
# @current_azure_storage_client.tableClient
|
69
|
+
@current_azure_storage_client.table_client
|
66
70
|
end
|
67
71
|
|
68
72
|
def notifyClient
|
@@ -109,7 +113,9 @@ class LogStash::Outputs::Application_insights
|
|
109
113
|
options[:ca_file] = configuration[:ca_file] unless configuration[:ca_file].empty?
|
110
114
|
|
111
115
|
@current_azure_storage_client = Azure::Storage::Client.new( options )
|
112
|
-
|
116
|
+
# breaking change after azure-storage 0.10.1
|
117
|
+
# @current_azure_storage_auth_sas = Azure::Storage::Auth::SharedAccessSignature.new( @storage_account_name, storage_access_key )
|
118
|
+
@current_azure_storage_auth_sas = Azure::Storage::Core::Auth::SharedAccessSignature.new( @storage_account_name, storage_access_key )
|
113
119
|
end
|
114
120
|
|
115
121
|
end
|
@@ -43,6 +43,9 @@ class LogStash::Outputs::Application_insights
|
|
43
43
|
class NotRecoverableError < StandardError
|
44
44
|
end
|
45
45
|
|
46
|
+
class AssertError < StandardError
|
47
|
+
end
|
48
|
+
|
46
49
|
# exception that cause process to end
|
47
50
|
# LogStash::ConfigurationError, "ssl_truststore_location must be set when SSL is enabled"
|
48
51
|
# class ConfigurationError < StandardError
|
@@ -32,52 +32,53 @@ class LogStash::Outputs::Application_insights
|
|
32
32
|
|
33
33
|
def initialize ( filename, is_gzip_file )
|
34
34
|
@file_name = filename
|
35
|
-
@writer = write_file = File.new( @file_name,
|
35
|
+
@writer = write_file = File.new( @file_name, "wb" )
|
36
36
|
@writer = Zlib::GzipWriter.new( write_file ) if is_gzip_file
|
37
37
|
@read_file = nil
|
38
38
|
@bytesize = 0
|
39
39
|
@events_count = 0
|
40
40
|
@first_block_number = nil
|
41
41
|
@next_block_number = nil
|
42
|
-
|
42
|
+
@next_event_count = nil
|
43
43
|
end
|
44
44
|
|
45
45
|
def seal
|
46
|
-
if @writer
|
47
|
-
|
48
|
-
@writer = nil
|
49
|
-
end
|
46
|
+
@writer.close if @writer
|
47
|
+
@writer = nil
|
50
48
|
end
|
51
49
|
|
52
|
-
|
53
50
|
def close_read
|
54
51
|
@read_file.close if @read_file
|
55
52
|
@read_file = nil
|
56
53
|
end
|
57
54
|
|
55
|
+
def open_read
|
56
|
+
@read_file = File.new( @file_name, "rb" ) # File.new( @file_name, File::RDWR )
|
57
|
+
@file_size = @read_file.size
|
58
|
+
@blocks_num = ( @file_size + BLOB_BLOCK_MAX_BYTESIZE - 1 ) / BLOB_BLOCK_MAX_BYTESIZE
|
59
|
+
@events_per_block = @events_count / @blocks_num
|
60
|
+
|
61
|
+
@next_event_count = @events_per_block + ( @events_count % @blocks_num )
|
62
|
+
@first_block_number ||= Block.generate_block_numbers( @blocks_num )
|
63
|
+
@next_block_number = @first_block_number
|
64
|
+
end
|
58
65
|
|
59
66
|
def get_next_block
|
60
67
|
block = Block.new
|
61
|
-
block.done_time = @done_time
|
62
|
-
block.oldest_event_time = @oldest_event_time
|
63
|
-
|
64
|
-
unless @read_file
|
65
|
-
@read_file = File.new( @file_name, File::RDWR )
|
66
|
-
@file_size = @read_file.size
|
67
|
-
@blocks_num = ( @file_size + BLOB_BLOCK_MAX_BYTESIZE - 1 ) / BLOB_BLOCK_MAX_BYTESIZE
|
68
|
-
@events_per_block = @events_count / @blocks_num
|
69
|
-
block.events_count = @events_per_block + ( @events_count % @blocks_num )
|
70
|
-
@next_block_number = @first_block_number ||= Block.generate_block_number
|
71
|
-
block.block_numbers = [ @first_block_number ]
|
72
|
-
else
|
73
|
-
block.block_numbers = [ @next_block_number ]
|
74
|
-
block.events_count = @events_per_block
|
75
|
-
end
|
76
|
-
@next_block_number += 1
|
77
68
|
block.bytes = @read_file.read(BLOB_BLOCK_MAX_BYTESIZE)
|
78
69
|
return nil if block.bytes.nil? || 0 == block.bytes.length
|
70
|
+
|
79
71
|
block.bytesize = block.bytes.length
|
80
72
|
State.instance.inc_upload_bytesize( block.bytesize )
|
73
|
+
|
74
|
+
block.done_time = @done_time
|
75
|
+
block.oldest_event_time = @oldest_event_time
|
76
|
+
block.block_numbers = [ @next_block_number ]
|
77
|
+
block.events_count = @next_event_count
|
78
|
+
|
79
|
+
@next_event_count = @events_per_block
|
80
|
+
@next_block_number += 1
|
81
|
+
|
81
82
|
block
|
82
83
|
end
|
83
84
|
|
@@ -60,7 +60,10 @@ class LogStash::Outputs::Application_insights
|
|
60
60
|
def set_blob_sas_url
|
61
61
|
blob_url ="https://#{@storage_account_name}.blob.#{@configuration[:azure_storage_host_suffix]}/#{@container_name}/#{@blob_name}"
|
62
62
|
options_and_constrains = {:permissions => "r", :resource => "b", :expiry => ( Time.now.utc + @configuration[:blob_access_expiry_time] ).iso8601 }
|
63
|
-
|
63
|
+
# breaking change after azure-storage 0.10.1
|
64
|
+
# @blob_sas_url = @client.storage_auth_sas.signed_uri( URI( blob_url ), options_and_constrains )
|
65
|
+
use_account_sas = false
|
66
|
+
@blob_sas_url = @client.storage_auth_sas.signed_uri( URI( blob_url ), use_account_sas, options_and_constrains )
|
64
67
|
end
|
65
68
|
|
66
69
|
|
@@ -107,20 +107,25 @@ class LogStash::Outputs::Application_insights
|
|
107
107
|
|
108
108
|
break if :close == file_to_upload
|
109
109
|
|
110
|
-
|
110
|
+
file_to_upload.open_read
|
111
|
+
@file_size = file_to_upload.file_size
|
112
|
+
|
111
113
|
while block = file_to_upload.get_next_block
|
112
|
-
@file_size ||= file_to_upload.file_size
|
113
114
|
unless upload( block )
|
115
|
+
# start the file from the begining
|
116
|
+
file_to_upload.close_read
|
114
117
|
@channel.recover_later_file_upload( file_to_upload )
|
115
118
|
file_to_upload = nil
|
116
|
-
@uploaded_block_ids = [ ]
|
117
119
|
break
|
118
120
|
end
|
119
121
|
end
|
120
|
-
file_to_upload.dispose if file_to_upload
|
121
|
-
file_to_upload = nil
|
122
122
|
|
123
|
-
|
123
|
+
if file_to_upload
|
124
|
+
commit unless @uploaded_block_ids.empty?
|
125
|
+
file_to_upload.dispose
|
126
|
+
file_to_upload = nil
|
127
|
+
end
|
128
|
+
|
124
129
|
@uploaded_block_ids = [ ]
|
125
130
|
end
|
126
131
|
end
|
@@ -187,7 +192,11 @@ class LogStash::Outputs::Application_insights
|
|
187
192
|
if @file_pipe
|
188
193
|
# remove "loading" record from state table of all previous blocks uploaded , we will try the whole file on an alternative storage
|
189
194
|
@storage_recovery.recover_later( context_to_tuple, :state_table_update, @storage_account_name )
|
195
|
+
# memory is decrmeneted because the retry is done from the begining of the file
|
196
|
+
bytesize = @block_to_upload.bytesize
|
197
|
+
@block_to_upload.dispose
|
190
198
|
@block_to_upload = nil
|
199
|
+
State.instance.dec_upload_bytesize( bytesize )
|
191
200
|
return
|
192
201
|
else
|
193
202
|
info1 = "#{:commit} #{@storage_account_name}/#{@container_name}/#{@blob_name}, events: #{@uploaded_events_count}, size: #{@uploaded_bytesize}, blocks: #{@uploaded_block_numbers}, delay: #{Time.now.utc - @oldest_event_time}"
|
@@ -43,8 +43,13 @@ Gem::Specification.new do |s|
|
|
43
43
|
|
44
44
|
# Gem dependencies
|
45
45
|
s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 6.0.0"
|
46
|
-
s.add_runtime_dependency "azure-storage", "0.10.1.preview"
|
47
|
-
s.add_runtime_dependency "azure-
|
48
|
-
s.add_runtime_dependency "
|
46
|
+
# s.add_runtime_dependency "azure-storage", "0.10.1.preview"
|
47
|
+
# s.add_runtime_dependency "azure-storage", "0.11.3.preview"
|
48
|
+
s.add_runtime_dependency "azure-storage", "~> 0.11"
|
49
|
+
# s.add_runtime_dependency "azure-core", "0.1.2"
|
50
|
+
# s.add_runtime_dependency "azure-core", "0.1.5"
|
51
|
+
s.add_runtime_dependency "azure-core", "~> 0.1"
|
52
|
+
# s.add_runtime_dependency "application_insights", "0.5.3"
|
53
|
+
s.add_runtime_dependency "application_insights", "~> 0.5"
|
49
54
|
s.add_development_dependency "logstash-devutils"
|
50
55
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-application_insights
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Microsoft Corporation
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-11-
|
11
|
+
date: 2016-11-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -33,45 +33,45 @@ dependencies:
|
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
34
|
requirement: !ruby/object:Gem::Requirement
|
35
35
|
requirements:
|
36
|
-
- -
|
36
|
+
- - "~>"
|
37
37
|
- !ruby/object:Gem::Version
|
38
|
-
version: 0.
|
38
|
+
version: '0.11'
|
39
39
|
name: azure-storage
|
40
40
|
prerelease: false
|
41
41
|
type: :runtime
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
|
-
- -
|
44
|
+
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: 0.
|
46
|
+
version: '0.11'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
49
49
|
requirements:
|
50
|
-
- -
|
50
|
+
- - "~>"
|
51
51
|
- !ruby/object:Gem::Version
|
52
|
-
version: 0.1
|
52
|
+
version: '0.1'
|
53
53
|
name: azure-core
|
54
54
|
prerelease: false
|
55
55
|
type: :runtime
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
|
-
- -
|
58
|
+
- - "~>"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: 0.1
|
60
|
+
version: '0.1'
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
requirement: !ruby/object:Gem::Requirement
|
63
63
|
requirements:
|
64
|
-
- - "
|
64
|
+
- - "~>"
|
65
65
|
- !ruby/object:Gem::Version
|
66
|
-
version: 0.5
|
66
|
+
version: '0.5'
|
67
67
|
name: application_insights
|
68
68
|
prerelease: false
|
69
69
|
type: :runtime
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
71
71
|
requirements:
|
72
|
-
- - "
|
72
|
+
- - "~>"
|
73
73
|
- !ruby/object:Gem::Version
|
74
|
-
version: 0.5
|
74
|
+
version: '0.5'
|
75
75
|
- !ruby/object:Gem::Dependency
|
76
76
|
requirement: !ruby/object:Gem::Requirement
|
77
77
|
requirements:
|