logstash-output-application_insights 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/CONTRIBUTORS +9 -0
- data/DEVELOPER.md +0 -0
- data/Gemfile +26 -0
- data/LICENSE +17 -0
- data/README.md +495 -0
- data/Rakefile +22 -0
- data/lib/logstash/outputs/application_insights.rb +393 -0
- data/lib/logstash/outputs/application_insights/blob.rb +923 -0
- data/lib/logstash/outputs/application_insights/block.rb +118 -0
- data/lib/logstash/outputs/application_insights/channel.rb +259 -0
- data/lib/logstash/outputs/application_insights/channels.rb +142 -0
- data/lib/logstash/outputs/application_insights/client.rb +110 -0
- data/lib/logstash/outputs/application_insights/clients.rb +113 -0
- data/lib/logstash/outputs/application_insights/config.rb +341 -0
- data/lib/logstash/outputs/application_insights/constants.rb +208 -0
- data/lib/logstash/outputs/application_insights/exceptions.rb +55 -0
- data/lib/logstash/outputs/application_insights/flow_control.rb +80 -0
- data/lib/logstash/outputs/application_insights/multi_io_logger.rb +69 -0
- data/lib/logstash/outputs/application_insights/shutdown.rb +96 -0
- data/lib/logstash/outputs/application_insights/state.rb +89 -0
- data/lib/logstash/outputs/application_insights/storage_cleanup.rb +214 -0
- data/lib/logstash/outputs/application_insights/sub_channel.rb +75 -0
- data/lib/logstash/outputs/application_insights/telemetry.rb +99 -0
- data/lib/logstash/outputs/application_insights/timer.rb +90 -0
- data/lib/logstash/outputs/application_insights/utils.rb +139 -0
- data/lib/logstash/outputs/application_insights/version.rb +24 -0
- data/logstash-output-application-insights.gemspec +50 -0
- data/spec/outputs/application_insights_spec.rb +42 -0
- metadata +151 -0
@@ -0,0 +1,110 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
# ----------------------------------------------------------------------------------
|
4
|
+
# Logstash Output Application Insights
|
5
|
+
#
|
6
|
+
# Copyright (c) Microsoft Corporation
|
7
|
+
#
|
8
|
+
# All rights reserved.
|
9
|
+
#
|
10
|
+
# Licensed under the Apache License, Version 2.0 (the License);
|
11
|
+
# you may not use this file except in compliance with the License.
|
12
|
+
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
#
|
18
|
+
# See the Apache Version 2.0 License for specific language governing
|
19
|
+
# permissions and limitations under the License.
|
20
|
+
# ----------------------------------------------------------------------------------
|
21
|
+
|
22
|
+
class LogStash::Outputs::Application_insights
|
23
|
+
class Client
|
24
|
+
|
25
|
+
public
|
26
|
+
|
27
|
+
def initialize ( storage_account_name, force = nil )
|
28
|
+
@storage_account_name = storage_account_name
|
29
|
+
@storage_account = Clients.instance.active_storage_account( storage_account_name, force )
|
30
|
+
|
31
|
+
@tuple = @storage_account[:clients_Q].pop
|
32
|
+
( @current_storage_account_key_index, @current_azure_storage_auth_sas, @current_azure_storage_client) = @tuple
|
33
|
+
if @current_storage_account_key_index != @storage_account[:valid_index]
|
34
|
+
@current_storage_account_key_index = @storage_account[:valid_index]
|
35
|
+
set_current_storage_account_client
|
36
|
+
@tuple = [ @current_storage_account_key_index, @current_azure_storage_auth_sas, @current_azure_storage_client ]
|
37
|
+
end
|
38
|
+
@storage_account_key_index = @current_storage_account_key_index
|
39
|
+
end
|
40
|
+
|
41
|
+
def dispose ( io_failed_reason = nil )
|
42
|
+
if @tuple
|
43
|
+
if @current_storage_account_key_index == @storage_account_key_index
|
44
|
+
@storage_account[:clients_Q] << @tuple
|
45
|
+
else
|
46
|
+
@storage_account[:valid_index] = @current_storage_account_key_index
|
47
|
+
@storage_account[:clients_Q] << [ @current_storage_account_key_index, @current_azure_storage_auth_sas, @current_azure_storage_client ]
|
48
|
+
end
|
49
|
+
@tuple = nil
|
50
|
+
|
51
|
+
Clients.instance.failed_storage_account( @storage_account_name, io_failed_reason ) if io_failed_reason && :blobClient == @last_client_type
|
52
|
+
end
|
53
|
+
nil
|
54
|
+
end
|
55
|
+
|
56
|
+
def blobClient
|
57
|
+
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
58
|
+
@last_client_type = :blobClient
|
59
|
+
@current_azure_storage_client.blobClient
|
60
|
+
end
|
61
|
+
|
62
|
+
def tableClient
|
63
|
+
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
64
|
+
@last_client_type = :blobClient
|
65
|
+
@current_azure_storage_client.tableClient
|
66
|
+
end
|
67
|
+
|
68
|
+
def notifyClient
|
69
|
+
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
70
|
+
@last_client_type = :notifyClient
|
71
|
+
@current_azure_storage_client
|
72
|
+
end
|
73
|
+
|
74
|
+
def storage_auth_sas
|
75
|
+
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
76
|
+
@current_azure_storage_auth_sas
|
77
|
+
end
|
78
|
+
|
79
|
+
|
80
|
+
def switch_storage_account_key!
|
81
|
+
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
82
|
+
@current_storage_account_key_index = ( @current_storage_account_key_index + 1 ) % @storage_account[:keys].length
|
83
|
+
if @current_storage_account_key_index == @storage_account_key_index
|
84
|
+
rollback_storage_account_key
|
85
|
+
false
|
86
|
+
else
|
87
|
+
set_current_storage_account_client
|
88
|
+
true
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
private
|
93
|
+
|
94
|
+
def rollback_storage_account_key
|
95
|
+
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
96
|
+
( @current_storage_account_key_index, @current_azure_storage_auth_sas, @current_azure_storage_client) = @tuple
|
97
|
+
end
|
98
|
+
|
99
|
+
def set_current_storage_account_client
|
100
|
+
configuration = Config.current
|
101
|
+
alt_storage_access_key = @storage_account[:keys][@current_storage_account_key_index]
|
102
|
+
options = { :storage_account_name => @storage_account_name, :storage_access_key => alt_storage_access_key }
|
103
|
+
options[:ca_file] = configuration[:ca_file] unless configuration[:ca_file].empty?
|
104
|
+
@current_azure_storage_client = Azure::Storage::Client.new( options )
|
105
|
+
|
106
|
+
@current_azure_storage_auth_sas = Azure::Storage::Auth::SharedAccessSignature.new( @storage_account_name, alt_storage_access_key )
|
107
|
+
end
|
108
|
+
|
109
|
+
end
|
110
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
# ----------------------------------------------------------------------------------
|
4
|
+
# Logstash Output Application Insights
|
5
|
+
#
|
6
|
+
# Copyright (c) Microsoft Corporation
|
7
|
+
#
|
8
|
+
# All rights reserved.
|
9
|
+
#
|
10
|
+
# Licensed under the Apache License, Version 2.0 (the License);
|
11
|
+
# you may not use this file except in compliance with the License.
|
12
|
+
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
#
|
18
|
+
# See the Apache Version 2.0 License for specific language governing
|
19
|
+
# permissions and limitations under the License.
|
20
|
+
# ----------------------------------------------------------------------------------
|
21
|
+
|
22
|
+
class LogStash::Outputs::Application_insights
|
23
|
+
class Clients
|
24
|
+
|
25
|
+
public
|
26
|
+
|
27
|
+
def initialize
|
28
|
+
configuration = Config.current
|
29
|
+
@resurrect_delay = configuration[:resurrect_delay]
|
30
|
+
@storage_accounts = { }
|
31
|
+
@storage_account_names = [ ]
|
32
|
+
@current_storage_account_names = [ ]
|
33
|
+
@last_storage_account_key_index = 0
|
34
|
+
@state_semaphore = Mutex.new
|
35
|
+
create_blob_clients( configuration )
|
36
|
+
end
|
37
|
+
|
38
|
+
|
39
|
+
def active_storage_account ( storage_account_name , force )
|
40
|
+
raise StorageAccountsOffError if 0 == @current_storage_account_names.length && force.nil?
|
41
|
+
@storage_accounts[storage_account_name]
|
42
|
+
end
|
43
|
+
|
44
|
+
def create_blob_clients ( configuration )
|
45
|
+
configuration[:storage_account_name_key].each do |storage_account_name, storage_account_keys|
|
46
|
+
|
47
|
+
clients_Q = Queue.new
|
48
|
+
@storage_accounts[storage_account_name] = { :keys => storage_account_keys, :clients_Q => clients_Q, :valid_index => 0, :state => :on, :off_reason => [] }
|
49
|
+
40.times do
|
50
|
+
# lazy creation, clients will be created when first needed
|
51
|
+
clients_Q << [ nil, nil, nil]
|
52
|
+
end
|
53
|
+
@storage_account_names << storage_account_name
|
54
|
+
end
|
55
|
+
|
56
|
+
@current_storage_account_names = @storage_account_names.clone
|
57
|
+
end
|
58
|
+
|
59
|
+
|
60
|
+
def storage_account_state_on? ( storage_account_name = nil )
|
61
|
+
if storage_account_name
|
62
|
+
@storage_accounts[storage_account_name][:off_reason].empty?
|
63
|
+
else
|
64
|
+
@current_storage_account_names.length > 0
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def failed_storage_account( storage_account_name, io_failed_reason )
|
69
|
+
storage_account = @storage_accounts[storage_account_name]
|
70
|
+
@state_semaphore.synchronize {
|
71
|
+
unless storage_account[:off_reason].include?( io_failed_reason )
|
72
|
+
storage_account[:off_reason] << io_failed_reason
|
73
|
+
if storage_account[:off_reason].length == 1
|
74
|
+
current_storage_account_names = [ ]
|
75
|
+
@current_storage_account_names.each do |account_name|
|
76
|
+
current_storage_account_names << account_name unless account_name == storage_account_name
|
77
|
+
end
|
78
|
+
@current_storage_account_names = current_storage_account_names
|
79
|
+
|
80
|
+
Thread.new( storage_account_name ) do |account_name|
|
81
|
+
loop do
|
82
|
+
sleep( @resurrect_delay )
|
83
|
+
if Blob.new.test_storage( account_name )
|
84
|
+
@state_semaphore.synchronize {
|
85
|
+
storage_account = @storage_accounts[account_name]
|
86
|
+
storage_account[:off_reason] = [ ]
|
87
|
+
@current_storage_account_names << account_name
|
88
|
+
}
|
89
|
+
break
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
95
|
+
end
|
96
|
+
}
|
97
|
+
end
|
98
|
+
|
99
|
+
def get_random_active_storage( exclude )
|
100
|
+
@state_semaphore.synchronize { storage_account_name = ( @current_storage_account_names - exclude ).sample if @current_storage_account_names.length > 0 }
|
101
|
+
end
|
102
|
+
|
103
|
+
public
|
104
|
+
|
105
|
+
@@instance = Clients.new
|
106
|
+
|
107
|
+
def self.instance
|
108
|
+
@@instance
|
109
|
+
end
|
110
|
+
|
111
|
+
private_class_method :new
|
112
|
+
end
|
113
|
+
end
|
@@ -0,0 +1,341 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
# ----------------------------------------------------------------------------------
|
4
|
+
# Logstash Output Application Insights
|
5
|
+
#
|
6
|
+
# Copyright (c) Microsoft Corporation
|
7
|
+
#
|
8
|
+
# All rights reserved.
|
9
|
+
#
|
10
|
+
# Licensed under the Apache License, Version 2.0 (the License);
|
11
|
+
# you may not use this file except in compliance with the License.
|
12
|
+
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
#
|
18
|
+
# See the Apache Version 2.0 License for specific language governing
|
19
|
+
# permissions and limitations under the License.
|
20
|
+
# ----------------------------------------------------------------------------------
|
21
|
+
|
22
|
+
class LogStash::Outputs::Application_insights
|
23
|
+
class Config
|
24
|
+
|
25
|
+
public
|
26
|
+
|
27
|
+
@@configuration = {}
|
28
|
+
|
29
|
+
def self.current
|
30
|
+
@@configuration
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.validate_and_adjust_configuration ( configuration )
|
34
|
+
configuration.each_pair { |config_name, config_value|
|
35
|
+
raise ConfigurationError, "#{config_name.to_s} must be defined" unless config_value || BOOLEAN_PROPERTIES.include?( config_name )
|
36
|
+
case config_name
|
37
|
+
|
38
|
+
when :logger_level
|
39
|
+
logger_level = validate_and_adjust( config_name, config_value, String )
|
40
|
+
raise ConfigurationError, "#{config_name.to_s} can be set to only one of the valid log levels" unless LOGGER_LEVEL_MAP[logger_level.upcase.to_sym]
|
41
|
+
configuration[config_name] = LOGGER_LEVEL_MAP[logger_level.upcase.to_sym]
|
42
|
+
|
43
|
+
when :logger_files
|
44
|
+
config_value = [ config_value ] if config_value.is_a?( String )
|
45
|
+
logger_files = validate_and_adjust( config_name, config_value, Array )
|
46
|
+
|
47
|
+
i = 0
|
48
|
+
logger_files.map! do |file_name|
|
49
|
+
file_name = validate_and_adjust( "#{config_name.to_s}[#{i}]", file_name, String )
|
50
|
+
i += 1
|
51
|
+
if "stdout" == file_name.downcase
|
52
|
+
file_name = :stdout
|
53
|
+
file = STDOUT
|
54
|
+
elsif "stderr" == file_name.downcase
|
55
|
+
file_name = :stderr
|
56
|
+
file = STDERR
|
57
|
+
else
|
58
|
+
file = ::File.open( file_name, "a+" )
|
59
|
+
end
|
60
|
+
[ file_name, file ]
|
61
|
+
end
|
62
|
+
configuration[config_name] = logger_files
|
63
|
+
|
64
|
+
when :logger_progname
|
65
|
+
configuration[config_name] = validate_and_adjust( config_name, config_value, String )
|
66
|
+
|
67
|
+
when :logger_shift_size
|
68
|
+
configuration[config_name] = validate_and_adjust_integer( config_name, config_value, MIN_LOGGER_SHIFT_SIZE, MAX_LOGGER_SHIFT_SIZE )
|
69
|
+
|
70
|
+
when :logger_shift_age
|
71
|
+
if config_value.is_a?( String )
|
72
|
+
config_value = validate_and_adjust( config_name, config_value, String ).downcase
|
73
|
+
raise ConfigurationError, "#{config_name.to_s} if string, can be set to only one of the following values: #{VALID_LOGGER_SHIFT_AGES}" unless VALID_LOGGER_SHIFT_AGES. include?( config_value )
|
74
|
+
elsif config_value.is_a?( Integer )
|
75
|
+
config_value = validate_and_adjust_integer( config_name, config_value, MIN_LOGGER_SHIFT_AGE, MAX_LOGGER_SHIFT_AGE )
|
76
|
+
else
|
77
|
+
raise ConfigurationError, "#{config_name.to_s} must be either a string or integer"
|
78
|
+
end
|
79
|
+
configuration[config_name] = config_value
|
80
|
+
|
81
|
+
when :azure_storage_container_prefix
|
82
|
+
azure_storage_container_prefix = validate_and_adjust( config_name, config_value, String )
|
83
|
+
unless azure_storage_container_prefix.empty?
|
84
|
+
len = 63 - "-#{AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX}-yyyy-mm-dd".length
|
85
|
+
validate_max( "azure_storage_container_prefix length", azure_storage_container_prefix.length, len )
|
86
|
+
azure_storage_container_prefix += "-"
|
87
|
+
end
|
88
|
+
azure_storage_container_prefix = azure_storage_container_prefix.downcase
|
89
|
+
container_name = "#{azure_storage_container_prefix}#{AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX}-yyyy-mm-dd"
|
90
|
+
raise ConfigurationError, "#{config_name.to_s} must have only alphanumeric and dash characters, cannot start or end with a dash, and a dash cannot follow a dash" unless Utils.valid_container_name?( container_name )
|
91
|
+
configuration[config_name] = azure_storage_container_prefix + AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX
|
92
|
+
|
93
|
+
when :azure_storage_azure_storage_table_prefix
|
94
|
+
azure_storage_table_prefix = validate_and_adjust( config_name, config_value, String )
|
95
|
+
unless azure_storage_table_prefix.empty?
|
96
|
+
len = 63 - "-#{AZURE_STORAGE_TABLE_LOGSTASH_PREFIX}yyyymmdd".length
|
97
|
+
validate_max( "azure_storage_table_prefix length", azure_storage_table_prefix.length, len )
|
98
|
+
end
|
99
|
+
table_name = "#{azure_storage_table_prefix}#{AZURE_STORAGE_TABLE_LOGSTASH_PREFIX}yyyymmdd"
|
100
|
+
raise ConfigurationError, "#{config_name} must have only alphanumeric" unless Utils.valid_table_name?( table_name )
|
101
|
+
configuration[config_name] = azure_storage_table_prefix + AZURE_STORAGE_TABLE_LOGSTASH_PREFIX
|
102
|
+
|
103
|
+
when :ca_file
|
104
|
+
config_value = validate_and_adjust( config_name, config_value, String )
|
105
|
+
unless config_value.empty?
|
106
|
+
raise ConfigurationError, "#{config_name} must have a valid path name" unless Utils.valid_file_path?( config_value )
|
107
|
+
end
|
108
|
+
configuration[config_name] = validate_and_adjust( config_name, config_value, String )
|
109
|
+
|
110
|
+
when :azure_storage_blob_prefix
|
111
|
+
azure_storage_blob_prefix = validate_and_adjust( config_name, config_value, String )
|
112
|
+
unless azure_storage_blob_prefix.empty?
|
113
|
+
len = 1024 - "-#{AZURE_STORAGE_BLOB_LOGSTASH_PREFIX}_ikey-#{INSTRUMENTATION_KEY_TEMPLATE}_table-#{TABLE_ID_TEMPLATE}_yyyy-mm-dd-HH-MM-SS-LLL".length
|
114
|
+
validate_max( "azure_storage_blob_prefix length", azure_storage_blob_prefix.length, len )
|
115
|
+
azure_storage_blob_prefix += "-"
|
116
|
+
end
|
117
|
+
azure_storage_blob_prefix += AZURE_STORAGE_BLOB_LOGSTASH_PREFIX
|
118
|
+
|
119
|
+
raise ConfigurationError, "#{config_name.to_s} doesn't meet url format" unless Utils.url?( "http://storage/container/#{azure_storage_blob_prefix}_ikey-#{INSTRUMENTATION_KEY_TEMPLATE}_table-#{TABLE_ID_TEMPLATE}.json" )
|
120
|
+
configuration[config_name] = azure_storage_blob_prefix
|
121
|
+
|
122
|
+
when :table_id
|
123
|
+
configuration[config_name] = validate_and_adjust_guid( config_name, config_value )
|
124
|
+
|
125
|
+
when :blob_max_bytesize
|
126
|
+
configuration[config_name] = validate_and_adjust_integer( config_name, config_value, MIN_BLOB_MAX_BYTESIZE, MAX_BLOB_MAX_BYTESIZE )
|
127
|
+
|
128
|
+
when :blob_max_events
|
129
|
+
configuration[config_name] = validate_and_adjust_integer( config_name, config_value, MIN_BLOB_MAX_EVENTS, MAX_BLOB_MAX_EVENTS )
|
130
|
+
|
131
|
+
when :blob_retention_time
|
132
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_BLOB_RETENTION_TIME, MAX_BLOB_RETENTION_TIME )
|
133
|
+
|
134
|
+
when :blob_access_expiry_time
|
135
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_BLOB_ACCESS_EXPIRY_TIME, MAX_BLOB_ACCESS_EXPIRY_TIME )
|
136
|
+
|
137
|
+
when :resurrect_delay
|
138
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_STORAGE_RESURRECT_DELAY, MAX_STORAGE_RESURRECT_DELAY )
|
139
|
+
|
140
|
+
when :flow_control_suspend_bytes
|
141
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_FLOW_CONTROL_SUSPEND_BYTES, MAX_FLOW_CONTROL_SUSPEND_BYTES )
|
142
|
+
|
143
|
+
when :flow_control_resume_bytes
|
144
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_FLOW_CONTROL_RESUME_BYTES, MAX_FLOW_CONTROL_RESUME_BYTES )
|
145
|
+
|
146
|
+
when :flow_control_delay
|
147
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_FLOW_CONTROL_DELAY, MAX_FLOW_CONTROL_DELAY )
|
148
|
+
|
149
|
+
when :io_retry_delay
|
150
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_IO_RETRY_DELAY, MAX_IO_RETRY_DELAY )
|
151
|
+
|
152
|
+
when :io_max_retries
|
153
|
+
configuration[config_name] = validate_and_adjust_integer( config_name, config_value, MIN_IO_MAX_RETRIES, MAX_IO_MAX_RETRIES )
|
154
|
+
|
155
|
+
when :storage_account_name_key
|
156
|
+
config_value = validate_and_adjust( config_name, config_value, Array )
|
157
|
+
if config_value.empty?
|
158
|
+
raise ConfigurationError, "#{config_name.to_s} is empty, at least one storage account name should be defined" unless ENV['AZURE_STORAGE_ACCOUNT']
|
159
|
+
raise ConfigurationError, "#{config_name.to_s} is empty, at least one storage account access key should be defined" unless ENV['AZURE_STORAGE_ACCESS_KEY']
|
160
|
+
config_value = [ ENV['AZURE_STORAGE_ACCOUNT'], ENV['AZURE_STORAGE_ACCESS_KEY'] ]
|
161
|
+
end
|
162
|
+
|
163
|
+
storage_account_name_key = validate_and_adjust( config_name, config_value, Array, :disallow_empty )
|
164
|
+
unless storage_account_name_key[0].is_a?( Array )
|
165
|
+
raise ConfigurationError, "#{config_name.to_s} property is empty, should contain at least one pair: account_name, key" unless 2 == storage_account_name_key.length
|
166
|
+
storage_account_name_key = [ [ storage_account_name_key[0], storage_account_name_key[1] ]]
|
167
|
+
end
|
168
|
+
|
169
|
+
index = 0
|
170
|
+
storage_account_name_key.map! { |pair|
|
171
|
+
pair = validate_and_adjust( "#{config_name.to_s}[#{index}]", pair, Array, :disallow_empty )
|
172
|
+
raise ConfigurationError, "#{config_name.to_s}[#{index}] must have two items" unless 2 == pair.length
|
173
|
+
|
174
|
+
( name, keys ) = pair
|
175
|
+
name = validate_and_adjust( "#{config_name.to_s}[#{index}]:name", name, String, :disallow_empty )
|
176
|
+
raise ConfigurationError, "#{config_name.to_s}[#{index}]:name must between 3 to 24 characters" unless (name.length >= 3) && (name.length <= 24)
|
177
|
+
raise ConfigurationError, "##{config_name.to_s}[#{index}]:name bad format, must have only alphanumeric characters" unless Utils.alphanumeric?( name )
|
178
|
+
|
179
|
+
keys = [ keys ] if keys.is_a?( String )
|
180
|
+
keys = validate_and_adjust( "#{config_name}[#{index}]:keys", keys, Array, :disallow_empty )
|
181
|
+
keys.each_index do |i|
|
182
|
+
key = validate_and_adjust( "#{config_name}[#{index}:keys[#{i}]", keys[i], String, :disallow_empty )
|
183
|
+
raise ConfigurationError, "#{config_name}[#{index}:keys[#{i}] must have only valid base64 characters" unless Utils.base64?( key )
|
184
|
+
end
|
185
|
+
index += 1
|
186
|
+
[ name.downcase, keys ]
|
187
|
+
}
|
188
|
+
configuration[config_name] = storage_account_name_key
|
189
|
+
|
190
|
+
when :tables
|
191
|
+
tables = validate_and_adjust( config_name, config_value, Hash )
|
192
|
+
tables.each_pair { |table_id, properties|
|
193
|
+
table_id = validate_and_adjust_guid( "#{config_name}:table_id", table_id )
|
194
|
+
info = "#{config_name}[#{table_id}]"
|
195
|
+
properties = validate_and_adjust( info, properties, Hash )
|
196
|
+
properties = Utils.symbolize_hash_keys( properties )
|
197
|
+
validate_and_adjust_table_properties!( properties, configuration, info )
|
198
|
+
configuration[config_name][table_id] = properties
|
199
|
+
}
|
200
|
+
end
|
201
|
+
}
|
202
|
+
validate_and_adjust_table_properties!( configuration, configuration )
|
203
|
+
@@configuration = configuration
|
204
|
+
end
|
205
|
+
|
206
|
+
|
207
|
+
def self.symbolize_table_properties ( properties )
|
208
|
+
new_properties = {}
|
209
|
+
properties.each_pair { |property_name, property_value|
|
210
|
+
new_properties[property_name.to_sym] = property_value
|
211
|
+
}
|
212
|
+
new_properties
|
213
|
+
end
|
214
|
+
|
215
|
+
def self.validate_and_adjust_table_properties! ( properties, configuration, base_info = nil )
|
216
|
+
|
217
|
+
properties.each_pair { |property_name, property_value|
|
218
|
+
info = ( base_info.nil? ? "#{property_name}" : "#{base_info}[#{property_name}]" )
|
219
|
+
raise ConfigurationError, "#{info} must be defined" unless property_value || BOOLEAN_PROPERTIES.include?( property_name )
|
220
|
+
|
221
|
+
case property_name.downcase
|
222
|
+
|
223
|
+
when :intrumentation_key
|
224
|
+
properties[:intrumentation_key] = validate_and_adjust_guid( info, property_value )
|
225
|
+
when :blob_serialization
|
226
|
+
property_value = property_value.downcase
|
227
|
+
raise ConfigurationError, "#{info}, can be set to only one of the following values: #{VALID_EXT_EVENT_FORMAT}" unless VALID_EXT_EVENT_FORMAT.include?( property_value )
|
228
|
+
properties[:blob_serialization] = validate_and_adjust_ext( info ,property_value, configuration[:azure_storage_blob_prefix] ) # be careful depends on order
|
229
|
+
when :csv_default_value
|
230
|
+
properties[:csv_default_value] = validate_and_adjust( info, property_value, String )
|
231
|
+
when :csv_separator
|
232
|
+
properties[:csv_separator] = validate_and_adjust( info, property_value, String )
|
233
|
+
when :blob_max_delay
|
234
|
+
properties[:blob_max_delay] = validate_and_adjust_number( info, property_value, MIN_BLOB_MAX_DELAY, MAX_BLOB_MAX_DELAY )
|
235
|
+
when :event_separator
|
236
|
+
properties[:event_separator] = validate_and_adjust( info, property_value, String )
|
237
|
+
when :serialized_event_field
|
238
|
+
properties[:serialized_event_field] = ( property_value.nil? ? nil : validate_and_adjust( info, property_value, String ) )
|
239
|
+
when :case_insensitive_columns
|
240
|
+
properties[:case_insensitive_columns] = validate_and_adjust_boolean( info, property_value )
|
241
|
+
when :table_columns
|
242
|
+
if property_value.nil?
|
243
|
+
properties[:table_columns] = property_value
|
244
|
+
else
|
245
|
+
table_columns = property_value
|
246
|
+
table_columns = [ table_columns ] if table_columns.is_a?( String )
|
247
|
+
table_columns = validate_and_adjust( info, table_columns, Array)
|
248
|
+
new_table_columns = []
|
249
|
+
index = 0
|
250
|
+
table_columns.each do |column|
|
251
|
+
new_column = {}
|
252
|
+
|
253
|
+
column = { COLUMN_PROPERTY_NAME => column } if column.is_a?( String )
|
254
|
+
column = validate_and_adjust( "#{info}[#{index}]", column, Hash, :disallow_empty )
|
255
|
+
raise ConfigurationError, "#{info}[#{index}][#{COLUMN_PROPERTY_NAME}] must be defined" unless column[COLUMN_PROPERTY_NAME]
|
256
|
+
new_column[:name] = validate_and_adjust( "#{info}[#{index}][#{COLUMN_PROPERTY_NAME}]", column[COLUMN_PROPERTY_NAME], String )
|
257
|
+
|
258
|
+
if column[COLUMN_PROPERTY_DEFAULT]
|
259
|
+
new_column[:default] = validate_and_adjust( "#{info}[#{index}][#{COLUMN_PROPERTY_DEFAULT}]", column[COLUMN_PROPERTY_DEFAULT], String )
|
260
|
+
end
|
261
|
+
|
262
|
+
if column[COLUMN_PROPERTY_TYPE]
|
263
|
+
new_column[:type] = validate_and_adjust( "#{info}[#{index}][#{COLUMN_PROPERTY_TYPE}]", column[COLUMN_PROPERTY_TYPE], String ).downcase
|
264
|
+
raise ConfigurationError, "#{info}[#{index}][#{COLUMN_PROPERTY_TYPE}] can be only one of the following values: #{VALID_FIELDS_MAP_TYPES}" unless VALID_FIELDS_MAP_TYPES.any? {|type| type == new_column[:type]}
|
265
|
+
new_column[:type] = new_column[:type].to_sym
|
266
|
+
end
|
267
|
+
new_table_columns << new_column
|
268
|
+
index += 1
|
269
|
+
end
|
270
|
+
properties[:table_columns] = new_table_columns
|
271
|
+
end
|
272
|
+
|
273
|
+
else
|
274
|
+
end
|
275
|
+
}
|
276
|
+
end
|
277
|
+
|
278
|
+
|
279
|
+
private
|
280
|
+
|
281
|
+
def self.validate_and_adjust_ext ( property, ext, prefix )
|
282
|
+
ext = validate_and_adjust( property, ext, String )
|
283
|
+
raise ConfigurationError, "#{property.to_s} must be a valid extension string, have only alphanumeric, dash and underline characters" unless Utils.ext?( ext )
|
284
|
+
len = 1024 - "#{prefix}-#{AZURE_STORAGE_BLOB_LOGSTASH_PREFIX}_ikey-#{INSTRUMENTATION_KEY_TEMPLATE}_table-#{TABLE_ID_TEMPLATE}_yyyy-mm-dd-HH-MM-SS-LLL".length
|
285
|
+
raise ConfigurationError, "#{property.to_s} length cannot be more than #{len} characters" unless ext.length <= len
|
286
|
+
ext
|
287
|
+
end
|
288
|
+
|
289
|
+
def self.validate_and_adjust ( property, value, type, disallow_empty = false )
|
290
|
+
raise ConfigurationError, "#{property.to_s} must be an #{type.to_s}" unless value.is_a?( type )
|
291
|
+
raise ConfigurationError, "#{property.to_s} cannot be empty" if disallow_empty && value.empty?
|
292
|
+
return value unless String == type
|
293
|
+
Utils.unescape( value )
|
294
|
+
end
|
295
|
+
|
296
|
+
def self.validate_and_adjust_boolean( property, value )
|
297
|
+
raise ConfigurationError, "#{property.to_s} must be a Boolean" unless !!value == value
|
298
|
+
value
|
299
|
+
end
|
300
|
+
|
301
|
+
|
302
|
+
def self.validate_and_adjust_integer ( property, value, min, max )
|
303
|
+
if value.is_a?( String )
|
304
|
+
raise ConfigurationError, "#{property.to_s} must be an Integer or Integer string" unless Utils.integer?( value )
|
305
|
+
value = value.to_i
|
306
|
+
end
|
307
|
+
raise ConfigurationError, "#{property.to_s} must be an Integer" unless value.is_a?( Integer )
|
308
|
+
validate_max( property, value, max )
|
309
|
+
validate_min( property, value, min )
|
310
|
+
value
|
311
|
+
end
|
312
|
+
|
313
|
+
def self.validate_and_adjust_number ( property, value, min, max )
|
314
|
+
if value.is_a?( String )
|
315
|
+
raise ConfigurationError, "#{property.to_s} must be a number or number string" unless Utils.number?( value )
|
316
|
+
value = value.to_f
|
317
|
+
end
|
318
|
+
raise ConfigurationError, "#{property.to_s} must be an Integer" unless value.is_a?( Integer )
|
319
|
+
validate_max( property, value, max )
|
320
|
+
validate_min( property, value, min )
|
321
|
+
value
|
322
|
+
end
|
323
|
+
|
324
|
+
def self.validate_and_adjust_guid ( property, value )
|
325
|
+
value = validate_and_adjust( property, value, String )
|
326
|
+
raise ConfigurationError, "#{property.to_s} must be a Guid string, that followy this pattern xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" unless Utils.guid?( value )
|
327
|
+
value
|
328
|
+
end
|
329
|
+
|
330
|
+
def self.validate_max (property, value, max)
|
331
|
+
raise ConfigurationError, "#{property.to_s} cannot be bigger than #{max}" unless value <= max || 0 == max
|
332
|
+
value
|
333
|
+
end
|
334
|
+
|
335
|
+
def self.validate_min (property, value, min)
|
336
|
+
raise ConfigurationError, "#{property.to_s} cannot be less than #{min}" unless value >= min
|
337
|
+
value
|
338
|
+
end
|
339
|
+
|
340
|
+
end
|
341
|
+
end
|