logstash-output-application_insights 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/CONTRIBUTORS +9 -0
- data/DEVELOPER.md +0 -0
- data/Gemfile +26 -0
- data/LICENSE +17 -0
- data/README.md +495 -0
- data/Rakefile +22 -0
- data/lib/logstash/outputs/application_insights.rb +393 -0
- data/lib/logstash/outputs/application_insights/blob.rb +923 -0
- data/lib/logstash/outputs/application_insights/block.rb +118 -0
- data/lib/logstash/outputs/application_insights/channel.rb +259 -0
- data/lib/logstash/outputs/application_insights/channels.rb +142 -0
- data/lib/logstash/outputs/application_insights/client.rb +110 -0
- data/lib/logstash/outputs/application_insights/clients.rb +113 -0
- data/lib/logstash/outputs/application_insights/config.rb +341 -0
- data/lib/logstash/outputs/application_insights/constants.rb +208 -0
- data/lib/logstash/outputs/application_insights/exceptions.rb +55 -0
- data/lib/logstash/outputs/application_insights/flow_control.rb +80 -0
- data/lib/logstash/outputs/application_insights/multi_io_logger.rb +69 -0
- data/lib/logstash/outputs/application_insights/shutdown.rb +96 -0
- data/lib/logstash/outputs/application_insights/state.rb +89 -0
- data/lib/logstash/outputs/application_insights/storage_cleanup.rb +214 -0
- data/lib/logstash/outputs/application_insights/sub_channel.rb +75 -0
- data/lib/logstash/outputs/application_insights/telemetry.rb +99 -0
- data/lib/logstash/outputs/application_insights/timer.rb +90 -0
- data/lib/logstash/outputs/application_insights/utils.rb +139 -0
- data/lib/logstash/outputs/application_insights/version.rb +24 -0
- data/logstash-output-application-insights.gemspec +50 -0
- data/spec/outputs/application_insights_spec.rb +42 -0
- metadata +151 -0
@@ -0,0 +1,110 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
# ----------------------------------------------------------------------------------
|
4
|
+
# Logstash Output Application Insights
|
5
|
+
#
|
6
|
+
# Copyright (c) Microsoft Corporation
|
7
|
+
#
|
8
|
+
# All rights reserved.
|
9
|
+
#
|
10
|
+
# Licensed under the Apache License, Version 2.0 (the License);
|
11
|
+
# you may not use this file except in compliance with the License.
|
12
|
+
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
#
|
18
|
+
# See the Apache Version 2.0 License for specific language governing
|
19
|
+
# permissions and limitations under the License.
|
20
|
+
# ----------------------------------------------------------------------------------
|
21
|
+
|
22
|
+
class LogStash::Outputs::Application_insights
|
23
|
+
class Client
|
24
|
+
|
25
|
+
public
|
26
|
+
|
27
|
+
def initialize ( storage_account_name, force = nil )
|
28
|
+
@storage_account_name = storage_account_name
|
29
|
+
@storage_account = Clients.instance.active_storage_account( storage_account_name, force )
|
30
|
+
|
31
|
+
@tuple = @storage_account[:clients_Q].pop
|
32
|
+
( @current_storage_account_key_index, @current_azure_storage_auth_sas, @current_azure_storage_client) = @tuple
|
33
|
+
if @current_storage_account_key_index != @storage_account[:valid_index]
|
34
|
+
@current_storage_account_key_index = @storage_account[:valid_index]
|
35
|
+
set_current_storage_account_client
|
36
|
+
@tuple = [ @current_storage_account_key_index, @current_azure_storage_auth_sas, @current_azure_storage_client ]
|
37
|
+
end
|
38
|
+
@storage_account_key_index = @current_storage_account_key_index
|
39
|
+
end
|
40
|
+
|
41
|
+
def dispose ( io_failed_reason = nil )
|
42
|
+
if @tuple
|
43
|
+
if @current_storage_account_key_index == @storage_account_key_index
|
44
|
+
@storage_account[:clients_Q] << @tuple
|
45
|
+
else
|
46
|
+
@storage_account[:valid_index] = @current_storage_account_key_index
|
47
|
+
@storage_account[:clients_Q] << [ @current_storage_account_key_index, @current_azure_storage_auth_sas, @current_azure_storage_client ]
|
48
|
+
end
|
49
|
+
@tuple = nil
|
50
|
+
|
51
|
+
Clients.instance.failed_storage_account( @storage_account_name, io_failed_reason ) if io_failed_reason && :blobClient == @last_client_type
|
52
|
+
end
|
53
|
+
nil
|
54
|
+
end
|
55
|
+
|
56
|
+
def blobClient
|
57
|
+
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
58
|
+
@last_client_type = :blobClient
|
59
|
+
@current_azure_storage_client.blobClient
|
60
|
+
end
|
61
|
+
|
62
|
+
def tableClient
|
63
|
+
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
64
|
+
@last_client_type = :blobClient
|
65
|
+
@current_azure_storage_client.tableClient
|
66
|
+
end
|
67
|
+
|
68
|
+
def notifyClient
|
69
|
+
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
70
|
+
@last_client_type = :notifyClient
|
71
|
+
@current_azure_storage_client
|
72
|
+
end
|
73
|
+
|
74
|
+
def storage_auth_sas
|
75
|
+
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
76
|
+
@current_azure_storage_auth_sas
|
77
|
+
end
|
78
|
+
|
79
|
+
|
80
|
+
def switch_storage_account_key!
|
81
|
+
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
82
|
+
@current_storage_account_key_index = ( @current_storage_account_key_index + 1 ) % @storage_account[:keys].length
|
83
|
+
if @current_storage_account_key_index == @storage_account_key_index
|
84
|
+
rollback_storage_account_key
|
85
|
+
false
|
86
|
+
else
|
87
|
+
set_current_storage_account_client
|
88
|
+
true
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
private
|
93
|
+
|
94
|
+
def rollback_storage_account_key
|
95
|
+
raise UnexpectedBranchError, "client already disposed" unless @tuple
|
96
|
+
( @current_storage_account_key_index, @current_azure_storage_auth_sas, @current_azure_storage_client) = @tuple
|
97
|
+
end
|
98
|
+
|
99
|
+
def set_current_storage_account_client
|
100
|
+
configuration = Config.current
|
101
|
+
alt_storage_access_key = @storage_account[:keys][@current_storage_account_key_index]
|
102
|
+
options = { :storage_account_name => @storage_account_name, :storage_access_key => alt_storage_access_key }
|
103
|
+
options[:ca_file] = configuration[:ca_file] unless configuration[:ca_file].empty?
|
104
|
+
@current_azure_storage_client = Azure::Storage::Client.new( options )
|
105
|
+
|
106
|
+
@current_azure_storage_auth_sas = Azure::Storage::Auth::SharedAccessSignature.new( @storage_account_name, alt_storage_access_key )
|
107
|
+
end
|
108
|
+
|
109
|
+
end
|
110
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
# ----------------------------------------------------------------------------------
|
4
|
+
# Logstash Output Application Insights
|
5
|
+
#
|
6
|
+
# Copyright (c) Microsoft Corporation
|
7
|
+
#
|
8
|
+
# All rights reserved.
|
9
|
+
#
|
10
|
+
# Licensed under the Apache License, Version 2.0 (the License);
|
11
|
+
# you may not use this file except in compliance with the License.
|
12
|
+
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
#
|
18
|
+
# See the Apache Version 2.0 License for specific language governing
|
19
|
+
# permissions and limitations under the License.
|
20
|
+
# ----------------------------------------------------------------------------------
|
21
|
+
|
22
|
+
class LogStash::Outputs::Application_insights
|
23
|
+
class Clients
|
24
|
+
|
25
|
+
public
|
26
|
+
|
27
|
+
def initialize
|
28
|
+
configuration = Config.current
|
29
|
+
@resurrect_delay = configuration[:resurrect_delay]
|
30
|
+
@storage_accounts = { }
|
31
|
+
@storage_account_names = [ ]
|
32
|
+
@current_storage_account_names = [ ]
|
33
|
+
@last_storage_account_key_index = 0
|
34
|
+
@state_semaphore = Mutex.new
|
35
|
+
create_blob_clients( configuration )
|
36
|
+
end
|
37
|
+
|
38
|
+
|
39
|
+
def active_storage_account ( storage_account_name , force )
|
40
|
+
raise StorageAccountsOffError if 0 == @current_storage_account_names.length && force.nil?
|
41
|
+
@storage_accounts[storage_account_name]
|
42
|
+
end
|
43
|
+
|
44
|
+
def create_blob_clients ( configuration )
|
45
|
+
configuration[:storage_account_name_key].each do |storage_account_name, storage_account_keys|
|
46
|
+
|
47
|
+
clients_Q = Queue.new
|
48
|
+
@storage_accounts[storage_account_name] = { :keys => storage_account_keys, :clients_Q => clients_Q, :valid_index => 0, :state => :on, :off_reason => [] }
|
49
|
+
40.times do
|
50
|
+
# lazy creation, clients will be created when first needed
|
51
|
+
clients_Q << [ nil, nil, nil]
|
52
|
+
end
|
53
|
+
@storage_account_names << storage_account_name
|
54
|
+
end
|
55
|
+
|
56
|
+
@current_storage_account_names = @storage_account_names.clone
|
57
|
+
end
|
58
|
+
|
59
|
+
|
60
|
+
def storage_account_state_on? ( storage_account_name = nil )
|
61
|
+
if storage_account_name
|
62
|
+
@storage_accounts[storage_account_name][:off_reason].empty?
|
63
|
+
else
|
64
|
+
@current_storage_account_names.length > 0
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def failed_storage_account( storage_account_name, io_failed_reason )
|
69
|
+
storage_account = @storage_accounts[storage_account_name]
|
70
|
+
@state_semaphore.synchronize {
|
71
|
+
unless storage_account[:off_reason].include?( io_failed_reason )
|
72
|
+
storage_account[:off_reason] << io_failed_reason
|
73
|
+
if storage_account[:off_reason].length == 1
|
74
|
+
current_storage_account_names = [ ]
|
75
|
+
@current_storage_account_names.each do |account_name|
|
76
|
+
current_storage_account_names << account_name unless account_name == storage_account_name
|
77
|
+
end
|
78
|
+
@current_storage_account_names = current_storage_account_names
|
79
|
+
|
80
|
+
Thread.new( storage_account_name ) do |account_name|
|
81
|
+
loop do
|
82
|
+
sleep( @resurrect_delay )
|
83
|
+
if Blob.new.test_storage( account_name )
|
84
|
+
@state_semaphore.synchronize {
|
85
|
+
storage_account = @storage_accounts[account_name]
|
86
|
+
storage_account[:off_reason] = [ ]
|
87
|
+
@current_storage_account_names << account_name
|
88
|
+
}
|
89
|
+
break
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
95
|
+
end
|
96
|
+
}
|
97
|
+
end
|
98
|
+
|
99
|
+
def get_random_active_storage( exclude )
|
100
|
+
@state_semaphore.synchronize { storage_account_name = ( @current_storage_account_names - exclude ).sample if @current_storage_account_names.length > 0 }
|
101
|
+
end
|
102
|
+
|
103
|
+
public
|
104
|
+
|
105
|
+
@@instance = Clients.new
|
106
|
+
|
107
|
+
def self.instance
|
108
|
+
@@instance
|
109
|
+
end
|
110
|
+
|
111
|
+
private_class_method :new
|
112
|
+
end
|
113
|
+
end
|
@@ -0,0 +1,341 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
# ----------------------------------------------------------------------------------
|
4
|
+
# Logstash Output Application Insights
|
5
|
+
#
|
6
|
+
# Copyright (c) Microsoft Corporation
|
7
|
+
#
|
8
|
+
# All rights reserved.
|
9
|
+
#
|
10
|
+
# Licensed under the Apache License, Version 2.0 (the License);
|
11
|
+
# you may not use this file except in compliance with the License.
|
12
|
+
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
13
|
+
#
|
14
|
+
# Unless required by applicable law or agreed to in writing, software
|
15
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
16
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
17
|
+
#
|
18
|
+
# See the Apache Version 2.0 License for specific language governing
|
19
|
+
# permissions and limitations under the License.
|
20
|
+
# ----------------------------------------------------------------------------------
|
21
|
+
|
22
|
+
class LogStash::Outputs::Application_insights
|
23
|
+
class Config
|
24
|
+
|
25
|
+
public
|
26
|
+
|
27
|
+
@@configuration = {}
|
28
|
+
|
29
|
+
def self.current
|
30
|
+
@@configuration
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.validate_and_adjust_configuration ( configuration )
|
34
|
+
configuration.each_pair { |config_name, config_value|
|
35
|
+
raise ConfigurationError, "#{config_name.to_s} must be defined" unless config_value || BOOLEAN_PROPERTIES.include?( config_name )
|
36
|
+
case config_name
|
37
|
+
|
38
|
+
when :logger_level
|
39
|
+
logger_level = validate_and_adjust( config_name, config_value, String )
|
40
|
+
raise ConfigurationError, "#{config_name.to_s} can be set to only one of the valid log levels" unless LOGGER_LEVEL_MAP[logger_level.upcase.to_sym]
|
41
|
+
configuration[config_name] = LOGGER_LEVEL_MAP[logger_level.upcase.to_sym]
|
42
|
+
|
43
|
+
when :logger_files
|
44
|
+
config_value = [ config_value ] if config_value.is_a?( String )
|
45
|
+
logger_files = validate_and_adjust( config_name, config_value, Array )
|
46
|
+
|
47
|
+
i = 0
|
48
|
+
logger_files.map! do |file_name|
|
49
|
+
file_name = validate_and_adjust( "#{config_name.to_s}[#{i}]", file_name, String )
|
50
|
+
i += 1
|
51
|
+
if "stdout" == file_name.downcase
|
52
|
+
file_name = :stdout
|
53
|
+
file = STDOUT
|
54
|
+
elsif "stderr" == file_name.downcase
|
55
|
+
file_name = :stderr
|
56
|
+
file = STDERR
|
57
|
+
else
|
58
|
+
file = ::File.open( file_name, "a+" )
|
59
|
+
end
|
60
|
+
[ file_name, file ]
|
61
|
+
end
|
62
|
+
configuration[config_name] = logger_files
|
63
|
+
|
64
|
+
when :logger_progname
|
65
|
+
configuration[config_name] = validate_and_adjust( config_name, config_value, String )
|
66
|
+
|
67
|
+
when :logger_shift_size
|
68
|
+
configuration[config_name] = validate_and_adjust_integer( config_name, config_value, MIN_LOGGER_SHIFT_SIZE, MAX_LOGGER_SHIFT_SIZE )
|
69
|
+
|
70
|
+
when :logger_shift_age
|
71
|
+
if config_value.is_a?( String )
|
72
|
+
config_value = validate_and_adjust( config_name, config_value, String ).downcase
|
73
|
+
raise ConfigurationError, "#{config_name.to_s} if string, can be set to only one of the following values: #{VALID_LOGGER_SHIFT_AGES}" unless VALID_LOGGER_SHIFT_AGES. include?( config_value )
|
74
|
+
elsif config_value.is_a?( Integer )
|
75
|
+
config_value = validate_and_adjust_integer( config_name, config_value, MIN_LOGGER_SHIFT_AGE, MAX_LOGGER_SHIFT_AGE )
|
76
|
+
else
|
77
|
+
raise ConfigurationError, "#{config_name.to_s} must be either a string or integer"
|
78
|
+
end
|
79
|
+
configuration[config_name] = config_value
|
80
|
+
|
81
|
+
when :azure_storage_container_prefix
|
82
|
+
azure_storage_container_prefix = validate_and_adjust( config_name, config_value, String )
|
83
|
+
unless azure_storage_container_prefix.empty?
|
84
|
+
len = 63 - "-#{AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX}-yyyy-mm-dd".length
|
85
|
+
validate_max( "azure_storage_container_prefix length", azure_storage_container_prefix.length, len )
|
86
|
+
azure_storage_container_prefix += "-"
|
87
|
+
end
|
88
|
+
azure_storage_container_prefix = azure_storage_container_prefix.downcase
|
89
|
+
container_name = "#{azure_storage_container_prefix}#{AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX}-yyyy-mm-dd"
|
90
|
+
raise ConfigurationError, "#{config_name.to_s} must have only alphanumeric and dash characters, cannot start or end with a dash, and a dash cannot follow a dash" unless Utils.valid_container_name?( container_name )
|
91
|
+
configuration[config_name] = azure_storage_container_prefix + AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX
|
92
|
+
|
93
|
+
when :azure_storage_azure_storage_table_prefix
|
94
|
+
azure_storage_table_prefix = validate_and_adjust( config_name, config_value, String )
|
95
|
+
unless azure_storage_table_prefix.empty?
|
96
|
+
len = 63 - "-#{AZURE_STORAGE_TABLE_LOGSTASH_PREFIX}yyyymmdd".length
|
97
|
+
validate_max( "azure_storage_table_prefix length", azure_storage_table_prefix.length, len )
|
98
|
+
end
|
99
|
+
table_name = "#{azure_storage_table_prefix}#{AZURE_STORAGE_TABLE_LOGSTASH_PREFIX}yyyymmdd"
|
100
|
+
raise ConfigurationError, "#{config_name} must have only alphanumeric" unless Utils.valid_table_name?( table_name )
|
101
|
+
configuration[config_name] = azure_storage_table_prefix + AZURE_STORAGE_TABLE_LOGSTASH_PREFIX
|
102
|
+
|
103
|
+
when :ca_file
|
104
|
+
config_value = validate_and_adjust( config_name, config_value, String )
|
105
|
+
unless config_value.empty?
|
106
|
+
raise ConfigurationError, "#{config_name} must have a valid path name" unless Utils.valid_file_path?( config_value )
|
107
|
+
end
|
108
|
+
configuration[config_name] = validate_and_adjust( config_name, config_value, String )
|
109
|
+
|
110
|
+
when :azure_storage_blob_prefix
|
111
|
+
azure_storage_blob_prefix = validate_and_adjust( config_name, config_value, String )
|
112
|
+
unless azure_storage_blob_prefix.empty?
|
113
|
+
len = 1024 - "-#{AZURE_STORAGE_BLOB_LOGSTASH_PREFIX}_ikey-#{INSTRUMENTATION_KEY_TEMPLATE}_table-#{TABLE_ID_TEMPLATE}_yyyy-mm-dd-HH-MM-SS-LLL".length
|
114
|
+
validate_max( "azure_storage_blob_prefix length", azure_storage_blob_prefix.length, len )
|
115
|
+
azure_storage_blob_prefix += "-"
|
116
|
+
end
|
117
|
+
azure_storage_blob_prefix += AZURE_STORAGE_BLOB_LOGSTASH_PREFIX
|
118
|
+
|
119
|
+
raise ConfigurationError, "#{config_name.to_s} doesn't meet url format" unless Utils.url?( "http://storage/container/#{azure_storage_blob_prefix}_ikey-#{INSTRUMENTATION_KEY_TEMPLATE}_table-#{TABLE_ID_TEMPLATE}.json" )
|
120
|
+
configuration[config_name] = azure_storage_blob_prefix
|
121
|
+
|
122
|
+
when :table_id
|
123
|
+
configuration[config_name] = validate_and_adjust_guid( config_name, config_value )
|
124
|
+
|
125
|
+
when :blob_max_bytesize
|
126
|
+
configuration[config_name] = validate_and_adjust_integer( config_name, config_value, MIN_BLOB_MAX_BYTESIZE, MAX_BLOB_MAX_BYTESIZE )
|
127
|
+
|
128
|
+
when :blob_max_events
|
129
|
+
configuration[config_name] = validate_and_adjust_integer( config_name, config_value, MIN_BLOB_MAX_EVENTS, MAX_BLOB_MAX_EVENTS )
|
130
|
+
|
131
|
+
when :blob_retention_time
|
132
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_BLOB_RETENTION_TIME, MAX_BLOB_RETENTION_TIME )
|
133
|
+
|
134
|
+
when :blob_access_expiry_time
|
135
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_BLOB_ACCESS_EXPIRY_TIME, MAX_BLOB_ACCESS_EXPIRY_TIME )
|
136
|
+
|
137
|
+
when :resurrect_delay
|
138
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_STORAGE_RESURRECT_DELAY, MAX_STORAGE_RESURRECT_DELAY )
|
139
|
+
|
140
|
+
when :flow_control_suspend_bytes
|
141
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_FLOW_CONTROL_SUSPEND_BYTES, MAX_FLOW_CONTROL_SUSPEND_BYTES )
|
142
|
+
|
143
|
+
when :flow_control_resume_bytes
|
144
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_FLOW_CONTROL_RESUME_BYTES, MAX_FLOW_CONTROL_RESUME_BYTES )
|
145
|
+
|
146
|
+
when :flow_control_delay
|
147
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_FLOW_CONTROL_DELAY, MAX_FLOW_CONTROL_DELAY )
|
148
|
+
|
149
|
+
when :io_retry_delay
|
150
|
+
configuration[config_name] = validate_and_adjust_number( config_name, config_value, MIN_IO_RETRY_DELAY, MAX_IO_RETRY_DELAY )
|
151
|
+
|
152
|
+
when :io_max_retries
|
153
|
+
configuration[config_name] = validate_and_adjust_integer( config_name, config_value, MIN_IO_MAX_RETRIES, MAX_IO_MAX_RETRIES )
|
154
|
+
|
155
|
+
when :storage_account_name_key
|
156
|
+
config_value = validate_and_adjust( config_name, config_value, Array )
|
157
|
+
if config_value.empty?
|
158
|
+
raise ConfigurationError, "#{config_name.to_s} is empty, at least one storage account name should be defined" unless ENV['AZURE_STORAGE_ACCOUNT']
|
159
|
+
raise ConfigurationError, "#{config_name.to_s} is empty, at least one storage account access key should be defined" unless ENV['AZURE_STORAGE_ACCESS_KEY']
|
160
|
+
config_value = [ ENV['AZURE_STORAGE_ACCOUNT'], ENV['AZURE_STORAGE_ACCESS_KEY'] ]
|
161
|
+
end
|
162
|
+
|
163
|
+
storage_account_name_key = validate_and_adjust( config_name, config_value, Array, :disallow_empty )
|
164
|
+
unless storage_account_name_key[0].is_a?( Array )
|
165
|
+
raise ConfigurationError, "#{config_name.to_s} property is empty, should contain at least one pair: account_name, key" unless 2 == storage_account_name_key.length
|
166
|
+
storage_account_name_key = [ [ storage_account_name_key[0], storage_account_name_key[1] ]]
|
167
|
+
end
|
168
|
+
|
169
|
+
index = 0
|
170
|
+
storage_account_name_key.map! { |pair|
|
171
|
+
pair = validate_and_adjust( "#{config_name.to_s}[#{index}]", pair, Array, :disallow_empty )
|
172
|
+
raise ConfigurationError, "#{config_name.to_s}[#{index}] must have two items" unless 2 == pair.length
|
173
|
+
|
174
|
+
( name, keys ) = pair
|
175
|
+
name = validate_and_adjust( "#{config_name.to_s}[#{index}]:name", name, String, :disallow_empty )
|
176
|
+
raise ConfigurationError, "#{config_name.to_s}[#{index}]:name must between 3 to 24 characters" unless (name.length >= 3) && (name.length <= 24)
|
177
|
+
raise ConfigurationError, "##{config_name.to_s}[#{index}]:name bad format, must have only alphanumeric characters" unless Utils.alphanumeric?( name )
|
178
|
+
|
179
|
+
keys = [ keys ] if keys.is_a?( String )
|
180
|
+
keys = validate_and_adjust( "#{config_name}[#{index}]:keys", keys, Array, :disallow_empty )
|
181
|
+
keys.each_index do |i|
|
182
|
+
key = validate_and_adjust( "#{config_name}[#{index}:keys[#{i}]", keys[i], String, :disallow_empty )
|
183
|
+
raise ConfigurationError, "#{config_name}[#{index}:keys[#{i}] must have only valid base64 characters" unless Utils.base64?( key )
|
184
|
+
end
|
185
|
+
index += 1
|
186
|
+
[ name.downcase, keys ]
|
187
|
+
}
|
188
|
+
configuration[config_name] = storage_account_name_key
|
189
|
+
|
190
|
+
when :tables
|
191
|
+
tables = validate_and_adjust( config_name, config_value, Hash )
|
192
|
+
tables.each_pair { |table_id, properties|
|
193
|
+
table_id = validate_and_adjust_guid( "#{config_name}:table_id", table_id )
|
194
|
+
info = "#{config_name}[#{table_id}]"
|
195
|
+
properties = validate_and_adjust( info, properties, Hash )
|
196
|
+
properties = Utils.symbolize_hash_keys( properties )
|
197
|
+
validate_and_adjust_table_properties!( properties, configuration, info )
|
198
|
+
configuration[config_name][table_id] = properties
|
199
|
+
}
|
200
|
+
end
|
201
|
+
}
|
202
|
+
validate_and_adjust_table_properties!( configuration, configuration )
|
203
|
+
@@configuration = configuration
|
204
|
+
end
|
205
|
+
|
206
|
+
|
207
|
+
def self.symbolize_table_properties ( properties )
|
208
|
+
new_properties = {}
|
209
|
+
properties.each_pair { |property_name, property_value|
|
210
|
+
new_properties[property_name.to_sym] = property_value
|
211
|
+
}
|
212
|
+
new_properties
|
213
|
+
end
|
214
|
+
|
215
|
+
def self.validate_and_adjust_table_properties! ( properties, configuration, base_info = nil )
|
216
|
+
|
217
|
+
properties.each_pair { |property_name, property_value|
|
218
|
+
info = ( base_info.nil? ? "#{property_name}" : "#{base_info}[#{property_name}]" )
|
219
|
+
raise ConfigurationError, "#{info} must be defined" unless property_value || BOOLEAN_PROPERTIES.include?( property_name )
|
220
|
+
|
221
|
+
case property_name.downcase
|
222
|
+
|
223
|
+
when :intrumentation_key
|
224
|
+
properties[:intrumentation_key] = validate_and_adjust_guid( info, property_value )
|
225
|
+
when :blob_serialization
|
226
|
+
property_value = property_value.downcase
|
227
|
+
raise ConfigurationError, "#{info}, can be set to only one of the following values: #{VALID_EXT_EVENT_FORMAT}" unless VALID_EXT_EVENT_FORMAT.include?( property_value )
|
228
|
+
properties[:blob_serialization] = validate_and_adjust_ext( info ,property_value, configuration[:azure_storage_blob_prefix] ) # be careful depends on order
|
229
|
+
when :csv_default_value
|
230
|
+
properties[:csv_default_value] = validate_and_adjust( info, property_value, String )
|
231
|
+
when :csv_separator
|
232
|
+
properties[:csv_separator] = validate_and_adjust( info, property_value, String )
|
233
|
+
when :blob_max_delay
|
234
|
+
properties[:blob_max_delay] = validate_and_adjust_number( info, property_value, MIN_BLOB_MAX_DELAY, MAX_BLOB_MAX_DELAY )
|
235
|
+
when :event_separator
|
236
|
+
properties[:event_separator] = validate_and_adjust( info, property_value, String )
|
237
|
+
when :serialized_event_field
|
238
|
+
properties[:serialized_event_field] = ( property_value.nil? ? nil : validate_and_adjust( info, property_value, String ) )
|
239
|
+
when :case_insensitive_columns
|
240
|
+
properties[:case_insensitive_columns] = validate_and_adjust_boolean( info, property_value )
|
241
|
+
when :table_columns
|
242
|
+
if property_value.nil?
|
243
|
+
properties[:table_columns] = property_value
|
244
|
+
else
|
245
|
+
table_columns = property_value
|
246
|
+
table_columns = [ table_columns ] if table_columns.is_a?( String )
|
247
|
+
table_columns = validate_and_adjust( info, table_columns, Array)
|
248
|
+
new_table_columns = []
|
249
|
+
index = 0
|
250
|
+
table_columns.each do |column|
|
251
|
+
new_column = {}
|
252
|
+
|
253
|
+
column = { COLUMN_PROPERTY_NAME => column } if column.is_a?( String )
|
254
|
+
column = validate_and_adjust( "#{info}[#{index}]", column, Hash, :disallow_empty )
|
255
|
+
raise ConfigurationError, "#{info}[#{index}][#{COLUMN_PROPERTY_NAME}] must be defined" unless column[COLUMN_PROPERTY_NAME]
|
256
|
+
new_column[:name] = validate_and_adjust( "#{info}[#{index}][#{COLUMN_PROPERTY_NAME}]", column[COLUMN_PROPERTY_NAME], String )
|
257
|
+
|
258
|
+
if column[COLUMN_PROPERTY_DEFAULT]
|
259
|
+
new_column[:default] = validate_and_adjust( "#{info}[#{index}][#{COLUMN_PROPERTY_DEFAULT}]", column[COLUMN_PROPERTY_DEFAULT], String )
|
260
|
+
end
|
261
|
+
|
262
|
+
if column[COLUMN_PROPERTY_TYPE]
|
263
|
+
new_column[:type] = validate_and_adjust( "#{info}[#{index}][#{COLUMN_PROPERTY_TYPE}]", column[COLUMN_PROPERTY_TYPE], String ).downcase
|
264
|
+
raise ConfigurationError, "#{info}[#{index}][#{COLUMN_PROPERTY_TYPE}] can be only one of the following values: #{VALID_FIELDS_MAP_TYPES}" unless VALID_FIELDS_MAP_TYPES.any? {|type| type == new_column[:type]}
|
265
|
+
new_column[:type] = new_column[:type].to_sym
|
266
|
+
end
|
267
|
+
new_table_columns << new_column
|
268
|
+
index += 1
|
269
|
+
end
|
270
|
+
properties[:table_columns] = new_table_columns
|
271
|
+
end
|
272
|
+
|
273
|
+
else
|
274
|
+
end
|
275
|
+
}
|
276
|
+
end
|
277
|
+
|
278
|
+
|
279
|
+
private
|
280
|
+
|
281
|
+
def self.validate_and_adjust_ext ( property, ext, prefix )
|
282
|
+
ext = validate_and_adjust( property, ext, String )
|
283
|
+
raise ConfigurationError, "#{property.to_s} must be a valid extension string, have only alphanumeric, dash and underline characters" unless Utils.ext?( ext )
|
284
|
+
len = 1024 - "#{prefix}-#{AZURE_STORAGE_BLOB_LOGSTASH_PREFIX}_ikey-#{INSTRUMENTATION_KEY_TEMPLATE}_table-#{TABLE_ID_TEMPLATE}_yyyy-mm-dd-HH-MM-SS-LLL".length
|
285
|
+
raise ConfigurationError, "#{property.to_s} length cannot be more than #{len} characters" unless ext.length <= len
|
286
|
+
ext
|
287
|
+
end
|
288
|
+
|
289
|
+
def self.validate_and_adjust ( property, value, type, disallow_empty = false )
|
290
|
+
raise ConfigurationError, "#{property.to_s} must be an #{type.to_s}" unless value.is_a?( type )
|
291
|
+
raise ConfigurationError, "#{property.to_s} cannot be empty" if disallow_empty && value.empty?
|
292
|
+
return value unless String == type
|
293
|
+
Utils.unescape( value )
|
294
|
+
end
|
295
|
+
|
296
|
+
def self.validate_and_adjust_boolean( property, value )
|
297
|
+
raise ConfigurationError, "#{property.to_s} must be a Boolean" unless !!value == value
|
298
|
+
value
|
299
|
+
end
|
300
|
+
|
301
|
+
|
302
|
+
def self.validate_and_adjust_integer ( property, value, min, max )
|
303
|
+
if value.is_a?( String )
|
304
|
+
raise ConfigurationError, "#{property.to_s} must be an Integer or Integer string" unless Utils.integer?( value )
|
305
|
+
value = value.to_i
|
306
|
+
end
|
307
|
+
raise ConfigurationError, "#{property.to_s} must be an Integer" unless value.is_a?( Integer )
|
308
|
+
validate_max( property, value, max )
|
309
|
+
validate_min( property, value, min )
|
310
|
+
value
|
311
|
+
end
|
312
|
+
|
313
|
+
def self.validate_and_adjust_number ( property, value, min, max )
|
314
|
+
if value.is_a?( String )
|
315
|
+
raise ConfigurationError, "#{property.to_s} must be a number or number string" unless Utils.number?( value )
|
316
|
+
value = value.to_f
|
317
|
+
end
|
318
|
+
raise ConfigurationError, "#{property.to_s} must be an Integer" unless value.is_a?( Integer )
|
319
|
+
validate_max( property, value, max )
|
320
|
+
validate_min( property, value, min )
|
321
|
+
value
|
322
|
+
end
|
323
|
+
|
324
|
+
def self.validate_and_adjust_guid ( property, value )
|
325
|
+
value = validate_and_adjust( property, value, String )
|
326
|
+
raise ConfigurationError, "#{property.to_s} must be a Guid string, that followy this pattern xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" unless Utils.guid?( value )
|
327
|
+
value
|
328
|
+
end
|
329
|
+
|
330
|
+
def self.validate_max (property, value, max)
|
331
|
+
raise ConfigurationError, "#{property.to_s} cannot be bigger than #{max}" unless value <= max || 0 == max
|
332
|
+
value
|
333
|
+
end
|
334
|
+
|
335
|
+
def self.validate_min (property, value, min)
|
336
|
+
raise ConfigurationError, "#{property.to_s} cannot be less than #{min}" unless value >= min
|
337
|
+
value
|
338
|
+
end
|
339
|
+
|
340
|
+
end
|
341
|
+
end
|