logstash-output-application_insights 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +5 -0
  3. data/CONTRIBUTORS +9 -0
  4. data/DEVELOPER.md +0 -0
  5. data/Gemfile +26 -0
  6. data/LICENSE +17 -0
  7. data/README.md +495 -0
  8. data/Rakefile +22 -0
  9. data/lib/logstash/outputs/application_insights.rb +393 -0
  10. data/lib/logstash/outputs/application_insights/blob.rb +923 -0
  11. data/lib/logstash/outputs/application_insights/block.rb +118 -0
  12. data/lib/logstash/outputs/application_insights/channel.rb +259 -0
  13. data/lib/logstash/outputs/application_insights/channels.rb +142 -0
  14. data/lib/logstash/outputs/application_insights/client.rb +110 -0
  15. data/lib/logstash/outputs/application_insights/clients.rb +113 -0
  16. data/lib/logstash/outputs/application_insights/config.rb +341 -0
  17. data/lib/logstash/outputs/application_insights/constants.rb +208 -0
  18. data/lib/logstash/outputs/application_insights/exceptions.rb +55 -0
  19. data/lib/logstash/outputs/application_insights/flow_control.rb +80 -0
  20. data/lib/logstash/outputs/application_insights/multi_io_logger.rb +69 -0
  21. data/lib/logstash/outputs/application_insights/shutdown.rb +96 -0
  22. data/lib/logstash/outputs/application_insights/state.rb +89 -0
  23. data/lib/logstash/outputs/application_insights/storage_cleanup.rb +214 -0
  24. data/lib/logstash/outputs/application_insights/sub_channel.rb +75 -0
  25. data/lib/logstash/outputs/application_insights/telemetry.rb +99 -0
  26. data/lib/logstash/outputs/application_insights/timer.rb +90 -0
  27. data/lib/logstash/outputs/application_insights/utils.rb +139 -0
  28. data/lib/logstash/outputs/application_insights/version.rb +24 -0
  29. data/logstash-output-application-insights.gemspec +50 -0
  30. data/spec/outputs/application_insights_spec.rb +42 -0
  31. metadata +151 -0
@@ -0,0 +1,118 @@
1
+ # encoding: utf-8
2
+
3
+ # ----------------------------------------------------------------------------------
4
+ # Logstash Output Application Insights
5
+ #
6
+ # Copyright (c) Microsoft Corporation
7
+ #
8
+ # All rights reserved.
9
+ #
10
+ # Licensed under the Apache License, Version 2.0 (the License);
11
+ # you may not use this file except in compliance with the License.
12
+ # You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS,
16
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ #
18
+ # See the Apache Version 2.0 License for specific language governing
19
+ # permissions and limitations under the License.
20
+ # ----------------------------------------------------------------------------------
21
+
22
+ class LogStash::Outputs::Application_insights
23
+ class Block
24
+
25
+ attr_reader :bytes
26
+ attr_reader :buffer
27
+ attr_reader :bytesize
28
+ attr_reader :events_count
29
+ attr_reader :block_numbers
30
+ attr_reader :done_time
31
+ attr_reader :oldest_event_time
32
+
33
+
34
+ public
35
+
36
+ @@Block_number = 0
37
+ @@semaphore = Mutex.new
38
+
39
+ def self.generate_block_number
40
+ @@semaphore.synchronize { @@Block_number = ( @@Block_number + 1 ) % 1000000 }
41
+ end
42
+
43
+
44
+
45
+ def initialize ( event_separator )
46
+ @buffer = [ ]
47
+ @bytesize = 0
48
+ @events_count = 0
49
+ @event_separator = event_separator
50
+ @event_separator_bytesize = @event_separator.bytesize
51
+ @block_numbers = nil
52
+ end
53
+
54
+ # concatenate two blocks into one
55
+ def concat ( other )
56
+ if @bytesize + other.bytesize <= BLOB_BLOCK_MAX_BYTESIZE
57
+ if @block_numbers
58
+ @block_numbers.concat( other.block_numbers ) if @block_numbers
59
+ @bytes += other.bytes
60
+ @done_time = other.done_time if other.done_time > @done_time
61
+ else
62
+ @buffer.concat( other.buffer )
63
+ end
64
+ @events_count += other.events_count
65
+ @oldest_event_time = other.oldest_event_time if other.oldest_event_time < @oldest_event_time
66
+ @bytesize += other.bytesize
67
+ end
68
+ end
69
+
70
+ def << (data)
71
+ @bytesize += data.bytesize + @event_separator_bytesize
72
+
73
+ # if first data, it will accept even it overflows
74
+ if is_overflowed? && @events_count > 0
75
+ @bytesize -= data.bytesize + @event_separator_bytesize
76
+ raise BlockTooSmallError if is_empty?
77
+ raise BlockOverflowError
78
+ end
79
+
80
+ @oldest_event_time ||= Time.now.utc
81
+ @events_count += 1
82
+ @buffer << data
83
+ end
84
+
85
+ def dispose
86
+ @bytes = nil
87
+ @buffer = nil
88
+ @bytesize = nil
89
+ @events_count = nil
90
+ @done_time = nil
91
+ @oldest_event_time = nil
92
+ @block_numbers = nil
93
+ end
94
+
95
+ def seal
96
+ @block_numbers = [ Block.generate_block_number ]
97
+ @done_time = Time.now.utc
98
+ @buffer << "" # required to add eol after last event
99
+ @bytes = @buffer.join( @event_separator )
100
+ @buffer = nil # release the memory of the array
101
+ end
102
+
103
+ def is_full?
104
+ @bytesize >= BLOB_BLOCK_MAX_BYTESIZE
105
+ end
106
+
107
+ private
108
+
109
+ def is_overflowed?
110
+ @bytesize > BLOB_BLOCK_MAX_BYTESIZE
111
+ end
112
+
113
+ def is_empty?
114
+ @bytesize <= 0
115
+ end
116
+
117
+ end
118
+ end
@@ -0,0 +1,259 @@
1
+ # encoding: utf-8
2
+
3
+ # ----------------------------------------------------------------------------------
4
+ # Logstash Output Application Insights
5
+ #
6
+ # Copyright (c) Microsoft Corporation
7
+ #
8
+ # All rights reserved.
9
+ #
10
+ # Licensed under the Apache License, Version 2.0 (the License);
11
+ # you may not use this file except in compliance with the License.
12
+ # You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS,
16
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ #
18
+ # See the Apache Version 2.0 License for specific language governing
19
+ # permissions and limitations under the License.
20
+ # ----------------------------------------------------------------------------------
21
+
22
+ class LogStash::Outputs::Application_insights
23
+ class Channel
24
+
25
+ attr_reader :intrumentation_key
26
+ attr_reader :table_id
27
+ attr_reader :failed_on_upload_retry_Q
28
+ attr_reader :failed_on_notify_retry_Q
29
+ attr_reader :event_format_ext
30
+ attr_reader :blob_max_delay
31
+
32
+ public
33
+
34
+ def initialize ( intrumentation_key, table_id )
35
+ @closing = false
36
+ configuration = Config.current
37
+
38
+ @logger = configuration[:logger]
39
+
40
+ @logger.debug { "Create a new channel, intrumentation_key / table_id : #{intrumentation_key} / #{table_id}" }
41
+ @intrumentation_key = intrumentation_key
42
+ @table_id = table_id
43
+ set_table_properties( configuration )
44
+ @semaphore = Mutex.new
45
+ @failed_on_upload_retry_Q = Queue.new
46
+ @failed_on_notify_retry_Q = Queue.new
47
+ @workers_channel = { }
48
+ @active_blobs = [ Blob.new( self, 1 ) ]
49
+ @state = State.instance
50
+
51
+ launch_upload_recovery_thread
52
+ launch_notify_recovery_thread
53
+ end
54
+
55
+ def close
56
+ @closing = true
57
+ @active_blobs.each do |blob|
58
+ blob.close
59
+ end
60
+ end
61
+
62
+ def stopped?
63
+ @closing
64
+ end
65
+
66
+ def << ( event )
67
+ if @serialized_event_field && event[@serialized_event_field]
68
+ serialized_event = serialize_serialized_event_field( event[@serialized_event_field] )
69
+ else
70
+ serialized_event = ( EXT_EVENT_FORMAT_CSV == @serialization ? serialize_to_csv( event ) : serialize_to_json( event ) )
71
+ end
72
+
73
+ if serialized_event
74
+ sub_channel = @workers_channel[Thread.current] || @semaphore.synchronize { @workers_channel[Thread.current] = Sub_channel.new( @event_separator ) }
75
+ sub_channel << serialized_event
76
+ else
77
+ @logger.warn { "event not uploaded, no relevant data in event. table_id: #{table_id}, event: #{event}" }
78
+ end
79
+ end
80
+
81
+ def flush
82
+ block_list = collect_blocks
83
+ enqueue_blocks( block_list )
84
+ end
85
+
86
+
87
+ private
88
+
89
+ def collect_blocks
90
+ workers_channel = @semaphore.synchronize { @workers_channel.dup }
91
+ full_block_list = [ ]
92
+ prev_last_block = nil
93
+
94
+ workers_channel.each_value do |worker_channel|
95
+ block_list = worker_channel.get_block_list!
96
+ unless block_list.empty?
97
+ last_block = block_list.pop
98
+ full_block_list.concat( block_list )
99
+ if prev_last_block
100
+ unless prev_last_block.concat( last_block )
101
+ full_block_list << prev_last_block
102
+ prev_last_block = last_block
103
+ end
104
+ else
105
+ prev_last_block = last_block
106
+ end
107
+ end
108
+ end
109
+ full_block_list << prev_last_block if prev_last_block
110
+ full_block_list
111
+ end
112
+
113
+
114
+ def enqueue_blocks ( block_list )
115
+ block_list.each do |block|
116
+ block.seal
117
+ find_blob << block
118
+ end
119
+ end
120
+
121
+
122
+ def launch_upload_recovery_thread
123
+ #recovery thread
124
+ Thread.new do
125
+ next_block = nil
126
+ loop do
127
+ block_to_upload = next_block || @failed_on_upload_retry_Q.pop
128
+ next_block = nil
129
+ until Clients.instance.storage_account_state_on? do
130
+ Stud.stoppable_sleep( 60 ) { stopped? }
131
+ end
132
+ if block_to_upload
133
+ find_blob << block_to_upload
134
+ end
135
+ end
136
+ end
137
+ end
138
+
139
+
140
+ # thread that failed to notify due to Application Isights error, such as wrong key or wrong schema
141
+ def launch_notify_recovery_thread
142
+ #recovery thread
143
+ Thread.new do
144
+ loop do
145
+ tuple ||= @failed_on_notify_retry_Q.pop
146
+ begin
147
+ Stud.stoppable_sleep( 60 ) { stopped? }
148
+ end until Clients.instance.storage_account_state_on? || stopped?
149
+ if stopped?
150
+ @state.dec_pending_notifications
151
+ else
152
+ Blob.new.notify( tuple )
153
+ end
154
+ end
155
+ end
156
+ end
157
+
158
+
159
+ def serialize_serialized_event_field ( data )
160
+ serialized_data = nil
161
+ if data.is_a?( String )
162
+ serialized_data = data
163
+ elsif EXT_EVENT_FORMAT_CSV == @serialization
164
+ if data.is_a?( Array )
165
+ serialized_data = data.to_csv( :col_sep => @csv_separator )
166
+ elsif data.is_a?( Hash )
167
+ serialized_data = serialize_to_csv( data )
168
+ end
169
+ elsif EXT_EVENT_FORMAT_JSON == @serialization
170
+ if data.is_a?( Hash )
171
+ serialized_data = serialize_to_json( data )
172
+ elsif data.is_a?( Array ) && !@table_columns.nil?
173
+ serialized_data = serialize_to_json( Hash[@table_columns.map {|column| column[:name]}.zip( data )] )
174
+ end
175
+ end
176
+ serialized_data
177
+ end
178
+
179
+
180
+ def serialize_to_json ( event )
181
+ return event.to_json unless !@table_columns.nil?
182
+
183
+ fields = ( @case_insensitive_columns ? Utils.downcase_hash_keys( event.to_hash ) : event )
184
+
185
+ json_hash = { }
186
+ @table_columns.each do |column|
187
+ value = fields[column[:field_name]] || column[:default]
188
+ json_hash[column[:name]] = value if value
189
+ end
190
+ return nil if json_hash.empty?
191
+ json_hash.to_json
192
+ end
193
+
194
+
195
+ def serialize_to_csv ( event )
196
+ return nil unless !@table_columns.nil?
197
+
198
+ fields = ( @case_insensitive_columns ? Utils.downcase_hash_keys( event.to_hash ) : event )
199
+
200
+ csv_array = [ ]
201
+ @table_columns.each do |column|
202
+ value = fields[column[:field_name]] || column[:default] || @csv_default_value
203
+ type = (column[:type] || value.class.name).downcase.to_sym
204
+ csv_array << ( [:hash, :array, :json, :dynamic, :object].include?( type ) ? value.to_json : value )
205
+ end
206
+ return nil if csv_array.empty?
207
+ csv_array.to_csv( :col_sep => @csv_separator )
208
+ end
209
+
210
+
211
+ def find_blob
212
+ min_blob = @active_blobs[0]
213
+ @active_blobs.each do |blob|
214
+ return blob if 0 == blob.queue_size
215
+ min_blob = blob if blob.queue_size < min_blob.queue_size
216
+ end
217
+ @active_blobs << ( min_blob = Blob.new( self, @active_blobs.length + 1 ) ) if min_blob.queue_size > 2 && @active_blobs.length < 40
218
+ min_blob
219
+ end
220
+
221
+
222
+ def set_table_properties ( configuration )
223
+ table_properties = configuration[:tables][@table_id]
224
+
225
+ if table_properties
226
+ @blob_max_delay = table_properties[:blob_max_delay]
227
+ @event_separator = table_properties[:event_separator]
228
+ @serialized_event_field = table_properties[:serialized_event_field]
229
+ @table_columns = table_properties[:table_columns]
230
+ @serialization = table_properties[:blob_serialization]
231
+ @case_insensitive_columns = table_properties[:case_insensitive_columns]
232
+ @csv_default_value = table_properties[:csv_default_value]
233
+ @csv_separator = table_properties[:csv_separator]
234
+ end
235
+ @blob_max_delay ||= configuration[:blob_max_delay]
236
+ @event_separator ||= configuration[:event_separator]
237
+ @serialized_event_field ||= configuration[:serialized_event_field]
238
+ @table_columns ||= configuration[:table_columns]
239
+ @serialization ||= configuration[:blob_serialization]
240
+ @case_insensitive_columns ||= configuration[:case_insensitive_columns]
241
+ @csv_default_value ||= configuration[:csv_default_value]
242
+ @csv_separator ||= configuration[:csv_separator]
243
+
244
+ # add field_name to each column, it is required to differentiate between the filed name and the column name
245
+ unless @table_columns.nil?
246
+ @table_columns = @table_columns.map do |column|
247
+ new_column = column.dup
248
+ new_column[:field_name] = ( @case_insensitive_columns ? new_column[:name].downcase : new_column[:name] )
249
+ new_column
250
+ end
251
+ end
252
+
253
+ # in the future, when compression is introduced, the serialization may be different from the extension
254
+ @event_format_ext = @serialization
255
+
256
+ end
257
+
258
+ end
259
+ end
@@ -0,0 +1,142 @@
1
+ # encoding: utf-8
2
+
3
+ # ----------------------------------------------------------------------------------
4
+ # Logstash Output Application Insights
5
+ #
6
+ # Copyright (c) Microsoft Corporation
7
+ #
8
+ # All rights reserved.
9
+ #
10
+ # Licensed under the Apache License, Version 2.0 (the License);
11
+ # you may not use this file except in compliance with the License.
12
+ # You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS,
16
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ #
18
+ # See the Apache Version 2.0 License for specific language governing
19
+ # permissions and limitations under the License.
20
+ # ----------------------------------------------------------------------------------
21
+
22
+ class LogStash::Outputs::Application_insights
23
+ class Channels
24
+
25
+ public
26
+
27
+ def initialize
28
+ configuration = Config.current
29
+
30
+ @logger = configuration[:logger]
31
+
32
+ @intrumentation_key_table_id_db = {}
33
+ @channels = [ ]
34
+ @create_semaphore = Mutex.new
35
+
36
+ @default_intrumentation_key = configuration[:intrumentation_key]
37
+ @default_table_id = configuration[:table_id]
38
+ @tables = configuration[:tables]
39
+
40
+ @flow_control = Flow_control.instance
41
+
42
+ # launch tread that forward events from channels to azure storage
43
+ periodic_forward_events
44
+ end
45
+
46
+
47
+ def receive ( event, encoded_event )
48
+ if LogStash::SHUTDOWN == event
49
+ @logger.info { "received a LogStash::SHUTDOWN event, start shutdown" }
50
+
51
+ elsif LogStash::FLUSH == event
52
+ @logger.info { "received a LogStash::FLUSH event, start shutdown" }
53
+ end
54
+
55
+ table_id = event[METADATA_FIELD_TABLE_ID] || event[FIELD_TABLE_ID] || @default_table_id
56
+ intrumentation_key = event[METADATA_FIELD_INSTRUMENTATION_KEY] || event[FIELD_INSTRUMENTATION_KEY] || ( @tables[table_id][TABLE_PROPERTY_INSTRUMENTATION_KEY] if @tables[table_id] ) || @default_intrumentation_key
57
+
58
+ @flow_control.pass_or_wait
59
+ channel( intrumentation_key, table_id ) << event
60
+ end
61
+
62
+
63
+ def channel ( intrumentation_key, table_id )
64
+ begin
65
+ dispatch_channel( intrumentation_key, table_id )
66
+
67
+ rescue NoChannelError
68
+ begin
69
+ create_channel( intrumentation_key, table_id )
70
+ rescue ChannelExistError # can happen due to race conditions
71
+ dispatch_channel( intrumentation_key, table_id )
72
+ end
73
+ end
74
+ end
75
+
76
+
77
+ def periodic_forward_events
78
+ Thread.new do
79
+ loop do
80
+ sleep( 0.5 )
81
+ channels = @create_semaphore.synchronize { @channels.dup }
82
+ channels.each do |channel|
83
+ channel.flush
84
+ end
85
+ end
86
+ end
87
+ end
88
+
89
+ private
90
+
91
+ # return channel
92
+ def dispatch_channel ( intrumentation_key, table_id )
93
+ begin
94
+ channel = @intrumentation_key_table_id_db[intrumentation_key][table_id]
95
+ channel.intrumentation_key # don't remove it, it is to emit an exception in case channel not created yet'
96
+ channel
97
+ rescue => e
98
+ raise NoChannelError if @intrumentation_key_table_id_db[intrumentation_key].nil? || @intrumentation_key_table_id_db[intrumentation_key][table_id].nil?
99
+ @logger.error { "Channel dispatch failed - error: #{e.inspect}" }
100
+ raise e
101
+ end
102
+ end
103
+
104
+
105
+ # return channel
106
+ def create_channel ( intrumentation_key, table_id )
107
+ @create_semaphore.synchronize {
108
+ raise ChannelExistError if @intrumentation_key_table_id_db[intrumentation_key] && @intrumentation_key_table_id_db[intrumentation_key][table_id]
109
+ @intrumentation_key_table_id_db[intrumentation_key] ||= {}
110
+ channel = Channel.new( intrumentation_key, table_id )
111
+ @intrumentation_key_table_id_db[intrumentation_key][table_id] = channel
112
+ @channels << channel
113
+ channel
114
+ }
115
+ end
116
+
117
+ public
118
+
119
+ def close
120
+ @channels.each do |channel|
121
+ channel.close
122
+ end
123
+ end
124
+
125
+ def mark_invalid_intrumentation_key ( intrumentation_key )
126
+ # TODO should go to lost and found container
127
+ end
128
+
129
+ def mark_invalid_table_id ( table_id )
130
+ # TODO should go to lost and found container
131
+ end
132
+
133
+ public
134
+
135
+ @@instance = Channels.new
136
+ def self.instance
137
+ @@instance
138
+ end
139
+
140
+ private_class_method :new
141
+ end
142
+ end