logstash-output-awslogs 0.1.3 → 0.1.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/logstash/outputs/awslogs.rb +48 -15
- data/logstash-output-awslogs.gemspec +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: cf9d3bfff7bf14387d23842f9b02f53c3ab558ca
|
|
4
|
+
data.tar.gz: 13d1412afb9220821f2b22964706075a4fe4a437
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 60cb4ae5ac77ad1c56ef81133659cf2b7d5046d96b78ba0f036ba048aa558e82ae46ac2fffecdec437a7bbe04a9258adff82be795a6346d4ba0bc5febd621c1f
|
|
7
|
+
data.tar.gz: f7021983778d560315b7c6ec8397daa406861fe0b0393b44d104b2e103f41e187037eb990e0adf969de0b4a48fa2c6c1c4b5b6d171af15a15f877a4916a7f2e3
|
|
@@ -25,6 +25,7 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
|
|
|
25
25
|
public
|
|
26
26
|
def multi_receive_encoded(events_and_encoded)
|
|
27
27
|
to_send = {}
|
|
28
|
+
sequence_tokens = {}
|
|
28
29
|
|
|
29
30
|
events_and_encoded.each do |event, encoded|
|
|
30
31
|
event_log_stream_name = event.sprintf(log_stream_name)
|
|
@@ -36,9 +37,46 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
|
|
|
36
37
|
end
|
|
37
38
|
to_send[next_sequence_token_key].push({
|
|
38
39
|
timestamp: (event.timestamp.time.to_f * 1000).to_int,
|
|
39
|
-
message: event.get(
|
|
40
|
+
message: event.get("message"),
|
|
40
41
|
})
|
|
41
42
|
end
|
|
43
|
+
|
|
44
|
+
to_send.each do |event_log_names, _events|
|
|
45
|
+
event_log_group_name = event_log_names[0]
|
|
46
|
+
event_log_stream_name = event_log_names[1]
|
|
47
|
+
unless sequence_tokens.keys.include? event_log_group_name
|
|
48
|
+
sequence_tokens.store(event_log_group_name, [])
|
|
49
|
+
begin
|
|
50
|
+
@client.describe_log_streams({log_group_name: event_log_group_name}).each do |response|
|
|
51
|
+
response.log_streams.each do |log_stream_data|
|
|
52
|
+
sequence_tokens[event_log_group_name].push({ "#{log_stream_data.log_stream_name}": "#{log_stream_data.upload_sequence_token}"})
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => e
|
|
56
|
+
@logger.info("Will create log group/stream and retry")
|
|
57
|
+
begin
|
|
58
|
+
@client.create_log_group(:log_group_name => send_opts[:log_group_name])
|
|
59
|
+
rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException => e
|
|
60
|
+
@logger.info("Log group #{send_opts[:log_group_name]} already exists")
|
|
61
|
+
rescue Exception => e
|
|
62
|
+
@logger.error(e)
|
|
63
|
+
end
|
|
64
|
+
begin
|
|
65
|
+
@client.create_log_stream(:log_group_name => send_opts[:log_group_name], :log_stream_name => send_opts[:log_stream_name])
|
|
66
|
+
rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException => e
|
|
67
|
+
@logger.info("Log stream #{send_opts[:log_stream_name]} already exists")
|
|
68
|
+
rescue Exception => e
|
|
69
|
+
@logger.error(e)
|
|
70
|
+
end
|
|
71
|
+
retry
|
|
72
|
+
rescue Aws::CloudWatchLogs::Errors::ThrottlingException => e
|
|
73
|
+
@logger.info("Logs throttling, retry")
|
|
74
|
+
retry
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
|
|
42
80
|
to_send.each do |event_log_names, log_events|
|
|
43
81
|
event_log_group_name = event_log_names[0]
|
|
44
82
|
event_log_stream_name = event_log_names[1]
|
|
@@ -54,22 +92,11 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
|
|
|
54
92
|
|
|
55
93
|
if @next_sequence_tokens.keys.include? next_sequence_token_key
|
|
56
94
|
send_opts[:sequence_token] = @next_sequence_tokens[next_sequence_token_key]
|
|
95
|
+
elsif sequence_tokens[event_log_group_name].keys.include? event_log_stream_name
|
|
96
|
+
send_opts[:sequence_token] = sequence_tokens[event_log_group_name][event_log_stream_name]
|
|
57
97
|
else
|
|
58
98
|
begin
|
|
59
|
-
resp = @client.describe_log_streams({
|
|
60
|
-
log_group_name: event_log_group_name,
|
|
61
|
-
log_stream_name_prefix: event_log_stream_name,
|
|
62
|
-
})
|
|
63
|
-
if resp.log_streams.length < 1
|
|
64
99
|
@client.create_log_stream(ident_opts)
|
|
65
|
-
else
|
|
66
|
-
resp.log_streams.each do |log_stream_data|
|
|
67
|
-
if log_stream_data.log_stream_name == event_log_stream_name
|
|
68
|
-
send_opts[:sequence_token] = log_stream_data.upload_sequence_token
|
|
69
|
-
break
|
|
70
|
-
end
|
|
71
|
-
end
|
|
72
|
-
end
|
|
73
100
|
rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => e
|
|
74
101
|
@logger.info("Will create log group/stream and retry")
|
|
75
102
|
begin
|
|
@@ -87,6 +114,9 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
|
|
|
87
114
|
@logger.error(e)
|
|
88
115
|
end
|
|
89
116
|
retry
|
|
117
|
+
rescue Aws::CloudWatchLogs::Errors::ThrottlingException => e
|
|
118
|
+
@logger.info("Logs throttling, retry")
|
|
119
|
+
retry
|
|
90
120
|
end
|
|
91
121
|
end
|
|
92
122
|
begin
|
|
@@ -109,7 +139,10 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
|
|
|
109
139
|
end
|
|
110
140
|
retry
|
|
111
141
|
# TODO: handle rejected events with debug message
|
|
112
|
-
|
|
142
|
+
@next_sequence_tokens.store(next_sequence_token_key, resp.next_sequence_token)
|
|
143
|
+
rescue Aws::CloudWatchLogs::Errors::ThrottlingException => e
|
|
144
|
+
@logger.info("Logs throttling, retry")
|
|
145
|
+
retry
|
|
113
146
|
end
|
|
114
147
|
end
|
|
115
148
|
end # def multi_receive_encoded
|