logstash-output-awslogs 0.1.7 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash/outputs/awslogs.rb +64 -55
- data/logstash-output-awslogs.gemspec +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 777dc69291b2c3a307405c13583b5d3a2de1b2b9
|
|
4
|
+
data.tar.gz: 5829780c1044b8ed0461a6159cfaa1b3d3b987e9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4502d60ae7e11290500e1c4ef7ef5557b819bfc9a494f0ec31fdc17b147102cfc90a6633566df3c3fcfb761db0b583b29a667674583d9b1476f1d46abcddc780
|
|
7
|
+
data.tar.gz: 736b23303a5673eded828ff4a70bde9eaa3191bea43e7dda6d505f357325baaac6bb05b5db38c4f148a54bf2dc48fd8f24f18080b530feb25d65458287cac6db
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
3
|
-
require
|
|
4
|
-
require
|
|
5
|
-
require
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'logstash/outputs/base'
|
|
4
|
+
require 'logstash/namespace'
|
|
5
|
+
require 'logstash/plugin_mixins/aws_config'
|
|
6
|
+
require 'aws-sdk'
|
|
6
7
|
|
|
7
8
|
Aws.eager_autoload!
|
|
8
9
|
|
|
@@ -10,72 +11,74 @@ Aws.eager_autoload!
|
|
|
10
11
|
class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
|
|
11
12
|
include LogStash::PluginMixins::AwsConfig::V2
|
|
12
13
|
|
|
13
|
-
config_name
|
|
14
|
-
default :codec,
|
|
14
|
+
config_name 'awslogs'
|
|
15
|
+
default :codec, 'line'
|
|
15
16
|
|
|
16
|
-
config :log_group_name, :
|
|
17
|
-
config :log_stream_name, :
|
|
17
|
+
config :log_group_name, validate: :string, required: true
|
|
18
|
+
config :log_stream_name, validate: :string, required: true
|
|
18
19
|
|
|
19
20
|
public
|
|
21
|
+
|
|
20
22
|
def register
|
|
21
23
|
@client = Aws::CloudWatchLogs::Client.new(aws_options_hash)
|
|
22
24
|
@next_sequence_tokens = {}
|
|
23
25
|
end # def register
|
|
24
26
|
|
|
25
27
|
public
|
|
28
|
+
|
|
26
29
|
def multi_receive_encoded(events_and_encoded)
|
|
27
30
|
to_send = {}
|
|
28
31
|
sequence_tokens = {}
|
|
29
32
|
|
|
30
|
-
events_and_encoded.each do |event,
|
|
33
|
+
events_and_encoded.each do |event, _encoded|
|
|
31
34
|
event_log_stream_name = event.sprintf(log_stream_name)
|
|
32
35
|
event_log_group_name = event.sprintf(log_group_name)
|
|
33
36
|
|
|
34
37
|
next_sequence_token_key = [event_log_group_name, event_log_stream_name]
|
|
35
|
-
|
|
38
|
+
unless to_send.keys.include? next_sequence_token_key
|
|
36
39
|
to_send.store(next_sequence_token_key, [])
|
|
37
40
|
end
|
|
38
|
-
to_send[next_sequence_token_key].push(
|
|
41
|
+
to_send[next_sequence_token_key].push(
|
|
39
42
|
timestamp: (event.timestamp.time.to_f * 1000).to_int,
|
|
40
|
-
message: event.get(
|
|
41
|
-
|
|
43
|
+
message: event.get('message')
|
|
44
|
+
)
|
|
42
45
|
end
|
|
43
46
|
|
|
44
47
|
to_send.each do |event_log_names, _events|
|
|
45
48
|
event_log_group_name = event_log_names[0]
|
|
46
49
|
event_log_stream_name = event_log_names[1]
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => e
|
|
55
|
-
@logger.info("Will create log group/stream and retry")
|
|
56
|
-
begin
|
|
57
|
-
@client.create_log_group(:log_group_name => send_opts[:log_group_name])
|
|
58
|
-
rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException => e
|
|
59
|
-
@logger.info("Log group #{send_opts[:log_group_name]} already exists")
|
|
60
|
-
rescue Exception => e
|
|
61
|
-
@logger.error(e)
|
|
62
|
-
end
|
|
63
|
-
begin
|
|
64
|
-
@client.create_log_stream(:log_group_name => send_opts[:log_group_name], :log_stream_name => send_opts[:log_stream_name])
|
|
65
|
-
rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException => e
|
|
66
|
-
@logger.info("Log stream #{send_opts[:log_stream_name]} already exists")
|
|
67
|
-
rescue Exception => e
|
|
68
|
-
@logger.error(e)
|
|
50
|
+
next if sequence_tokens.keys.include? event_log_group_name
|
|
51
|
+
|
|
52
|
+
sequence_tokens.store(event_log_group_name, {})
|
|
53
|
+
begin
|
|
54
|
+
@client.describe_log_streams(log_group_name: event_log_group_name).each do |response|
|
|
55
|
+
response.log_streams.each do |log_stream_data|
|
|
56
|
+
sequence_tokens[event_log_group_name][log_stream_data.log_stream_name.to_s] = log_stream_data.upload_sequence_token.to_s
|
|
69
57
|
end
|
|
70
|
-
retry
|
|
71
|
-
rescue Aws::CloudWatchLogs::Errors::ThrottlingException => e
|
|
72
|
-
@logger.info("Logs throttling, retry")
|
|
73
|
-
retry
|
|
74
58
|
end
|
|
59
|
+
rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => e
|
|
60
|
+
@logger.info('Will create log group/stream and retry')
|
|
61
|
+
begin
|
|
62
|
+
@client.create_log_group(log_group_name: event_log_group_name)
|
|
63
|
+
rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException => e
|
|
64
|
+
@logger.info("Log group #{event_log_group_name} already exists")
|
|
65
|
+
rescue Exception => e
|
|
66
|
+
@logger.error(e)
|
|
67
|
+
end
|
|
68
|
+
begin
|
|
69
|
+
@client.create_log_stream(log_group_name: event_log_group_name, log_stream_name: event_log_stream_name)
|
|
70
|
+
rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException => e
|
|
71
|
+
@logger.info("Log stream #{event_log_stream_name} already exists")
|
|
72
|
+
rescue Exception => e
|
|
73
|
+
@logger.error(e)
|
|
74
|
+
end
|
|
75
|
+
retry
|
|
76
|
+
rescue Aws::CloudWatchLogs::Errors::ThrottlingException => e
|
|
77
|
+
@logger.info('Logs throttling, retry')
|
|
78
|
+
retry
|
|
75
79
|
end
|
|
76
80
|
end
|
|
77
81
|
|
|
78
|
-
|
|
79
82
|
to_send.each do |event_log_names, log_events|
|
|
80
83
|
event_log_group_name = event_log_names[0]
|
|
81
84
|
event_log_stream_name = event_log_names[1]
|
|
@@ -83,11 +86,11 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
|
|
|
83
86
|
|
|
84
87
|
ident_opts = {
|
|
85
88
|
log_group_name: event_log_group_name,
|
|
86
|
-
log_stream_name: event_log_stream_name
|
|
89
|
+
log_stream_name: event_log_stream_name
|
|
87
90
|
}
|
|
88
|
-
send_opts = ident_opts.merge(
|
|
89
|
-
log_events: log_events
|
|
90
|
-
|
|
91
|
+
send_opts = ident_opts.merge(
|
|
92
|
+
log_events: log_events
|
|
93
|
+
)
|
|
91
94
|
|
|
92
95
|
if @next_sequence_tokens.keys.include? next_sequence_token_key
|
|
93
96
|
send_opts[:sequence_token] = @next_sequence_tokens[next_sequence_token_key]
|
|
@@ -95,18 +98,24 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
|
|
|
95
98
|
send_opts[:sequence_token] = sequence_tokens[event_log_group_name][event_log_stream_name]
|
|
96
99
|
else
|
|
97
100
|
begin
|
|
98
|
-
|
|
101
|
+
@client.create_log_stream(ident_opts)
|
|
102
|
+
@client.describe_log_streams(log_group_name: event_log_group_name).each do |response|
|
|
103
|
+
response.log_streams.each do |log_stream_data|
|
|
104
|
+
sequence_tokens[event_log_group_name][log_stream_data.log_stream_name.to_s] = log_stream_data.upload_sequence_token.to_s
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
send_opts[:sequence_token] = sequence_tokens[event_log_group_name][event_log_stream_name]
|
|
99
108
|
rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => e
|
|
100
|
-
@logger.info(
|
|
109
|
+
@logger.info('Will create log group/stream and retry')
|
|
101
110
|
begin
|
|
102
|
-
@client.create_log_group(:
|
|
111
|
+
@client.create_log_group(log_group_name: send_opts[:log_group_name])
|
|
103
112
|
rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException => e
|
|
104
113
|
@logger.info("Log group #{send_opts[:log_group_name]} already exists")
|
|
105
114
|
rescue Exception => e
|
|
106
115
|
@logger.error(e)
|
|
107
116
|
end
|
|
108
117
|
begin
|
|
109
|
-
@client.create_log_stream(:
|
|
118
|
+
@client.create_log_stream(log_group_name: send_opts[:log_group_name], log_stream_name: send_opts[:log_stream_name])
|
|
110
119
|
rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException => e
|
|
111
120
|
@logger.info("Log stream #{send_opts[:log_stream_name]} already exists")
|
|
112
121
|
rescue Exception => e
|
|
@@ -114,23 +123,23 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
|
|
|
114
123
|
end
|
|
115
124
|
retry
|
|
116
125
|
rescue Aws::CloudWatchLogs::Errors::ThrottlingException => e
|
|
117
|
-
@logger.info(
|
|
126
|
+
@logger.info('Logs throttling, retry')
|
|
118
127
|
retry
|
|
119
128
|
end
|
|
120
129
|
end
|
|
121
130
|
begin
|
|
122
131
|
resp = @client.put_log_events(send_opts)
|
|
123
132
|
rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException => e
|
|
124
|
-
@logger.info(
|
|
133
|
+
@logger.info('Will create log group/stream and retry')
|
|
125
134
|
begin
|
|
126
|
-
@client.create_log_group(:
|
|
135
|
+
@client.create_log_group(log_group_name: send_opts[:log_group_name])
|
|
127
136
|
rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException => e
|
|
128
137
|
@logger.info("Log group #{send_opts[:log_group_name]} already exists")
|
|
129
138
|
rescue Exception => e
|
|
130
139
|
@logger.error(e)
|
|
131
140
|
end
|
|
132
141
|
begin
|
|
133
|
-
@client.create_log_stream(:
|
|
142
|
+
@client.create_log_stream(log_group_name: send_opts[:log_group_name], log_stream_name: send_opts[:log_stream_name])
|
|
134
143
|
rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException => e
|
|
135
144
|
@logger.info("Log stream #{send_opts[:log_stream_name]} already exists")
|
|
136
145
|
rescue Exception => e
|
|
@@ -138,11 +147,11 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
|
|
|
138
147
|
end
|
|
139
148
|
retry
|
|
140
149
|
# TODO: handle rejected events with debug message
|
|
141
|
-
@next_sequence_tokens.store(next_sequence_token_key, resp.next_sequence_token)
|
|
142
150
|
rescue Aws::CloudWatchLogs::Errors::ThrottlingException => e
|
|
143
|
-
@logger.info(
|
|
151
|
+
@logger.info('Logs throttling, retry')
|
|
144
152
|
retry
|
|
145
153
|
end
|
|
154
|
+
@next_sequence_tokens.store(next_sequence_token_key, resp.next_sequence_token)
|
|
146
155
|
end
|
|
147
156
|
end # def multi_receive_encoded
|
|
148
157
|
end # class LogStash::Outputs::Awslogs
|