logstash-input-cloudwatch_logs 0.9.4 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash/inputs/cloudwatch_logs.rb +53 -15
- data/logstash-input-cloudwatch_logs.gemspec +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0792d0166cdc9a56dad01df68c80c1b5328f437c
|
4
|
+
data.tar.gz: 61cadbd40105d2618df7e4dbf363874d3a37ee44
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cafe9292164f2d45606933163159af7157a78346253699d2e8ebe1614390eeeb1db23af44c124e6ced5f530b377a3f276d9e74ec8707bd89146e4684020acdf7
|
7
|
+
data.tar.gz: 9c602ce2aa6436a0f47b6652124ab021a5e5a185c5a8dd235c36a611c311672ab5d1760f11e7292a6f94eaf1f10ea6c99d290b81eefc9c2a511735b3e83548c3
|
@@ -38,6 +38,10 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
|
|
38
38
|
# Value is in seconds.
|
39
39
|
config :interval, :validate => :number, :default => 60
|
40
40
|
|
41
|
+
# Decide if log_group is a prefix or an absolute name
|
42
|
+
config :log_group_prefix, :validate => :boolean, :default => false
|
43
|
+
|
44
|
+
|
41
45
|
# def register
|
42
46
|
public
|
43
47
|
def register
|
@@ -62,18 +66,44 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
|
|
62
66
|
|
63
67
|
# def list_new_streams
|
64
68
|
public
|
65
|
-
def list_new_streams(
|
69
|
+
def list_new_streams()
|
70
|
+
if @log_group_prefix
|
71
|
+
log_groups = @cloudwatch.describe_log_groups(log_group_name_prefix: @log_group)
|
72
|
+
groups = log_groups.log_groups.map {|n| n.log_group_name}
|
73
|
+
else
|
74
|
+
groups = [@log_group]
|
75
|
+
end
|
76
|
+
objects = []
|
77
|
+
for log_group in groups
|
78
|
+
objects.concat(list_new_streams_for_log_group(log_group))
|
79
|
+
end
|
80
|
+
objects
|
81
|
+
end
|
82
|
+
|
83
|
+
# def list_new_streams_for_log_group
|
84
|
+
public
|
85
|
+
def list_new_streams_for_log_group(log_group, token = nil, objects = [], stepback=0)
|
66
86
|
params = {
|
67
|
-
|
68
|
-
|
69
|
-
|
87
|
+
:log_group_name => log_group,
|
88
|
+
:order_by => "LastEventTime",
|
89
|
+
:descending => false
|
70
90
|
}
|
71
91
|
|
92
|
+
@logger.debug("CloudWatch Logs for log_group #{log_group}")
|
93
|
+
|
72
94
|
if token != nil
|
73
95
|
params[:next_token] = token
|
74
96
|
end
|
75
97
|
|
76
|
-
|
98
|
+
begin
|
99
|
+
streams = @cloudwatch.describe_log_streams(params)
|
100
|
+
rescue Aws::CloudWatchLogs::Errors::ThrottlingException
|
101
|
+
@logger.debug("CloudWatch Logs stepping back ", :stepback => 2 ** stepback * 60)
|
102
|
+
sleep(2 ** stepback * 60)
|
103
|
+
stepback += 1
|
104
|
+
@logger.debug("CloudWatch Logs repeating list_new_streams again with token", :token => token)
|
105
|
+
return list_new_streams_for_log_group(log_group, token=token, objects=objects, stepback=stepback)
|
106
|
+
end
|
77
107
|
|
78
108
|
objects.push(*streams.log_streams)
|
79
109
|
if streams.next_token == nil
|
@@ -81,10 +111,9 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
|
|
81
111
|
objects
|
82
112
|
else
|
83
113
|
@logger.debug("CloudWatch Logs calling list_new_streams again on token", :token => streams.next_token)
|
84
|
-
|
114
|
+
list_new_streams_for_log_group(log_group, streams.next_token, objects)
|
85
115
|
end
|
86
|
-
|
87
|
-
end # def list_new_streams
|
116
|
+
end # def list_new_streams_for_log_group
|
88
117
|
|
89
118
|
# def process_log
|
90
119
|
private
|
@@ -92,9 +121,9 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
|
|
92
121
|
|
93
122
|
@codec.decode(log.message.to_str) do |event|
|
94
123
|
event.set("@timestamp", parse_time(log.timestamp))
|
95
|
-
event
|
96
|
-
event
|
97
|
-
event
|
124
|
+
event["[cloudwatch][ingestion_time]"] = parse_time(log.ingestion_time)
|
125
|
+
event["[cloudwatch][log_group]"] = stream.arn.split(/:/)[6]
|
126
|
+
event["[cloudwatch][log_stream]"] = stream.log_stream_name
|
98
127
|
decorate(event)
|
99
128
|
|
100
129
|
queue << event
|
@@ -131,17 +160,17 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
|
|
131
160
|
|
132
161
|
# def process_log_stream
|
133
162
|
private
|
134
|
-
def process_log_stream(queue, stream, last_read, current_window, token = nil)
|
163
|
+
def process_log_stream(queue, stream, last_read, current_window, token = nil, stepback=0)
|
135
164
|
@logger.debug("CloudWatch Logs processing stream",
|
136
165
|
:log_stream => stream.log_stream_name,
|
137
|
-
:log_group =>
|
166
|
+
:log_group => stream.arn.split(":")[6],
|
138
167
|
:lastRead => last_read,
|
139
168
|
:currentWindow => current_window,
|
140
169
|
:token => token
|
141
170
|
)
|
142
171
|
|
143
172
|
params = {
|
144
|
-
:log_group_name =>
|
173
|
+
:log_group_name => stream.arn.split(":")[6],
|
145
174
|
:log_stream_name => stream.log_stream_name,
|
146
175
|
:start_from_head => true
|
147
176
|
}
|
@@ -150,7 +179,16 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
|
|
150
179
|
params[:next_token] = token
|
151
180
|
end
|
152
181
|
|
153
|
-
|
182
|
+
|
183
|
+
begin
|
184
|
+
logs = @cloudwatch.get_log_events(params)
|
185
|
+
rescue Aws::CloudWatchLogs::Errors::ThrottlingException
|
186
|
+
@logger.debug("CloudWatch Logs stepping back ", :stepback => 2 ** stepback * 60)
|
187
|
+
sleep(2 ** stepback * 60)
|
188
|
+
stepback += 1
|
189
|
+
@logger.debug("CloudWatch Logs repeating process_log_stream again with token", :token => token)
|
190
|
+
return process_log_stream(queue, stream, last_read, current_window, token, stepback)
|
191
|
+
end
|
154
192
|
|
155
193
|
logs.events.each do |log|
|
156
194
|
if log.ingestion_time > last_read
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-cloudwatch_logs'
|
4
|
-
s.version = '0.
|
4
|
+
s.version = '0.10.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = 'Stream events from CloudWatch Logs.'
|
7
7
|
s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-cloudwatch_logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.10.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Luke Waite
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-04-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|