logstash-input-cloudwatch_logs 0.9.4 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 339654ac7148de2451f7af090306b642adf378c4
4
- data.tar.gz: 97ebff4e4493b1d5b329befc15dc24c48f84e32f
3
+ metadata.gz: 0792d0166cdc9a56dad01df68c80c1b5328f437c
4
+ data.tar.gz: 61cadbd40105d2618df7e4dbf363874d3a37ee44
5
5
  SHA512:
6
- metadata.gz: 4534bd8fa2f9d75ae3b8fbf4b1bceefd4e0d90a276934ed38c36b201fe91d9b3caaf41fcbd518f068802ca93564e03bf9efba148c2cc3fc59ad41c0433ba8fcd
7
- data.tar.gz: 37e8b58c2203be326587abb04739057f81f84d2c6e9059696b902d8088a0211904ab566a796f14c142f6eef8a319441e32a696136ff54b4f37d9452b8fe43cf1
6
+ metadata.gz: cafe9292164f2d45606933163159af7157a78346253699d2e8ebe1614390eeeb1db23af44c124e6ced5f530b377a3f276d9e74ec8707bd89146e4684020acdf7
7
+ data.tar.gz: 9c602ce2aa6436a0f47b6652124ab021a5e5a185c5a8dd235c36a611c311672ab5d1760f11e7292a6f94eaf1f10ea6c99d290b81eefc9c2a511735b3e83548c3
@@ -38,6 +38,10 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
38
38
  # Value is in seconds.
39
39
  config :interval, :validate => :number, :default => 60
40
40
 
41
+ # Decide if log_group is a prefix or an absolute name
42
+ config :log_group_prefix, :validate => :boolean, :default => false
43
+
44
+
41
45
  # def register
42
46
  public
43
47
  def register
@@ -62,18 +66,44 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
62
66
 
63
67
  # def list_new_streams
64
68
  public
65
- def list_new_streams(token = nil, objects = [])
69
+ def list_new_streams()
70
+ if @log_group_prefix
71
+ log_groups = @cloudwatch.describe_log_groups(log_group_name_prefix: @log_group)
72
+ groups = log_groups.log_groups.map {|n| n.log_group_name}
73
+ else
74
+ groups = [@log_group]
75
+ end
76
+ objects = []
77
+ for log_group in groups
78
+ objects.concat(list_new_streams_for_log_group(log_group))
79
+ end
80
+ objects
81
+ end
82
+
83
+ # def list_new_streams_for_log_group
84
+ public
85
+ def list_new_streams_for_log_group(log_group, token = nil, objects = [], stepback=0)
66
86
  params = {
67
- :log_group_name => @log_group,
68
- :order_by => "LastEventTime",
69
- :descending => false
87
+ :log_group_name => log_group,
88
+ :order_by => "LastEventTime",
89
+ :descending => false
70
90
  }
71
91
 
92
+ @logger.debug("CloudWatch Logs for log_group #{log_group}")
93
+
72
94
  if token != nil
73
95
  params[:next_token] = token
74
96
  end
75
97
 
76
- streams = @cloudwatch.describe_log_streams(params)
98
+ begin
99
+ streams = @cloudwatch.describe_log_streams(params)
100
+ rescue Aws::CloudWatchLogs::Errors::ThrottlingException
101
+ @logger.debug("CloudWatch Logs stepping back ", :stepback => 2 ** stepback * 60)
102
+ sleep(2 ** stepback * 60)
103
+ stepback += 1
104
+ @logger.debug("CloudWatch Logs repeating list_new_streams again with token", :token => token)
105
+ return list_new_streams_for_log_group(log_group, token=token, objects=objects, stepback=stepback)
106
+ end
77
107
 
78
108
  objects.push(*streams.log_streams)
79
109
  if streams.next_token == nil
@@ -81,10 +111,9 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
81
111
  objects
82
112
  else
83
113
  @logger.debug("CloudWatch Logs calling list_new_streams again on token", :token => streams.next_token)
84
- list_new_streams(streams.next_token, objects)
114
+ list_new_streams_for_log_group(log_group, streams.next_token, objects)
85
115
  end
86
-
87
- end # def list_new_streams
116
+ end # def list_new_streams_for_log_group
88
117
 
89
118
  # def process_log
90
119
  private
@@ -92,9 +121,9 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
92
121
 
93
122
  @codec.decode(log.message.to_str) do |event|
94
123
  event.set("@timestamp", parse_time(log.timestamp))
95
- event.set("[cloudwatch][ingestion_time]", parse_time(log.ingestion_time))
96
- event.set("[cloudwatch][log_group]", @log_group)
97
- event.set("[cloudwatch][log_stream]", stream.log_stream_name)
124
+ event["[cloudwatch][ingestion_time]"] = parse_time(log.ingestion_time)
125
+ event["[cloudwatch][log_group]"] = stream.arn.split(/:/)[6]
126
+ event["[cloudwatch][log_stream]"] = stream.log_stream_name
98
127
  decorate(event)
99
128
 
100
129
  queue << event
@@ -131,17 +160,17 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
131
160
 
132
161
  # def process_log_stream
133
162
  private
134
- def process_log_stream(queue, stream, last_read, current_window, token = nil)
163
+ def process_log_stream(queue, stream, last_read, current_window, token = nil, stepback=0)
135
164
  @logger.debug("CloudWatch Logs processing stream",
136
165
  :log_stream => stream.log_stream_name,
137
- :log_group => @log_group,
166
+ :log_group => stream.arn.split(":")[6],
138
167
  :lastRead => last_read,
139
168
  :currentWindow => current_window,
140
169
  :token => token
141
170
  )
142
171
 
143
172
  params = {
144
- :log_group_name => @log_group,
173
+ :log_group_name => stream.arn.split(":")[6],
145
174
  :log_stream_name => stream.log_stream_name,
146
175
  :start_from_head => true
147
176
  }
@@ -150,7 +179,16 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
150
179
  params[:next_token] = token
151
180
  end
152
181
 
153
- logs = @cloudwatch.get_log_events(params)
182
+
183
+ begin
184
+ logs = @cloudwatch.get_log_events(params)
185
+ rescue Aws::CloudWatchLogs::Errors::ThrottlingException
186
+ @logger.debug("CloudWatch Logs stepping back ", :stepback => 2 ** stepback * 60)
187
+ sleep(2 ** stepback * 60)
188
+ stepback += 1
189
+ @logger.debug("CloudWatch Logs repeating process_log_stream again with token", :token => token)
190
+ return process_log_stream(queue, stream, last_read, current_window, token, stepback)
191
+ end
154
192
 
155
193
  logs.events.each do |log|
156
194
  if log.ingestion_time > last_read
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-cloudwatch_logs'
4
- s.version = '0.9.4'
4
+ s.version = '0.10.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = 'Stream events from CloudWatch Logs.'
7
7
  s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-cloudwatch_logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.4
4
+ version: 0.10.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Luke Waite
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-03-31 00:00:00.000000000 Z
11
+ date: 2017-04-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement