logstash-input-cloudwatch_logs 0.9.4 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 339654ac7148de2451f7af090306b642adf378c4
-  data.tar.gz: 97ebff4e4493b1d5b329befc15dc24c48f84e32f
+  metadata.gz: 0792d0166cdc9a56dad01df68c80c1b5328f437c
+  data.tar.gz: 61cadbd40105d2618df7e4dbf363874d3a37ee44
 SHA512:
-  metadata.gz: 4534bd8fa2f9d75ae3b8fbf4b1bceefd4e0d90a276934ed38c36b201fe91d9b3caaf41fcbd518f068802ca93564e03bf9efba148c2cc3fc59ad41c0433ba8fcd
-  data.tar.gz: 37e8b58c2203be326587abb04739057f81f84d2c6e9059696b902d8088a0211904ab566a796f14c142f6eef8a319441e32a696136ff54b4f37d9452b8fe43cf1
+  metadata.gz: cafe9292164f2d45606933163159af7157a78346253699d2e8ebe1614390eeeb1db23af44c124e6ced5f530b377a3f276d9e74ec8707bd89146e4684020acdf7
+  data.tar.gz: 9c602ce2aa6436a0f47b6652124ab021a5e5a185c5a8dd235c36a611c311672ab5d1760f11e7292a6f94eaf1f10ea6c99d290b81eefc9c2a511735b3e83548c3

data/lib/logstash/inputs/cloudwatch_logs.rb

@@ -38,6 +38,10 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
   # Value is in seconds.
   config :interval, :validate => :number, :default => 60
 
+  # Decide if log_group is a prefix or an absolute name
+  config :log_group_prefix, :validate => :boolean, :default => false
+
+
   # def register
   public
   def register
@@ -62,18 +66,44 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
 
   # def list_new_streams
   public
-  def list_new_streams(token = nil, objects = [])
+  def list_new_streams()
+    if @log_group_prefix
+      log_groups = @cloudwatch.describe_log_groups(log_group_name_prefix: @log_group)
+      groups = log_groups.log_groups.map {|n| n.log_group_name}
+    else
+      groups = [@log_group]
+    end
+    objects = []
+    for log_group in groups
+      objects.concat(list_new_streams_for_log_group(log_group))
+    end
+    objects
+  end
+
+  # def list_new_streams_for_log_group
+  public
+  def list_new_streams_for_log_group(log_group, token = nil, objects = [], stepback=0)
     params = {
-        :log_group_name => @log_group,
-        :order_by => "LastEventTime",
-        :descending => false
+      :log_group_name => log_group,
+      :order_by => "LastEventTime",
+      :descending => false
     }
 
+    @logger.debug("CloudWatch Logs for log_group #{log_group}")
+
     if token != nil
       params[:next_token] = token
     end
 
-    streams = @cloudwatch.describe_log_streams(params)
+    begin
+      streams = @cloudwatch.describe_log_streams(params)
+    rescue Aws::CloudWatchLogs::Errors::ThrottlingException
+      @logger.debug("CloudWatch Logs stepping back ", :stepback => 2 ** stepback * 60)
+      sleep(2 ** stepback * 60)
+      stepback += 1
+      @logger.debug("CloudWatch Logs repeating list_new_streams again with token", :token => token)
+      return list_new_streams_for_log_group(log_group, token=token, objects=objects, stepback=stepback)
+    end
 
     objects.push(*streams.log_streams)
     if streams.next_token == nil
@@ -81,10 +111,9 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
       objects
     else
       @logger.debug("CloudWatch Logs calling list_new_streams again on token", :token => streams.next_token)
-      list_new_streams(streams.next_token, objects)
+      list_new_streams_for_log_group(log_group, streams.next_token, objects)
     end
-
-  end # def list_new_streams
+  end # def list_new_streams_for_log_group
 
   # def process_log
   private
@@ -92,9 +121,9 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
 
     @codec.decode(log.message.to_str) do |event|
       event.set("@timestamp", parse_time(log.timestamp))
-      event.set("[cloudwatch][ingestion_time]", parse_time(log.ingestion_time))
-      event.set("[cloudwatch][log_group]", @log_group)
-      event.set("[cloudwatch][log_stream]", stream.log_stream_name)
+      event["[cloudwatch][ingestion_time]"] = parse_time(log.ingestion_time)
+      event["[cloudwatch][log_group]"] = stream.arn.split(/:/)[6]
+      event["[cloudwatch][log_stream]"] = stream.log_stream_name
       decorate(event)
 
       queue << event
@@ -131,17 +160,17 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
 
   # def process_log_stream
   private
-  def process_log_stream(queue, stream, last_read, current_window, token = nil)
+  def process_log_stream(queue, stream, last_read, current_window, token = nil, stepback=0)
     @logger.debug("CloudWatch Logs processing stream",
                   :log_stream => stream.log_stream_name,
-                  :log_group => @log_group,
+                  :log_group => stream.arn.split(":")[6],
                   :lastRead => last_read,
                   :currentWindow => current_window,
                   :token => token
     )
 
     params = {
-        :log_group_name => @log_group,
+        :log_group_name => stream.arn.split(":")[6],
         :log_stream_name => stream.log_stream_name,
         :start_from_head => true
     }
@@ -150,7 +179,16 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
       params[:next_token] = token
     end
 
-    logs = @cloudwatch.get_log_events(params)
+
+    begin
+      logs = @cloudwatch.get_log_events(params)
+    rescue Aws::CloudWatchLogs::Errors::ThrottlingException
+      @logger.debug("CloudWatch Logs stepping back ", :stepback => 2 ** stepback * 60)
+      sleep(2 ** stepback * 60)
+      stepback += 1
+      @logger.debug("CloudWatch Logs repeating process_log_stream again with token", :token => token)
+      return process_log_stream(queue, stream, last_read, current_window, token, stepback)
+    end
 
     logs.events.each do |log|
       if log.ingestion_time > last_read

data/logstash-input-cloudwatch_logs.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-cloudwatch_logs'
-  s.version         = '0.9.4'
+  s.version         = '0.10.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'Stream events from CloudWatch Logs.'
   s.description     = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-cloudwatch_logs
 version: !ruby/object:Gem::Version
-  version: 0.9.4
+  version: 0.10.0
 platform: ruby
 authors:
 - Luke Waite
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-31 00:00:00.000000000 Z
+date: 2017-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement