logstash-input-cloudwatch_logs 0.10.3 → 1.0.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d3cf76d4fcae5d3c2145be37be2ee196bc443e67
-  data.tar.gz: 07f81ee5d24b68ee92727b056ddcf0189035bcb8
+  metadata.gz: 0add333171d946f22e690ecd2fa6167115136761
+  data.tar.gz: 3f85bcf9b77be0aec9466e8fe014841597da164d
 SHA512:
-  metadata.gz: 2038764e97fea3b2ef55bdca11b7910491317fc2bdcd55922e85ef649fc9830ff0015196463e29c45d873ab48a538ec240a8af2561201071206ad54d30cc93cb
-  data.tar.gz: 663df20c6fb609a1a4eb4f98d7a44bdf42376b6b842df2817710dfe810e1692571fd20a30e511f5a8bff75115056e5ebb151fab729499d6a69d5505a7b241b65
+  metadata.gz: 0ddf9aba860508e42ae5ee675d687d97e123152eee16555c82e0ebf72e64ae09d7dcbaeefddd2a22c532bd17ddb009fee7b82bf7529418a8d1a2861ce5dd5f44
+  data.tar.gz: d2e00e97521353b991fb39b053d9b9d57ae46e4d07e019f775d2f81b565f337b8e886df84dd306021168bad9c8181b0e5289572c00b8653b8c47ef07bf9c8036

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release Notes for `logstash-input-cloudwatch_logs`
 
+## v1.0.0 Pre-Release (2017-06-24)
+* BREAKING CHANGE: `log_group` must now be an array, adds support for specifying multiple groups or prefixes
+* Refactored ingestion, fixes multiple memory leaks (Fixes [#24](https://github.com/lukewaite/logstash-input-cloudwatch-logs/issues/4))
+* Pull only log_events since last ingestion (Fixes [#10](https://github.com/lukewaite/logstash-input-cloudwatch-logs/issues/10))
+* Incrementally write to since_db on each page of data from the CWL API (Fixes [#4](https://github.com/lukewaite/logstash-input-cloudwatch-logs/issues/4))
+
 ## v0.10.3  (2017-05-07)
 
 ### Fixed

data/README.md

@@ -20,7 +20,7 @@ and ingest all logs available in all of the matching groups.
 ### Parameters
 | Parameter | Input Type | Required | Default |
 |-----------|------------|----------|---------|
-| log_group | string | Yes | |
+| log_group | array | Yes | |
 | log_group_prefix | boolean | No | `false` |
 | sincedb_path | string | No | `$HOME/.sincedb*` |
 | interval | number | No | 60 |
@@ -40,7 +40,7 @@ Other standard logstash parameters are available such as:
 
     input {
         cloudwatch_logs {
-            log_group => "/aws/lambda/my-lambda"
+            log_group => [ "/aws/lambda/my-lambda" ]
             access_key_id => "AKIAXXXXXX" 
             secret_access_key => "SECRET"
         }

data/lib/logstash/inputs/cloudwatch_logs.rb

@@ -4,11 +4,10 @@ require "logstash/namespace"
 require "logstash/plugin_mixins/aws_config"
 require "logstash/timestamp"
 require "time"
-require "tmpdir"
 require "stud/interval"
-require "stud/temporary"
 require "aws-sdk"
-require "logstash/inputs/cloudwatch/patch"
+require "logstash/inputs/cloudwatch_logs/patch"
+require "fileutils"
 
 Aws.eager_autoload!
 
@@ -28,9 +27,9 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
 
   default :codec, "plain"
 
-  # Log group to pull logs from for this plugin. Will pull in all
-  # streams inside of this log group.
-  config :log_group, :validate => :string, :required => true
+  # Log group(s) to use as an input. If `log_group_prefix` is set
+  # to `true`, then each member of the array is treated as a prefix
+  config :log_group, :validate => :array, :required => true
 
   # Where to write the since database (keeps track of the date
   # the last handled log stream was updated). The default will write
@@ -50,93 +49,127 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
   public
   def register
     require "digest/md5"
-
-    @logger.info("Registering cloudwatch_logs input", :log_group => @log_group)
+    @logger.trace("Registering cloudwatch_logs input", :log_group => @log_group)
+    settings = defined?(LogStash::SETTINGS) ? LogStash::SETTINGS : nil
+    @sincedb = {}
 
     Aws::ConfigService::Client.new(aws_options_hash)
-
     @cloudwatch = Aws::CloudWatchLogs::Client.new(aws_options_hash)
+
+    if @sincedb_path.nil?
+      if settings
+        datapath = File.join(settings.get_value("path.data"), "plugins", "inputs", "cloudwatch_logs")
+        # Ensure that the filepath exists before writing, since it's deeply nested.
+        FileUtils::mkdir_p datapath
+        @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest(@log_group.join(",")))
+      end
+    end
+
+    # This section is going to be deprecated eventually, as path.data will be
+    # the default, not an environment variable (SINCEDB_DIR or HOME)
+    if @sincedb_path.nil? # If it is _still_ nil...
+      if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil?
+        @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \
+                      "to keep track of the files I'm watching. Either set " \
+                      "HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \
+                      "in your Logstash config for the file input with " \
+                      "path '#{@path.inspect}'")
+        raise
+      end
+
+      #pick SINCEDB_DIR if available, otherwise use HOME
+      sincedb_dir = ENV["SINCEDB_DIR"] || ENV["HOME"]
+
+      @sincedb_path = File.join(sincedb_dir, ".sincedb_" + Digest::MD5.hexdigest(@log_group.join(",")))
+
+      @logger.info("No sincedb_path set, generating one based on the log_group setting",
+                   :sincedb_path => @sincedb_path, :log_group => @log_group)
+    end
+
   end #def register
 
   # def run
   public
   def run(queue)
-    while !stop?
-      process_group(queue)
-      Stud.stoppable_sleep(@interval)
+    @queue = queue
+    _sincedb_open
+
+    Stud.interval(@interval) do
+      groups = find_log_groups
+
+      groups.each do |group|
+        @logger.debug("calling process_group on #{group}")
+        process_group(group)
+      end # groups.each
     end
+
   end # def run
 
-  # def list_new_streams
   public
-  def list_new_streams()
+  def find_log_groups
     if @log_group_prefix
-      log_groups = @cloudwatch.describe_log_groups(log_group_name_prefix: @log_group)
-      groups = log_groups.log_groups.map {|n| n.log_group_name}
-      while log_groups.next_token
-        log_groups = @cloudwatch.describe_log_groups(log_group_name_prefix: @log_group, next_token: log_groups.next_token)
-        groups += log_groups.log_groups.map {|n| n.log_group_name}
+      @logger.debug("log_group prefix is enabled, searching for log groups")
+      groups = []
+      next_token = nil
+      @log_group.each do |group|
+        loop do
+          log_groups = @cloudwatch.describe_log_groups(log_group_name_prefix: group, next_token: next_token)
+          groups += log_groups.log_groups.map {|n| n.log_group_name}
+          next_token = log_groups.next_token
+          @logger.debug("found #{log_groups.log_groups.length} log groups matching prefix #{group}")
+          break if next_token.nil?
+        end
       end
     else
-      groups = [@log_group]
-    end
-    objects = []
-    for log_group in groups
-      objects.concat(list_new_streams_for_log_group(log_group))
+      @logger.debug("log_group_prefix not enabled")
+      groups = @log_group
     end
-    objects
-  end
-
-  # def list_new_streams_for_log_group
-  public
-  def list_new_streams_for_log_group(log_group, token = nil, objects = [], stepback=0)
-    params = {
-      :log_group_name => log_group,
-      :order_by => "LastEventTime",
-      :descending => false
-    }
-
-    @logger.debug("CloudWatch Logs for log_group #{log_group}")
+    groups
+  end # def find_log_groups
 
-    if token != nil
-      params[:next_token] = token
-    end
+  private
+  def process_group(group)
+    next_token = nil
+    loop do
+      if !@sincedb.member?(group)
+        @sincedb[group] = 0
+      end
+      params = {
+          :log_group_name => group,
+          :start_time => @sincedb[group],
+          :limit => 10,
+          :interleaved => true,
+          :next_token => next_token
+      }
+      resp = @cloudwatch.filter_log_events(params)
+
+      resp.events.each do |event|
+        process_log(event, group)
+      end
 
-    begin
-      streams = @cloudwatch.describe_log_streams(params)
-    rescue Aws::CloudWatchLogs::Errors::ThrottlingException
-      @logger.debug("CloudWatch Logs stepping back ", :stepback => 2 ** stepback * 60)
-      sleep(2 ** stepback * 60)
-      stepback += 1
-      @logger.debug("CloudWatch Logs repeating list_new_streams again with token", :token => token)
-      return list_new_streams_for_log_group(log_group, token=token, objects=objects, stepback=stepback)
-    end
+      _sincedb_write
 
-    objects.push(*streams.log_streams)
-    if streams.next_token == nil
-      @logger.debug("CloudWatch Logs hit end of tokens for streams")
-      objects
-    else
-      @logger.debug("CloudWatch Logs calling list_new_streams again on token", :token => streams.next_token)
-      list_new_streams_for_log_group(log_group, streams.next_token, objects)
+      next_token = resp.next_token
+      break if next_token.nil?
     end
-  end # def list_new_streams_for_log_group
+  end #def process_group
 
   # def process_log
   private
-  def process_log(queue, log, stream)
+  def process_log(log, group)
 
     @codec.decode(log.message.to_str) do |event|
       event.set("@timestamp", parse_time(log.timestamp))
-      event.set("[cloudwatch][ingestion_time]", parse_time(log.ingestion_time))
-      event.set("[cloudwatch][log_group]", stream.arn.split(/:/)[6])
-      event.set("[cloudwatch][log_stream]", stream.log_stream_name)
+      event.set("[cloudwatch_logs][ingestion_time]", parse_time(log.ingestion_time))
+      event.set("[cloudwatch_logs][log_group]", group)
+      event.set("[cloudwatch_logs][log_stream]", log.log_stream_name)
+      event.set("[cloudwatch_logs][event_id]", log.event_id)
       decorate(event)
 
-      queue << event
+      @queue << event
+      @sincedb[group] = log.timestamp + 1
     end
-  end
-  # def process_log
+  end # def process_log
 
   # def parse_time
   private
@@ -144,111 +177,39 @@ class LogStash::Inputs::CloudWatch_Logs < LogStash::Inputs::Base
     LogStash::Timestamp.at(data.to_i / 1000, (data.to_i % 1000) * 1000)
   end # def parse_time
 
-  # def process_group
-  public
-  def process_group(queue)
-    objects = list_new_streams
-
-    last_read = sincedb.read
-    current_window = DateTime.now.strftime('%Q')
-
-    if last_read < 0
-      last_read = 1
-    end
-
-    objects.each do |stream|
-      if stream.last_ingestion_time && stream.last_ingestion_time > last_read
-        process_log_stream(queue, stream, last_read, current_window)
-      end
-    end
-
-    sincedb.write(current_window)
-  end # def process_group
-
-  # def process_log_stream
   private
-  def process_log_stream(queue, stream, last_read, current_window, token = nil, stepback=0)
-    @logger.debug("CloudWatch Logs processing stream",
-                  :log_stream => stream.log_stream_name,
-                  :log_group => stream.arn.split(":")[6],
-                  :lastRead => last_read,
-                  :currentWindow => current_window,
-                  :token => token
-    )
-
-    params = {
-        :log_group_name => stream.arn.split(":")[6],
-        :log_stream_name => stream.log_stream_name,
-        :start_from_head => true
-    }
-
-    if token != nil
-      params[:next_token] = token
-    end
-
-
+  def _sincedb_open
     begin
-      logs = @cloudwatch.get_log_events(params)
-    rescue Aws::CloudWatchLogs::Errors::ThrottlingException
-      @logger.debug("CloudWatch Logs stepping back ", :stepback => 2 ** stepback * 60)
-      sleep(2 ** stepback * 60)
-      stepback += 1
-      @logger.debug("CloudWatch Logs repeating process_log_stream again with token", :token => token)
-      return process_log_stream(queue, stream, last_read, current_window, token, stepback)
-    end
-
-    logs.events.each do |log|
-      if log.ingestion_time > last_read
-        process_log(queue, log, stream)
+      File.open(@sincedb_path) do |db|
+        @logger.debug? && @logger.debug("_sincedb_open: reading from #{@sincedb_path}")
+        db.each do |line|
+          group, pos = line.split(" ", 2)
+          @logger.debug? && @logger.debug("_sincedb_open: setting #{group} to #{pos.to_i}")
+          @sincedb[group] = pos.to_i
+        end
       end
+    rescue
+      #No existing sincedb to load
+      @logger.debug? && @logger.debug("_sincedb_open: error: #{@sincedb_path}: #{$!}")
     end
+  end # def _sincedb_open
 
-    # if there are more pages, continue
-    if logs.events.count != 0 && logs.next_forward_token != nil
-      process_log_stream(queue, stream, last_read, current_window, logs.next_forward_token)
+  private
+  def _sincedb_write
+    begin
+      IO.write(@sincedb_path, serialize_sincedb, 0)
+    rescue Errno::EACCES
+      # probably no file handles free
+      # maybe it will work next time
+      @logger.debug? && @logger.debug("_sincedb_write: error: #{@sincedb_path}: #{$!}")
     end
-  end # def process_log_stream
+  end # def _sincedb_write
 
-  private
-  def sincedb
-    @sincedb ||= if @sincedb_path.nil?
-                   @logger.info("Using default generated file for the sincedb", :filename => sincedb_file)
-                   SinceDB::File.new(sincedb_file)
-                 else
-                   @logger.info("Using the provided sincedb_path",
-                                :sincedb_path => @sincedb_path)
-                   SinceDB::File.new(@sincedb_path)
-                 end
-  end
 
   private
-  def sincedb_file
-    File.join(ENV["HOME"], ".sincedb_" + Digest::MD5.hexdigest("#{@log_group}"))
-  end
-
-  module SinceDB
-    class File
-      def initialize(file)
-        @sincedb_path = file
-      end
-
-      def newer?(date)
-        date > read
-      end
-
-      def read
-        if ::File.exists?(@sincedb_path)
-          since = ::File.read(@sincedb_path).chomp.strip.to_i
-        else
-          since = 1
-        end
-        return since
-      end
-
-      def write(since = nil)
-        since = DateTime.now.strftime('%Q') if since.nil?
-        ::File.open(@sincedb_path, 'w') { |file| file.write(since.to_s) }
-      end
-    end
+  def serialize_sincedb
+    @sincedb.map do |group, pos|
+      [group, pos].join(" ")
+    end.join("\n") + "\n"
   end
 end # class LogStash::Inputs::CloudWatch_Logs

data/logstash-input-cloudwatch_logs.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-cloudwatch_logs'
-  s.version         = '0.10.3'
+  s.version         = '1.0.0.pre'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'Stream events from CloudWatch Logs.'
   s.description     = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'

data/spec/inputs/cloudwatch_logs_spec.rb

@@ -13,7 +13,7 @@ describe LogStash::Inputs::CloudWatch_Logs do
       {
           'access_key_id' => '1234',
           'secret_access_key' => 'secret',
-          'log_group' => 'sample-log-group',
+          'log_group' => ['sample-log-group'],
           'region' => 'us-east-1'
       }
     }
@@ -23,4 +23,21 @@ describe LogStash::Inputs::CloudWatch_Logs do
       expect {subject.register}.to_not raise_error
     end
   end
+
+  describe '#run' do
+    let(:config) {
+      {
+          'access_key_id' => '1234',
+          'secret_access_key' => 'secret',
+          'log_group' => ['sample-log-group'],
+          'region' => 'us-east-1'
+      }
+    }
+    subject {LogStash::Inputs::CloudWatch_Logs.new(config)}
+
+    it "runs" do
+      subject.register
+      expect{subject.run({})}.to_not raise_error
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-cloudwatch_logs
 version: !ruby/object:Gem::Version
-  version: 0.10.3
+  version: 1.0.0.pre
 platform: ruby
 authors:
 - Luke Waite
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-05-07 00:00:00.000000000 Z
+date: 2017-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,8 +84,8 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
-- lib/logstash/inputs/cloudwatch/patch.rb
 - lib/logstash/inputs/cloudwatch_logs.rb
+- lib/logstash/inputs/cloudwatch_logs/patch.rb
 - logstash-input-cloudwatch_logs.gemspec
 - spec/inputs/cloudwatch_logs_spec.rb
 homepage: ''
@@ -105,9 +105,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.4.8