logstash-lib 1.3.2

data/lib/logstash/filters/checksum.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+require "yaml"
+
+# This filter let's you create a checksum based on various parts
+# of the logstash event.
+# This can be useful for deduplication of messages or simply to provide
+# a custom unique identifier.
+#
+# This is VERY experimental and is largely a proof-of-concept
+class LogStash::Filters::Checksum < LogStash::Filters::Base
+
+  config_name "checksum"
+  milestone 1
+  
+  ALGORITHMS = ["md5", "sha", "sha1", "sha256", "sha384",]
+
+  # A list of keys to use in creating the string to checksum
+  # Keys will be sorted before building the string
+  # keys and values will then be concatenated with pipe delimeters
+  # and checksummed
+  config :keys, :validate => :array, :default => ["message", "@timestamp", "type"]
+
+  config :algorithm, :validate => ALGORITHMS, :default => "sha256"
+
+  public
+  def register
+    require 'openssl'
+    @to_checksum = ""
+  end
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    @logger.debug("Running checksum filter", :event => event)
+
+    @keys.sort.each do |k|
+      @logger.debug("Adding key to string", :current_key => k)
+      @to_checksum << "|#{k}|#{event[k]}"
+    end
+    @to_checksum << "|"
+    @logger.debug("Final string built", :to_checksum => @to_checksum)
+
+    digested_string = OpenSSL::Digest.hexdigest(@algorithm, @to_checksum)
+    @logger.debug("Digested string", :digested_string => digested_string)
+    event['logstash_checksum'] = digested_string
+  end
+end # class LogStash::Filters::Checksum

data/lib/logstash/filters/cidr.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+require "ipaddr"
+
+# The CIDR filter is for checking IP addresses in events against a list of
+# network blocks that might contain it. Multiple addresses can be checked
+# against multiple networks, any match succeeds. Upon success additional tags
+# and/or fields can be added to the event.
+
+class LogStash::Filters::CIDR < LogStash::Filters::Base
+
+  config_name "cidr"
+  milestone 1
+
+  # The IP address(es) to check with. Example:
+  #
+  #     filter {
+  #       %PLUGIN% {
+  #         add_tag => [ "testnet" ]
+  #         address => [ "%{src_ip}", "%{dst_ip}" ]
+  #         network => [ "192.0.2.0/24" ]
+  #       }
+  #     }
+  config :address, :validate => :array, :default => []
+
+  # The IP network(s) to check against. Example:
+  #
+  #     filter {
+  #       %PLUGIN% {
+  #         add_tag => [ "linklocal" ]
+  #         address => [ "%{clientip}" ]
+  #         network => [ "169.254.0.0/16", "fe80::/64" ]
+  #       }
+  #     }
+  config :network, :validate => :array, :default => []
+
+  public
+  def register
+    # Nothing
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    address = @address.collect do |a|
+      begin
+        IPAddr.new(event.sprintf(a))
+      rescue ArgumentError => e
+        @logger.warn("Invalid IP address, skipping", :address => a, :event => event)
+        nil
+      end
+    end
+    address.compact!
+
+    network = @network.collect do |n|
+      begin
+        IPAddr.new(event.sprintf(n))
+      rescue ArgumentError => e
+        @logger.warn("Invalid IP network, skipping", :network => n, :event => event)
+        nil
+      end
+    end
+    network.compact!
+
+    # Try every combination of address and network, first match wins
+    address.product(network).each do |a, n|
+      @logger.debug("Checking IP inclusion", :address => a, :network => n)
+      if n.include?(a)
+        filter_matched(event)
+        return
+      end
+    end
+  end # def filter
+end # class LogStash::Filters::CIDR

data/lib/logstash/filters/cipher.rb

@@ -0,0 +1,145 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# This filter parses a source and apply a cipher or decipher before
+# storing it in the target.
+#
+class LogStash::Filters::Cipher < LogStash::Filters::Base
+  config_name "cipher"
+  milestone 1
+
+  # The field to perform filter
+  #
+  # Example, to use the @message field (default) :
+  #
+  #     filter { cipher { source => "message" } }
+  config :source, :validate => :string, :default => "message"
+
+  # The name of the container to put the result
+  #
+  # Example, to place the result into crypt :
+  #
+  #     filter { cipher { target => "crypt" } }
+  config :target, :validate => :string, :default => "message"
+
+  # Do we have to perform a base64 decode or encode?
+  #
+  # If we are decrypting, base64 decode will be done before.
+  # If we are encrypting, base64 will be done after.
+  #
+  config :base64, :validate => :boolean, :default => true
+
+  # The key to use
+  config :key, :validate => :string
+
+  # The key size to pad
+  #
+  # It depends of the cipher algorythm.I your key don't need
+  # padding, don't set this parameter
+  #
+  # Example, for AES-256, we must have 32 char long key
+  #     filter { cipher { key_size => 32 }
+  #
+  config :key_size, :validate => :number, :default => 32
+
+  # The character used to pad the key
+  config :key_pad, :default => "\0"
+
+  # The cipher algorythm
+  #
+  # A list of supported algorithms can be obtained by
+  #
+  #     puts OpenSSL::Cipher.ciphers
+  config :algorithm, :validate => :string, :required => true
+
+  # Encrypting or decrypting some data
+  #
+  # Valid values are encrypt or decrypt
+  config :mode, :validate => :string, :required => true
+
+  # Cypher padding to use. Enables or disables padding. 
+  #
+  # By default encryption operations are padded using standard block padding 
+  # and the padding is checked and removed when decrypting. If the pad 
+  # parameter is zero then no padding is performed, the total amount of data 
+  # encrypted or decrypted must then be a multiple of the block size or an 
+  # error will occur.
+  #
+  # See EVP_CIPHER_CTX_set_padding for further information.
+  #
+  # We are using Openssl jRuby which uses default padding to PKCS5Padding
+  # If you want to change it, set this parameter. If you want to disable
+  # it, Set this parameter to 0
+  #     filter { cipher { padding => 0 }}
+  config :cipher_padding, :validate => :string
+
+  # The initialization vector to use
+  #
+  # The cipher modes CBC, CFB, OFB and CTR all need an "initialization
+  # vector", or short, IV. ECB mode is the only mode that does not require
+  # an IV, but there is almost no legitimate use case for this mode
+  # because of the fact that it does not sufficiently hide plaintext patterns.
+  config :iv, :validate => :string
+
+  def register
+    require 'base64' if @base64
+    init_cipher
+  end # def register
+
+
+  def filter(event)
+    return unless filter?(event)
+
+
+    #If decrypt or encrypt fails, we keep it it intact.
+    begin
+      #@logger.debug("Event to filter", :event => event)
+      data = event[@source]
+      if @mode == "decrypt"
+        data =  Base64.decode64(data) if @base64 == true
+      end
+      result = @cipher.update(data) + @cipher.final
+      if @mode == "encrypt"
+        data =  Base64.encode64(data) if @base64 == true
+      end
+    rescue => e
+      @logger.warn("Exception catch on cipher filter", :event => event, :error => e)
+    else
+      event[@target]= result
+      #Is it necessary to add 'if !result.nil?' ? exception have been already catched.
+      #In doubt, I keep it.
+      filter_matched(event) if !result.nil?
+      #Too much bad result can be a problem, reinit cipher prevent this.
+      init_cipher
+    end
+  end # def filter
+
+  def init_cipher
+
+    @cipher = OpenSSL::Cipher.new(@algorithm)
+    if @mode == "encrypt"
+      @cipher.encrypt
+    elsif @mode == "decrypt"
+      @cipher.decrypt
+    else
+      @logger.error("Invalid cipher mode. Valid values are \"encrypt\" or \"decrypt\"", :mode => @mode)
+      raise "Bad configuration, aborting."
+    end
+
+    if @key.length != @key_size
+      @logger.debug("key length is " + @key.length.to_s + ", padding it to " + @key_size.to_s + " with '" + @key_pad.to_s + "'")
+      @key = @key[0,@key_size].ljust(@key_size,@key_pad)
+    end
+
+    @cipher.key = @key
+
+    @cipher.iv = @iv if @iv
+
+    @cipher.padding = @cipher_padding if @cipher_padding
+
+    @logger.debug("Cipher initialisation done", :mode => @mode, :key => @key, :iv => @iv, :cipher_padding => @cipher_padding)
+  end # def init_cipher
+
+
+end # class LogStash::Filters::Cipher

data/lib/logstash/filters/clone.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The clone filter is for duplicating events.
+# A clone will be made for each type in the clone list.
+# The original event is left unchanged.
+class LogStash::Filters::Clone < LogStash::Filters::Base
+
+  config_name "clone"
+  milestone 2
+
+  # A new clone will be created with the given type for each type in this list.
+  config :clones, :validate => :array, :default => []
+
+  public
+  def register
+    # Nothing to do
+  end
+
+  public
+  def filter(event)
+    return unless filter?(event)
+    @clones.each do |type|
+      clone = event.clone
+      clone["type"] = type
+      filter_matched(clone)
+      @logger.debug("Cloned event", :clone => clone, :event => event)
+
+      # Push this new event onto the stack at the LogStash::FilterWorker
+      yield clone
+    end
+  end
+
+end # class LogStash::Filters::Clone

data/lib/logstash/filters/collate.rb

@@ -0,0 +1,114 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# Collate events by time or count.
+#
+# The original goal of this filter was to merge the logs from different sources
+# by the time of log, for example, in real-time log collection, logs can be
+# collated by amount of 3000 logs or can be collated in 30 seconds.
+#
+# The config looks like this:
+#
+#     filter {
+#       collate {
+#         size => 3000
+#         interval => "30s"
+#         order => "ascending"
+#       }
+#     }
+class LogStash::Filters::Collate < LogStash::Filters::Base
+
+  config_name "collate"
+  milestone 1
+
+  # How many logs should be collated.
+  config :count, :validate => :number, :default => 1000
+
+  # The 'interval' is the time window which how long the logs should be collated. (default 1m)
+  config :interval, :validate => :string, :default => "1m"
+
+  # The 'order' collated events should appear in.
+  config :order, :validate => ["ascending", "descending"], :default => "ascending"
+
+  public
+  def register
+    require "thread"
+    require "rufus/scheduler"
+
+    @mutex = Mutex.new
+    @collatingDone = false
+    @collatingArray = Array.new
+    @scheduler = Rufus::Scheduler.start_new
+    @job = @scheduler.every @interval do
+      @logger.info("Scheduler Activated")
+      @mutex.synchronize{
+        collate
+      }
+    end
+  end # def register
+
+  public
+  def filter(event)
+    @logger.info("do collate filter")
+    if event == LogStash::SHUTDOWN
+      @job.trigger()
+      @job.unschedule()
+      @logger.info("collate filter thread shutdown.")
+      return
+    end
+
+    # if the event is collated, a "collated" tag will be marked, so for those uncollated event, cancel them first.
+    if event["tags"].nil? || !event.tags.include?("collated")
+      event.cancel
+    else
+      return
+    end
+
+    @mutex.synchronize{
+      @collatingArray.push(event.clone)
+
+      if (@collatingArray.length == @count)
+        collate
+      end
+
+      if (@collatingDone)
+        while collatedEvent = @collatingArray.pop
+          collatedEvent["tags"] = Array.new if collatedEvent["tags"].nil?
+          collatedEvent["tags"] << "collated"
+          filter_matched(collatedEvent)
+          yield collatedEvent
+        end # while @collatingArray.pop
+        # reset collatingDone flag
+        @collatingDone = false
+      end
+    }
+  end # def filter
+
+  private
+  def collate
+    if (@order == "ascending")
+      @collatingArray.sort! { |eventA, eventB| eventB.timestamp <=> eventA.timestamp }
+    else 
+      @collatingArray.sort! { |eventA, eventB| eventA.timestamp <=> eventB.timestamp }
+    end
+    @collatingDone = true
+  end # def collate
+
+  # Flush any pending messages.
+  public
+  def flush
+    events = []
+    if (@collatingDone)
+      @mutex.synchronize{
+        while collatedEvent = @collatingArray.pop
+          collatedEvent["tags"] << "collated"
+          events << collatedEvent
+        end # while @collatingArray.pop
+      }
+      # reset collatingDone flag.
+      @collatingDone = false
+    end
+    return events
+  end # def flush
+end #

data/lib/logstash/filters/csv.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+require "csv"
+
+# CSV filter. Takes an event field containing CSV data, parses it,
+# and stores it as individual fields (can optionally specify the names).
+class LogStash::Filters::CSV < LogStash::Filters::Base
+  config_name "csv"
+  milestone 2
+
+  # The CSV data in the value of the source field will be expanded into a
+  # datastructure.
+  config :source, :validate => :string, :default => "message"
+
+  # Define a list of column names (in the order they appear in the CSV,
+  # as if it were a header line). If this is not specified or there
+  # are not enough columns specified, the default column name is "columnX"
+  # (where X is the field number, starting from 1).
+  config :columns, :validate => :array, :default => []
+
+  # Define the column separator value. If this is not specified the default
+  # is a comma ','.
+  # Optional.
+  config :separator, :validate => :string, :default => ","
+
+  # Define the character used to quote CSV fields. If this is not specified
+  # the default is a double quote '"'.
+  # Optional.
+  config :quote_char, :validate => :string, :default => '"'
+
+  # Define target for placing the data.
+  # Defaults to writing to the root of the event.
+  config :target, :validate => :string
+
+  public
+  def register
+
+    # Nothing to do here
+
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    @logger.debug("Running csv filter", :event => event)
+
+    matches = 0
+
+    if event[@source]
+      if event[@source].is_a?(String)
+        event[@source] = [event[@source]]
+      end
+
+      if event[@source].length > 1
+        @logger.warn("csv filter only works on fields of length 1",
+                     :source => @source, :value => event[@source],
+                     :event => event)
+        return
+      end
+
+      raw = event[@source].first
+      begin
+        values = CSV.parse_line(raw, :col_sep => @separator, :quote_char => @quote_char)
+
+        if @target.nil?
+          # Default is to write to the root of the event.
+          dest = event
+        else
+          dest = event[@target] ||= {}
+        end
+
+        values.each_index do |i|
+          field_name = @columns[i] || "column#{i+1}"
+          dest[field_name] = values[i]
+        end
+
+        filter_matched(event)
+      rescue => e
+        event.tag "_csvparsefailure"
+        @logger.warn("Trouble parsing csv", :source => @source, :raw => raw,
+                      :exception => e)
+        return
+      end # begin
+    end # if event
+
+    @logger.debug("Event after csv filter", :event => event)
+
+  end # def filter
+
+end # class LogStash::Filters::Csv
+