logstash-lib 1.3.2

data/lib/logstash/filters/elapsed.rb

@@ -0,0 +1,256 @@
+# elapsed filter
+#
+# This filter tracks a pair of start/end events and calculates the elapsed
+# time between them.
+
+require "logstash/filters/base"
+require "logstash/namespace"
+require 'thread'
+
+# The elapsed filter tracks a pair of start/end events and uses their
+# timestamps to calculate the elapsed time between them.
+#
+# The filter has been developed to track the execution time of processes and
+# other long tasks.
+#
+# The configuration looks like this:
+#
+#     filter {
+#       elapsed {
+#         start_tag => "start event tag"
+#         end_tag => "end event tag"
+#         unique_id_field => "id field name"
+#         timeout => seconds
+#         new_event_on_match => true/false
+#       }
+#     }
+#
+# The events managed by this filter must have some particular properties.
+# The event describing the start of the task (the "start event") must contain
+# a tag equal to 'start_tag'. On the other side, the event describing the end
+# of the task (the "end event") must contain a tag equal to 'end_tag'. Both
+# these two kinds of event need to own an ID field which identify uniquely that
+# particular task. The name of this field is stored in 'unique_id_field'.
+#
+# You can use a Grok filter to prepare the events for the elapsed filter.
+# An example of configuration can be:
+#
+#     filter {
+#       grok {
+#         match => ["message", "%{TIMESTAMP_ISO8601} START id: (?<task_id>.*)"]
+#         add_tag => [ "taskStarted" ]
+#       }
+#
+#       grok {
+#         match => ["message", "%{TIMESTAMP_ISO8601} END id: (?<task_id>.*)"]
+#         add_tag => [ "taskTerminated"]
+#       }
+#
+#       elapsed {
+#         start_tag => "taskStarted"
+#         end_tag => "taskTerminated"
+#         unique_id_field => "task_id"
+#       }
+#     }
+#
+# The elapsed filter collects all the "start events". If two, or more, "start
+# events" have the same ID, only the first one is recorded, the others are
+# discarded.
+#
+# When an "end event" matching a previously collected "start event" is
+# received, there is a match. The configuration property 'new_event_on_match'
+# tells where to insert the elapsed information: they can be added to the
+# "end event" or a new "match event" can be created. Both events store the
+# following information:
+# - the tags "elapsed" and "elapsed.match"
+# - the field "elapsed.time" with the difference, in seconds, between
+#   the two events timestamps
+# - an ID filed with the task ID
+# - the field "elapsed.timestamp_start" with the timestamp of the "start event"
+#
+# If the "end event" does not arrive before "timeout" seconds, the
+# "start event" is discarded and an "expired event" is generated. This event
+# contains:
+# - the tags "elapsed" and "elapsed.expired_error"
+# - a field called "elapsed.time" with the age, in seconds, of the
+#   "start event"
+# - an ID filed with the task ID
+# - the field "elapsed.timestamp_start" with the timestamp of the "start event"
+#
+class LogStash::Filters::Elapsed < LogStash::Filters::Base
+  PREFIX = "elapsed."
+  ELAPSED_FIELD = PREFIX + "time"
+  TIMESTAMP_START_EVENT_FIELD = PREFIX + "timestamp_start"
+  HOST_FIELD = "host"
+
+  ELAPSED_TAG = "elapsed"
+  EXPIRED_ERROR_TAG = PREFIX + "expired_error"
+  END_WITHOUT_START_TAG = PREFIX + "end_wtihout_start"
+  MATCH_TAG = PREFIX + "match"
+
+  config_name "elapsed"
+  milestone 1
+
+  # The name of the tag identifying the "start event"
+  config :start_tag, :validate => :string, :required => true
+
+  # The name of the tag identifying the "end event"
+  config :end_tag, :validate => :string, :required => true
+
+  # The name of the field containing the task ID.
+  # This value must uniquely identify the task in the system, otherwise
+  # it's impossible to match the couple of events.
+  config :unique_id_field, :validate => :string, :required => true
+
+  # The amount of seconds after an "end event" can be considered lost.
+  # The corresponding "start event" is discarded and an "expired event"
+  # is generated. The default value is 30 minutes (1800 seconds).
+  config :timeout, :validate => :number, :required => false, :default => 1800
+
+  # This property manage what to do when an "end event" matches a "start event".
+  # If it's set to 'false' (default value), the elapsed information are added
+  # to the "end event"; if it's set to 'true' a new "match event" is created.
+  config :new_event_on_match, :validate => :boolean, :required => false, :default => false
+
+  public
+  def register
+    @mutex = Mutex.new
+    # This is the state of the filter. The keys are the "unique_id_field",
+    # the values are couples of values: <start event, age>
+    @start_events = {}
+
+    @logger.info("Elapsed, timeout: #{@timeout} seconds")
+  end
+
+  # Getter method used for the tests
+  def start_events
+    @start_events
+  end
+
+  def filter(event)
+    return unless filter?(event)
+
+    unique_id = event[@unique_id_field]
+    return if unique_id.nil?
+
+    if(start_event?(event))
+      filter_matched(event)
+      @logger.info("Elapsed, 'start event' received", start_tag: @start_tag, unique_id_field: @unique_id_field)
+
+      @mutex.synchronize do
+        unless(@start_events.has_key?(unique_id))
+          @start_events[unique_id] = LogStash::Filters::Elapsed::Element.new(event)
+        end
+      end
+
+    elsif(end_event?(event))
+      filter_matched(event)
+      @logger.info("Elapsed, 'end event' received", end_tag: @end_tag, unique_id_field: @unique_id_field)
+
+      @mutex.lock
+      if(@start_events.has_key?(unique_id))
+        start_event = @start_events.delete(unique_id).event
+        @mutex.unlock
+        elapsed = event["@timestamp"] - start_event["@timestamp"]
+        if(@new_event_on_match)
+          elapsed_event = new_elapsed_event(elapsed, unique_id, start_event["@timestamp"])
+          filter_matched(elapsed_event)
+          yield elapsed_event if block_given?
+        else
+          return add_elapsed_info(event, elapsed, unique_id, start_event["@timestamp"])
+        end
+      else
+        @mutex.unlock
+        # The "start event" did not arrive.
+        event.tag(END_WITHOUT_START_TAG)
+      end
+    end
+  end # def filter
+
+  # The method is invoked by LogStash every 5 seconds.
+  def flush()
+    expired_elements = []
+
+    @mutex.synchronize do
+      increment_age_by(5)
+      expired_elements = remove_expired_elements()
+    end
+
+    return create_expired_events_from(expired_elements)
+  end
+
+  private
+  def increment_age_by(seconds)
+    @start_events.each_pair do |key, element|
+      element.age += seconds
+    end
+  end
+
+  # Remove the expired "start events" from the internal
+  # buffer and return them.
+  def remove_expired_elements()
+    expired = []
+    @start_events.delete_if do |key, element|
+      if(element.age >= @timeout)
+        expired << element
+        next true
+      end
+      next false
+    end
+
+    return expired
+  end
+
+  def create_expired_events_from(expired_elements)
+    events = []
+    expired_elements.each do |element|
+      error_event = LogStash::Event.new
+      error_event.tag(ELAPSED_TAG)
+      error_event.tag(EXPIRED_ERROR_TAG)
+
+      error_event[HOST_FIELD] = Socket.gethostname
+      error_event[@unique_id_field] = element.event[@unique_id_field]
+      error_event[ELAPSED_FIELD] = element.age
+      error_event[TIMESTAMP_START_EVENT_FIELD] = element.event["@timestamp"]
+
+      events << error_event
+      filter_matched(error_event)
+    end
+
+    return events
+  end
+
+  def start_event?(event)
+    return (event["tags"] != nil && event["tags"].include?(@start_tag))
+  end
+
+  def end_event?(event)
+    return (event["tags"] != nil && event["tags"].include?(@end_tag))
+  end
+
+  def new_elapsed_event(elapsed_time, unique_id, timestamp_start_event)
+      new_event = LogStash::Event.new
+      new_event[HOST_FIELD] = Socket.gethostname
+      return add_elapsed_info(new_event, elapsed_time, unique_id, timestamp_start_event)
+  end
+
+  def add_elapsed_info(event, elapsed_time, unique_id, timestamp_start_event)
+      event.tag(ELAPSED_TAG)
+      event.tag(MATCH_TAG)
+
+      event[ELAPSED_FIELD] = elapsed_time
+      event[@unique_id_field] = unique_id
+      event[TIMESTAMP_START_EVENT_FIELD] = timestamp_start_event
+
+      return event
+  end
+end # class LogStash::Filters::Elapsed
+
+class LogStash::Filters::Elapsed::Element
+  attr_accessor :event, :age
+
+  def initialize(event)
+    @event = event
+    @age = 0
+  end
+end

data/lib/logstash/filters/elasticsearch.rb

@@ -0,0 +1,73 @@
+require "logstash/filters/base"
+require "logstash/namespace"
+require "logstash/util/fieldreference"
+
+# Search elasticsearch for a previous log event and copy some fields from it
+# into the current event.  Below is a complete example of how this filter might
+# be used.  Whenever logstash receives an "end" event, it uses this elasticsearch
+# filter to find the matching "start" event based on some operation identifier.
+# Then it copies the @timestamp field from the "start" event into a new field on
+# the "end" event.  Finally, using a combination of the "date" filter and the
+# "ruby" filter, we calculate the time duration in hours between the two events.
+#
+#       if [type] == "end" {
+#          elasticsearch {
+#             hosts => ["es-server"]
+#             query => "type:start AND operation:%{[opid]}"
+#             fields => ["@timestamp", "started"]
+#          }
+#
+#          date {
+#             match => ["[started]", "ISO8601"]
+#             target => "[started]"
+#          }
+#
+#          ruby {
+#             code => "event['duration_hrs'] = (event['@timestamp'] - event['started']) / 3600 rescue nil"
+#          }
+#       }
+#
+class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
+  config_name "elasticsearch"
+  milestone 1
+
+  # List of elasticsearch hosts to use for querying.
+  config :hosts, :validate => :array
+
+  # Elasticsearch query string
+  config :query, :validate => :string
+
+  # Comma-delimited list of <field>:<direction> pairs that define the sort order
+  config :sort, :validate => :string, :default => "@timestamp:desc"
+
+  # Hash of fields to copy from old event (found via elasticsearch) into new event
+  config :fields, :validate => :hash, :default => {}
+
+  public
+  def register
+    require "elasticsearch"
+
+    @logger.info("New ElasticSearch filter", :hosts => @hosts)
+    @client = Elasticsearch::Client.new hosts: @hosts
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    begin
+      query_str = event.sprintf(@query)
+
+      results = @client.search q: query_str, sort: @sort, size: 1
+
+      @fields.each do |old, new|
+        event[new] = results['hits']['hits'][0]['_source'][old]
+      end
+
+      filter_matched(event)
+    rescue => e
+      @logger.warn("Failed to query elasticsearch for previous event",
+                   :query => query_str, :event => event, :error => e)
+    end
+  end # def filter
+end # class LogStash::Filters::Elasticsearch

data/lib/logstash/filters/environment.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# Set fields from environment variables
+class LogStash::Filters::Environment < LogStash::Filters::Base
+  config_name "environment"
+  milestone 1
+
+  # Specify a hash of fields to the environment variable
+  # A hash of matches of field => environment variable
+  config :add_field_from_env, :validate => :hash, :default => {}
+
+  public
+  def register
+    # Nothing
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+    @add_field_from_env.each do |field, env|
+      event[field] = ENV[env]
+    end
+    filter_matched(event)
+  end # def filter
+end # class LogStash::Filters::Environment

data/lib/logstash/filters/extractnumbers.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+require 'logstash/namespace'
+require 'logstash/filters/base'
+
+
+# This filter automatically extracts all numbers found inside a string
+#
+# This is useful when you have lines that don't match a grok pattern
+# or use json but you still need to extract numbers.
+#
+# Each numbers is returned in a @fields.intX or @fields.floatX field
+# where X indicates the position in the string.
+#
+# The fields produced by this filter are extra useful used in combination
+# with kibana number plotting features.
+class LogStash::Filters::ExtractNumbers < LogStash::Filters::Base
+  config_name 'extractnumbers'
+  milestone 1
+
+  # The source field for the data. By default is message.
+  config :source, :validate => :string, :default => 'message'
+
+  public
+  def register
+  end
+
+  public
+  def filter(event)
+    integers = nil
+    floats = nil
+
+    msg = event[@source]
+
+    if not msg
+      return
+    end
+
+    # If for some reason the field is an array of values, take the first only.
+    msg = msg.first if msg.is_a?(Array)
+
+
+    fields = msg.split
+    for elem in fields
+      int = str_as_integer(elem)
+      if int != nil
+        if not integers
+          integers = Array.new
+        end
+        integers.push(int)
+        next
+      end
+      f = str_as_float(elem)
+      if f != nil
+        if not floats
+          floats = Array.new
+        end
+        floats.push(f)
+      end
+    end
+
+    if integers
+      index = 0
+      for i in integers
+        index += 1
+        event["int" + index.to_s] = i
+      end
+    end
+    if floats
+      index = 0
+      for f in floats
+        index += 1
+        event["float" + index.to_s] = f
+      end
+    end
+  end
+
+  def str_as_integer(str)
+    Integer(str) rescue nil
+  end
+
+  def str_as_float(str)
+    Float(str) rescue nil
+  end
+end # class LogStash::Filters::ExtractNumbers

data/lib/logstash/filters/gelfify.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The GELFify filter parses RFC3164 severity levels to
+# corresponding GELF levels.
+class LogStash::Filters::Gelfify < LogStash::Filters::Base
+  config_name "gelfify"
+  milestone 2
+
+  SYSLOG_LEVEL_MAP = {
+    0 => 3, # Emergency => FATAL
+    1 => 5, # Alert     => WARN
+    2 => 3, # Critical  => FATAL
+    3 => 4, # Error     => ERROR
+    4 => 5, # Warning   => WARN
+    5 => 6, # Notice    => INFO
+    6 => 6, # Informat. => INFO
+    7 => 7  # Debug     => DEBUG
+  }
+
+  public
+  def register
+    # nothing
+  end # def register
+
+  public
+  def filter(event)
+    return unless event["type"] == @type
+    @logger.debug("GELFIFY FILTER: received event of type #{event["type"]}")
+
+    if event.include?("severity")
+      sev = event["severity"].to_i rescue nil
+      if sev.to_s != event["severity"].to_s
+        # severity isn't convertable to an integer.
+        # "foo".to_i => 0, which would default to EMERG.
+        @logger.debug("GELFIFY FILTER: existing severity field is not an int")
+      elsif SYSLOG_LEVEL_MAP[sev]
+        @logger.debug("GELFIFY FILTER: Severity level successfully mapped")
+        event["GELF_severity"] = SYSLOG_LEVEL_MAP[sev]
+      else
+        @logger.debug("GELFIFY FILTER: unknown severity #{sev}")
+      end
+    else
+      @logger.debug("GELFIFY FILTER: No 'severity' field found")
+    end
+
+    if !event.cancelled?
+      filter_matched(event)
+    end
+  end # def filter
+end # class LogStash::Filters::Gelfify