logstash-lib 1.3.2

data/lib/logstash/filters/metaevent.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+class LogStash::Filters::Metaevent < LogStash::Filters::Base
+  config_name "metaevent"
+  milestone 1
+
+  # syntax: `followed_by_tags => [ "tag", "tag" ]`
+  config :followed_by_tags, :validate => :array, :required => true
+
+  # syntax: `period => 60`
+  config :period, :validate => :number, :default => 5
+
+  def register
+    @logger.debug("registering")
+    @metaevents = []
+  end
+
+  def filter(event)
+    if filter?(event)
+      start_period(event)
+    elsif within_period(event)
+      if followed_by_tags_match(event)
+        trigger(event)
+      else
+        @logger.debug(["metaevent", @add_tag, "ignoring (tags don't match)", event])
+      end
+    else
+      @logger.debug(["metaevent", @add_tag, "ignoring (not in period)", event])
+    end
+  end
+
+  def flush
+    return if @metaevents.empty?
+
+    new_events = @metaevents
+    @metaevents = []
+    new_events
+  end
+
+  private
+
+  def start_period(event)
+    @logger.debug(["metaevent", @add_tag, "start_period", event])
+    @start_event = event
+  end
+
+  def trigger(event)
+    @logger.debug(["metaevent", @add_tag, "trigger", event])
+
+    event = LogStash::Event.new
+    event["source"] = Socket.gethostname
+    event["tags"] = [@add_tag]
+
+    @metaevents << event
+    @start_event = nil
+  end
+
+  def followed_by_tags_match(event)
+    (event["tags"] & @followed_by_tags).size == @followed_by_tags.size
+  end
+
+  def within_period(event)
+    time_delta = event["@timestamp"] - @start_event["@timestamp"]
+    time_delta >= 0 && time_delta <= @period
+  end
+end

data/lib/logstash/filters/metrics.rb

@@ -0,0 +1,237 @@
+# encoding: utf-8
+require "securerandom"
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The metrics filter is useful for aggregating metrics.
+#
+# For example, if you have a field 'response' that is
+# a http response code, and you want to count each
+# kind of response, you can do this:
+#
+#     filter {
+#       metrics {
+#         meter => [ "http.%{response}" ]
+#         add_tag => "metric"
+#       }
+#     }
+#
+# Metrics are flushed every 5 seconds by default or according to
+# 'flush_interval'. Metrics appear as
+# new events in the event stream and go through any filters
+# that occur after as well as outputs.
+#
+# In general, you will want to add a tag to your metrics and have an output
+# explicitly look for that tag.
+#
+# The event that is flushed will include every 'meter' and 'timer'
+# metric in the following way:
+#
+# #### 'meter' values
+#
+# For a `meter => "something"` you will receive the following fields:
+#
+# * "thing.count" - the total count of events
+# * "thing.rate_1m" - the 1-minute rate (sliding)
+# * "thing.rate_5m" - the 5-minute rate (sliding)
+# * "thing.rate_15m" - the 15-minute rate (sliding)
+#
+# #### 'timer' values
+#
+# For a `timer => [ "thing", "%{duration}" ]` you will receive the following fields:
+#
+# * "thing.count" - the total count of events
+# * "thing.rate_1m" - the 1-minute rate of events (sliding)
+# * "thing.rate_5m" - the 5-minute rate of events (sliding)
+# * "thing.rate_15m" - the 15-minute rate of events (sliding)
+# * "thing.min" - the minimum value seen for this metric
+# * "thing.max" - the maximum value seen for this metric
+# * "thing.stddev" - the standard deviation for this metric
+# * "thing.mean" - the mean for this metric
+#
+# #### Example: computing event rate
+#
+# For a simple example, let's track how many events per second are running
+# through logstash:
+#
+#     input {
+#       generator {
+#         type => "generated"
+#       }
+#     }
+#
+#     filter {
+#       metrics {
+#         type => "generated"
+#         meter => "events"
+#         add_tag => "metric"
+#       }
+#     }
+#
+#     output {
+#       stdout {
+#         # only emit events with the 'metric' tag
+#         tags => "metric"
+#         message => "rate: %{events.rate_1m}"
+#       }
+#     }
+#
+# Running the above:
+#
+#     % java -jar logstash.jar agent -f example.conf
+#     rate: 23721.983566819246
+#     rate: 24811.395722536377
+#     rate: 25875.892745934525
+#     rate: 26836.42375967113
+#
+# We see the output includes our 'events' 1-minute rate.
+#
+# In the real world, you would emit this to graphite or another metrics store,
+# like so:
+#
+#     output {
+#       graphite {
+#         metrics => [ "events.rate_1m", "%{events.rate_1m}" ]
+#       }
+#     }
+class LogStash::Filters::Metrics < LogStash::Filters::Base
+  config_name "metrics"
+  milestone 1
+
+  # syntax: `meter => [ "name of metric", "name of metric" ]`
+  config :meter, :validate => :array, :default => []
+
+  # syntax: `timer => [ "name of metric", "%{time_value}" ]`
+  config :timer, :validate => :hash, :default => {}
+
+  # Don't track events that have @timestamp older than some number of seconds.
+  #
+  # This is useful if you want to only include events that are near real-time
+  # in your metrics.
+  #
+  # Example, to only count events that are within 10 seconds of real-time, you
+  # would do this:
+  #
+  #     filter {
+  #       metrics {
+  #         meter => [ "hits" ]
+  #         ignore_older_than => 10
+  #       }
+  #     }
+  config :ignore_older_than, :validate => :number, :default => 0
+
+  # The flush interval, when the metrics event is created. Must be a multiple of 5s.
+  config :flush_interval, :validate => :number, :default => 5
+
+  # The clear interval, when all counter are reset.
+  #
+  # If set to -1, the default value, the metrics will never be cleared.
+  # Otherwise, should be a multiple of 5s.
+  config :clear_interval, :validate => :number, :default => -1
+
+  # The rates that should be measured, in minutes.
+  # Possible values are 1, 5, and 15.
+  config :rates, :validate => :array, :default => [1, 5, 15]
+
+  # The percentiles that should be measured
+  config :percentiles, :validate => :array, :default => [1, 5, 10, 90, 95, 99, 100]
+
+  def register
+    require "metriks"
+    require "socket"
+    @last_flush = 0 # how many seconds ago the metrics where flushed.
+    @last_clear = 0 # how many seconds ago the metrics where cleared.
+    @random_key_preffix = SecureRandom.hex
+    unless (@rates - [1, 5, 15]).empty?
+      raise LogStash::ConfigurationError, "Invalid rates configuration. possible rates are 1, 5, 15. Rates: #{rates}."
+    end
+    initialize_metrics
+  end # def register
+
+  def filter(event)
+    return unless filter?(event)
+
+    # TODO(piavlo): This should probably be moved to base filter class.
+    if @ignore_older_than > 0 && Time.now - event["@timestamp"] > @ignore_older_than
+      @logger.debug("Skipping metriks for old event", :event => event)
+      return
+    end
+
+    @meter.each do |m|
+      @metric_meters[event.sprintf(m)].mark
+    end
+
+    @timer.each do |name, value|
+      @metric_timers[event.sprintf(name)].update(event.sprintf(value).to_f)
+    end
+  end # def filter
+
+  def flush
+    # Add 5 seconds to @last_flush and @last_clear counters
+    # since this method is called every 5 seconds.
+    @last_flush += 5
+    @last_clear += 5
+
+    # Do nothing if there's nothing to do ;)
+    return unless should_flush?
+
+    event = LogStash::Event.new
+    event["message"] = Socket.gethostname
+    @metric_meters.each do |name, metric|
+      flush_rates event, name, metric
+      metric.clear if should_clear?
+    end
+
+    @metric_timers.each do |name, metric|
+      flush_rates event, name, metric
+      # These 4 values are not sliding, so they probably are not useful.
+      event["#{name}.min"] = metric.min
+      event["#{name}.max"] = metric.max
+      # timer's stddev currently returns variance, fix it.
+      event["#{name}.stddev"] = metric.stddev ** 0.5
+      event["#{name}.mean"] = metric.mean
+
+      @percentiles.each do |percentile|
+        event["#{name}.p#{percentile}"] = metric.snapshot.value(percentile / 100)
+      end
+      metric.clear if should_clear?
+    end
+
+    # Reset counter since metrics were flushed
+    @last_flush = 0
+
+    if should_clear?
+      #Reset counter since metrics were cleared
+      @last_clear = 0
+      initialize_metrics
+    end
+
+    filter_matched(event)
+    return [event]
+  end
+
+  private
+  def flush_rates(event, name, metric)
+      event["#{name}.count"] = metric.count
+      event["#{name}.rate_1m"] = metric.one_minute_rate if @rates.include? 1
+      event["#{name}.rate_5m"] = metric.five_minute_rate if @rates.include? 5
+      event["#{name}.rate_15m"] = metric.fifteen_minute_rate if @rates.include? 15
+  end
+
+  def initialize_metrics
+    @metric_meters = Hash.new { |h,k| h[k] = Metriks.meter metric_key(k) }
+    @metric_timers = Hash.new { |h,k| h[k] = Metriks.timer metric_key(k) }
+  end
+
+  def metric_key(key)
+    "#{@random_key_preffix}_#{key}"
+  end
+
+  def should_flush?
+    @last_flush >= @flush_interval && (@metric_meters.any? || @metric_timers.any?)
+  end
+
+  def should_clear?
+    @clear_interval > 0 && @last_clear >= @clear_interval
+  end
+end # class LogStash::Filters::Metrics

data/lib/logstash/filters/multiline.rb

@@ -0,0 +1,241 @@
+# encoding: utf-8
+# multiline filter
+#
+# This filter will collapse multiline messages into a single event.
+# 
+
+require "logstash/filters/base"
+require "logstash/namespace"
+require "set"
+
+# The multiline filter is for combining multiple events from a single source
+# into the same event.
+#
+# The original goal of this filter was to allow joining of multi-line messages
+# from files into a single event. For example - joining java exception and
+# stacktrace messages into a single event.
+#
+# TODO(sissel): Document any issues?
+# The config looks like this:
+#
+#     filter {
+#       multiline {
+#         type => "type"
+#         pattern => "pattern, a regexp"
+#         negate => boolean
+#         what => "previous" or "next"
+#       }
+#     }
+# 
+# The 'regexp' should match what you believe to be an indicator that
+# the field is part of a multi-line event
+#
+# The 'what' must be "previous" or "next" and indicates the relation
+# to the multi-line event.
+#
+# The 'negate' can be "true" or "false" (defaults false). If true, a 
+# message not matching the pattern will constitute a match of the multiline
+# filter and the what will be applied. (vice-versa is also true)
+#
+# For example, java stack traces are multiline and usually have the message
+# starting at the far-left, then each subsequent line indented. Do this:
+# 
+#     filter {
+#       multiline {
+#         type => "somefiletype"
+#         pattern => "^\s"
+#         what => "previous"
+#       }
+#     }
+#     
+# This says that any line starting with whitespace belongs to the previous line.
+#
+# Another example is C line continuations (backslash). Here's how to do that:
+#
+#     filter {
+#       multiline {
+#         type => "somefiletype "
+#         pattern => "\\$"
+#         what => "next"
+#       }
+#     }
+#     
+class LogStash::Filters::Multiline < LogStash::Filters::Base
+
+  config_name "multiline"
+  milestone 3
+
+  # The regular expression to match
+  config :pattern, :validate => :string, :required => true
+
+  # If the pattern matched, does event belong to the next or previous event?
+  config :what, :validate => ["previous", "next"], :required => true
+
+  # Negate the regexp pattern ('if not matched')
+  config :negate, :validate => :boolean, :default => false
+  
+  # The stream identity is how the multiline filter determines which stream an
+  # event belongs. This is generally used for differentiating, say, events
+  # coming from multiple files in the same file input, or multiple connections
+  # coming from a tcp input.
+  #
+  # The default value here is usually what you want, but there are some cases
+  # where you want to change it. One such example is if you are using a tcp
+  # input with only one client connecting at any time. If that client
+  # reconnects (due to error or client restart), then logstash will identify
+  # the new connection as a new stream and break any multiline goodness that
+  # may have occurred between the old and new connection. To solve this use
+  # case, you can use "%{@source_host}.%{@type}" instead.
+  config :stream_identity , :validate => :string, :default => "%{host}.%{path}.%{type}"
+  
+  # logstash ships by default with a bunch of patterns, so you don't
+  # necessarily need to define this yourself unless you are adding additional
+  # patterns.
+  #
+  # Pattern files are plain text with format:
+  #
+  #     NAME PATTERN
+  #
+  # For example:
+  #
+  #     NUMBER \d+
+  config :patterns_dir, :validate => :array, :default => []
+
+  # Detect if we are running from a jarfile, pick the right path.
+  @@patterns_path = Set.new
+  if __FILE__ =~ /file:\/.*\.jar!.*/
+    @@patterns_path += ["#{File.dirname(__FILE__)}/../../patterns/*"]
+  else
+    @@patterns_path += ["#{File.dirname(__FILE__)}/../../../patterns/*"]
+  end
+
+  public
+  def initialize(config = {})
+    super
+
+    @threadsafe = false
+
+    # This filter needs to keep state.
+    @types = Hash.new { |h,k| h[k] = [] }
+    @pending = Hash.new
+  end # def initialize
+
+  public
+  def register
+    require "grok-pure" # rubygem 'jls-grok'
+
+    @grok = Grok.new
+
+    @patterns_dir = @@patterns_path.to_a + @patterns_dir
+    @patterns_dir.each do |path|
+      # Can't read relative paths from jars, try to normalize away '../'
+      while path =~ /file:\/.*\.jar!.*\/\.\.\//
+        # replace /foo/bar/../baz => /foo/baz
+        path = path.gsub(/[^\/]+\/\.\.\//, "")
+      end
+
+      if File.directory?(path)
+        path = File.join(path, "*")
+      end
+
+      Dir.glob(path).each do |file|
+        @logger.info("Grok loading patterns from file", :path => file)
+        @grok.add_patterns_from_file(file)
+      end
+    end
+
+    @grok.compile(@pattern)
+
+    @logger.debug("Registered multiline plugin", :type => @type, :config => @config)
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+
+    if event["message"].is_a?(Array)
+      match = @grok.match(event["message"].first)
+    else
+      match = @grok.match(event["message"])
+    end
+    key = event.sprintf(@stream_identity)
+    pending = @pending[key]
+
+    @logger.debug("Multiline", :pattern => @pattern, :message => event["message"],
+                  :match => match, :negate => @negate)
+
+    # Add negate option
+    match = (match and !@negate) || (!match and @negate)
+
+    case @what
+    when "previous"
+      if match
+        event.tag "multiline"
+        # previous previous line is part of this event.
+        # append it to the event and cancel it
+        if pending
+          pending.append(event)
+        else
+          @pending[key] = event
+        end
+        event.cancel
+      else
+        # this line is not part of the previous event
+        # if we have a pending event, it's done, send it.
+        # put the current event into pending
+        if pending
+          tmp = event.to_hash
+          event.overwrite(pending)
+          @pending[key] = LogStash::Event.new(tmp)
+        else
+          @pending[key] = event
+          event.cancel
+        end # if/else pending
+      end # if/else match
+    when "next"
+      if match
+        event.tag "multiline"
+        # this line is part of a multiline event, the next
+        # line will be part, too, put it into pending.
+        if pending
+          pending.append(event)
+        else
+          @pending[key] = event
+        end
+        event.cancel
+      else
+        # if we have something in pending, join it with this message
+        # and send it. otherwise, this is a new message and not part of
+        # multiline, send it.
+        if pending
+          pending.append(event)
+          event.overwrite(pending.to_hash)
+          @pending.delete(key)
+        end
+      end # if/else match
+    else
+      # TODO(sissel): Make this part of the 'register' method.
+      @logger.warn("Unknown multiline 'what' value.", :what => @what)
+    end # case @what
+
+    if !event.cancelled?
+      event["message"] = event["message"].join("\n") if event["message"].is_a?(Array)
+      event["@timestamp"] = event["@timestamp"].first if event["@timestamp"].is_a?(Array)
+      filter_matched(event) if match
+    end
+  end # def filter
+
+  # Flush any pending messages. This is generally used for unit testing only.
+  #
+  # Note: flush is disabled now; it is preferable to use the multiline codec.
+  public
+  def __flush
+    events = []
+    @pending.each do |key, value|
+      value.uncancel
+      events << value
+    end
+    @pending.clear
+    return events
+  end # def flush
+end # class LogStash::Filters::Multiline