logstash-lib 1.3.2

data/lib/logstash/codecs/base.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/event"
+require "logstash/plugin"
+require "logstash/logging"
+
+# This is the base class for logstash codecs.
+module LogStash::Codecs; class Base < LogStash::Plugin
+  include LogStash::Config::Mixin
+  config_name "codec"
+
+  def initialize(params={})
+    super
+    config_init(params)
+    register if respond_to?(:register)
+  end
+
+  public
+  def decode(data)
+    raise "#{self.class}#decode must be overidden"
+  end # def decode
+
+  alias_method :<<, :decode
+
+  public
+  def encode(data)
+    raise "#{self.class}#encode must be overidden"
+  end # def encode
+
+  public 
+  def teardown; end;
+
+  public
+  def on_event(&block)
+    @on_event = block
+  end
+
+  public
+  def flush(&block)
+    # does nothing by default.
+    # if your codec needs a flush method (like you are spooling things)
+    # you must implement this.
+  end
+
+  public
+  def clone
+    return self.class.new(params)
+  end
+end; end # class LogStash::Codecs::Base

data/lib/logstash/codecs/compress_spooler.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+
+class LogStash::Codecs::CompressSpooler < LogStash::Codecs::Base
+  config_name 'compress_spooler'
+  milestone 1
+  config :spool_size, :validate => :number, :default => 50
+  config :compress_level, :validate => :number, :default => 6
+
+  public
+  def register
+    require "msgpack"
+    require "zlib"
+    @buffer = []
+  end
+
+  public
+  def decode(data)
+    z = Zlib::Inflate.new
+    data = MessagePack.unpack(z.inflate(data))
+    z.finish
+    z.close
+    data.each do |event|
+      event = LogStash::Event.new(event)
+      event["@timestamp"] = Time.at(event["@timestamp"]).utc if event["@timestamp"].is_a? Float
+      yield event
+    end
+  end # def decode
+
+  public
+  def encode(data)
+    if @buffer.length >= @spool_size
+      z = Zlib::Deflate.new(@compress_level)
+      @on_event.call z.deflate(MessagePack.pack(@buffer), Zlib::FINISH)
+      z.close
+      @buffer.clear
+    else
+      data["@timestamp"] = data["@timestamp"].to_f
+      @buffer << data.to_hash
+    end
+  end # def encode
+
+  public
+  def teardown
+    if !@buffer.nil? and @buffer.length > 0
+      @on_event.call @buffer
+    end
+    @buffer.clear
+  end
+end # class LogStash::Codecs::CompressSpooler

data/lib/logstash/codecs/dots.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+
+class LogStash::Codecs::Dots < LogStash::Codecs::Base
+  config_name "dots"
+  milestone 1
+
+  public
+  def decode(data)
+    raise "Not implemented"
+  end # def decode
+
+  public
+  def encode(data)
+    @on_event.call(".")
+  end # def encode
+
+end # class LogStash::Codecs::Dots

data/lib/logstash/codecs/edn.rb

@@ -0,0 +1,28 @@
+require "logstash/codecs/base"
+require "logstash/codecs/line"
+
+class LogStash::Codecs::EDN < LogStash::Codecs::Base
+  config_name "edn"
+
+  milestone 1
+
+  def register
+    require "edn"
+  end
+
+  public
+  def decode(data)
+    begin
+      yield LogStash::Event.new(EDN.read(data))
+    rescue
+      @logger.info("EDN parse failure. Falling back to plain-text", :error => e, :data => data)
+      yield LogStash::Event.new("message" => data)
+    end
+  end
+
+  public
+  def encode(data)
+    @on_event.call(data.to_hash.to_edn)
+  end
+
+end

data/lib/logstash/codecs/edn_lines.rb

@@ -0,0 +1,36 @@
+require "logstash/codecs/base"
+require "logstash/codecs/line"
+
+class LogStash::Codecs::EDNLines < LogStash::Codecs::Base
+  config_name "edn_lines"
+
+  milestone 1
+
+  def register
+    require "edn"
+  end
+
+  public
+  def initialize(params={})
+    super(params)
+    @lines = LogStash::Codecs::Line.new
+  end
+
+  public
+  def decode(data)
+    @lines.decode(data) do |event|
+      begin
+        yield LogStash::Event.new(EDN.read(event["message"]))
+      rescue => e
+        @logger.info("EDN parse failure. Falling back to plain-text", :error => e, :data => data)
+        yield LogStash::Event.new("message" => data)
+      end
+    end
+  end
+
+  public
+  def encode(data)
+    @on_event.call(data.to_hash.to_edn + "\n")
+  end
+
+end

data/lib/logstash/codecs/fluent.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/util/charset"
+
+# This codec handles fluentd's msgpack schema.
+#
+# For example, you can receive logs from fluent-logger-ruby with:
+#
+#     input {
+#       tcp {
+#         codec => fluent
+#         port => 4000
+#       }
+#     }
+#
+# And from your ruby code in your own application:
+#
+#     logger = Fluent::Logger::FluentLogger.new(nil, :host => "example.log", :port => 4000)
+#     logger.post("some_tag", { "your" => "data", "here" => "yay!" })
+#
+# Notes:
+#
+# * the fluent uses a second-precision time for events, so you will never see
+#   subsecond precision on events processed by this codec.
+#
+class LogStash::Codecs::Fluent < LogStash::Codecs::Base
+  config_name "fluent"
+  milestone 1
+
+  public
+  def register
+    require "msgpack"
+    @decoder = MessagePack::Unpacker.new
+  end
+
+  public
+  def decode(data)
+    @decoder.feed(data)
+    @decoder.each do |tag, epochtime, map|
+      event = LogStash::Event.new(map.merge(
+        "@timestamp" => Time.at(epochtime),
+        "tags" => tag
+      ))
+      yield event
+    end
+  end # def decode
+
+  public
+  def encode(event)
+    tag = event["tags"] || "log"
+    epochtime = event["@timestamp"].to_i
+    @on_event.call(MessagePack.pack([ tag, epochtime, event.to_hash ]))
+  end # def encode
+
+end # class LogStash::Codecs::Fluent

data/lib/logstash/codecs/graphite.rb

@@ -0,0 +1,114 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/codecs/line"
+require "json"
+
+# This codec will encode and decode Graphite formated lines.
+class LogStash::Codecs::Graphite < LogStash::Codecs::Base
+  config_name "graphite"
+
+  milestone 2
+
+  # The character encoding used in this codec. Examples include "UTF-8" and
+  # "CP1252"
+  #
+  # JSON requires valid UTF-8 strings, but in some cases, software that
+  # emits JSON does so in another encoding (nxlog, for example). In
+  # weird cases like this, you can set the charset setting to the
+  # actual encoding of the text and logstash will convert it for you.
+  #
+  # For nxlog users, you'll want to set this to "CP1252"
+  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
+  
+  EXCLUDE_ALWAYS = [ "@timestamp", "@version" ]
+
+  DEFAULT_METRICS_FORMAT = "*"
+  METRIC_PLACEHOLDER = "*"
+
+  # The metric(s) to use. This supports dynamic strings like %{host}
+  # for metric names and also for values. This is a hash field with key 
+  # of the metric name, value of the metric value. Example:
+  #
+  #     [ "%{host}/uptime", "%{uptime_1m}" ]
+  #
+  # The value will be coerced to a floating point value. Values which cannot be
+  # coerced will zero (0)
+  config :metrics, :validate => :hash, :default => {}
+
+  # Indicate that the event @fields should be treated as metrics and will be sent as is to graphite
+  config :fields_are_metrics, :validate => :boolean, :default => false
+
+  # Include only regex matched metric names
+  config :include_metrics, :validate => :array, :default => [ ".*" ]
+
+  # Exclude regex matched metric names, by default exclude unresolved %{field} strings
+  config :exclude_metrics, :validate => :array, :default => [ "%\{[^}]+\}" ]
+
+  # Defines format of the metric string. The placeholder '*' will be
+  # replaced with the name of the actual metric.
+  #
+  #     metrics_format => "foo.bar.*.sum"
+  #
+  # NOTE: If no metrics_format is defined the name of the metric will be used as fallback.
+  config :metrics_format, :validate => :string, :default => DEFAULT_METRICS_FORMAT
+  
+  
+  public
+  def initialize(params={})
+    super(params)
+    @lines = LogStash::Codecs::Line.new
+  end
+  
+  public
+  def decode(data)
+    @lines.decode(data) do |event|
+      name, value, time = event["message"].split(" ")
+      yield LogStash::Event.new(name => value.to_f, "@timestamp" => Time.at(time.to_i).gmtime)
+    end # @lines.decode
+  end # def decode
+
+  private
+  def construct_metric_name(metric)
+    if @metrics_format
+      return @metrics_format.gsub(METRIC_PLACEHOLDER, metric)
+    end
+
+    return metric
+  end
+  
+  public
+  def encode(event)
+    # Graphite message format: metric value timestamp\n
+
+    messages = []
+    timestamp = event.sprintf("%{+%s}")
+
+    if @fields_are_metrics
+      @logger.debug("got metrics event", :metrics => event.to_hash)
+      event.to_hash.each do |metric,value|
+        next if EXCLUDE_ALWAYS.include?(metric)
+        next unless @include_metrics.empty? || @include_metrics.any? { |regexp| metric.match(regexp) }
+        next if @exclude_metrics.any? {|regexp| metric.match(regexp)}
+        messages << "#{construct_metric_name(metric)} #{event.sprintf(value.to_s).to_f} #{timestamp}"
+      end # data.to_hash.each
+    else
+      @metrics.each do |metric, value|
+        @logger.debug("processing", :metric => metric, :value => value)
+        metric = event.sprintf(metric)
+        next unless @include_metrics.any? {|regexp| metric.match(regexp)}
+        next if @exclude_metrics.any? {|regexp| metric.match(regexp)}
+        messages << "#{construct_metric_name(event.sprintf(metric))} #{event.sprintf(value).to_f} #{timestamp}"
+      end # @metrics.each
+    end # if @fields_are_metrics
+
+    if messages.empty?
+      @logger.debug("Message is empty, not emiting anything.", :messages => messages)
+    else
+      message = messages.join("\n") + "\n"
+      @logger.debug("Emiting carbon messages", :messages => messages)
+
+      @on_event.call(message)
+    end # if messages.empty?
+  end # def encode
+
+end # class LogStash::Codecs::JSON

data/lib/logstash/codecs/json.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/codecs/line"
+require "json"
+
+# The codec should be used to decode full json messages.
+# If you are streaming JSON messages delimited by '\n' then 
+# see the json_lines codec.
+# Encoding will result in a single json string.
+class LogStash::Codecs::JSON < LogStash::Codecs::Base
+  config_name "json"
+
+  milestone 3
+
+  # The character encoding used in this codec. Examples include "UTF-8" and
+  # "CP1252"
+  #
+  # JSON requires valid UTF-8 strings, but in some cases, software that
+  # emits JSON does so in another encoding (nxlog, for example). In
+  # weird cases like this, you can set the charset setting to the
+  # actual encoding of the text and logstash will convert it for you.
+  #
+  # For nxlog users, you'll want to set this to "CP1252"
+  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
+  
+  public
+  def decode(data)
+    begin
+      yield LogStash::Event.new(JSON.parse(data))
+    rescue JSON::ParserError => e
+      @logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => data)
+      yield LogStash::Event.new("message" => data)
+    end
+  end # def decode
+
+  public
+  def encode(data)
+    @on_event.call(data.to_json)
+  end # def encode
+
+end # class LogStash::Codecs::JSON

data/lib/logstash/codecs/json_lines.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/codecs/line"
+require "json"
+
+# This codec will decode streamed JSON that is newline delimited.
+# For decoding JSON payload in the redis input for example, use the json codec instead.
+# Encoding will emit a single JSON string ending in a '\n'
+class LogStash::Codecs::JSONLines < LogStash::Codecs::Base
+  config_name "json_lines"
+
+  milestone 3
+
+  # The character encoding used in this codec. Examples include "UTF-8" and
+  # "CP1252"
+  #
+  # JSON requires valid UTF-8 strings, but in some cases, software that
+  # emits JSON does so in another encoding (nxlog, for example). In
+  # weird cases like this, you can set the charset setting to the
+  # actual encoding of the text and logstash will convert it for you.
+  #
+  # For nxlog users, you'll want to set this to "CP1252"
+  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
+
+  public
+  def initialize(params={})
+    super(params)
+    @lines = LogStash::Codecs::Line.new
+    @lines.charset = @charset
+  end
+  
+  public
+  def decode(data)
+
+    @lines.decode(data) do |event|
+      begin
+        yield LogStash::Event.new(JSON.parse(event["message"]))
+      rescue JSON::ParserError => e
+        @logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => data)
+        yield LogStash::Event.new("message" => data)
+      end
+    end
+  end # def decode
+
+  public
+  def encode(data)
+    # Tack on a \n for now because previously most of logstash's JSON
+    # outputs emitted one per line, and whitespace is OK in json.
+    @on_event.call(data.to_json + "\n")
+  end # def encode
+
+end # class LogStash::Codecs::JSON