logstash-logger-yajl 0.27.0

data/lib/logstash-logger/silenced_logging.rb

@@ -0,0 +1,83 @@
+# Adapted from:
+# https://github.com/rails/activerecord-session_store/blob/master/lib/active_record/session_store/extension/logger_silencer.rb
+# https://github.com/rails/rails/pull/16885
+
+require 'thread'
+
+# Add support for Rails-style logger silencing. Thread-safe and no dependencies.
+#
+# Setup:
+#   logger = Logger.new(STDOUT)
+#   logger.extend(LogStashLogger::SilencedLogging)
+#
+# Usage:
+#
+#   logger.silence do
+#     ...
+#   end
+#
+module LogStashLogger
+  module SilencedLogging
+    def self.extended(logger)
+      class << logger
+        attr_accessor :silencer
+        alias_method :level_without_thread_safety, :level
+        alias_method :level, :level_with_thread_safety
+        alias_method :add_without_thread_safety, :add
+        alias_method :add, :add_with_thread_safety
+
+        Logger::Severity.constants.each do |severity|
+          instance_eval <<-EOT, __FILE__, __LINE__ + 1
+            def #{severity.downcase}?                # def debug?
+              Logger::#{severity} >= level           #   DEBUG >= level
+            end                                      # end
+          EOT
+        end
+      end
+
+      logger.instance_eval do
+        self.silencer = true
+      end
+    end
+
+    def thread_level
+      Thread.current[thread_hash_level_key]
+    end
+
+    def thread_level=(level)
+      Thread.current[thread_hash_level_key] = level
+    end
+
+    def level_with_thread_safety
+      thread_level || level_without_thread_safety
+    end
+
+    def add_with_thread_safety(severity, message = nil, progname = nil, &block)
+      if (defined?(@logdev) && @logdev.nil?) || (severity || UNKNOWN) < level
+        true
+      else
+        add_without_thread_safety(severity, message, progname, &block)
+      end
+    end
+
+    # Silences the logger for the duration of the block.
+    def silence(temporary_level = Logger::ERROR)
+      if silencer
+        begin
+          self.thread_level = temporary_level
+          yield self
+        ensure
+          self.thread_level = nil
+        end
+      else
+        yield self
+      end
+    end
+
+    private
+
+    def thread_hash_level_key
+      @thread_hash_level_key ||= :"ThreadSafeLogger##{object_id}@level"
+    end
+  end
+end

data/lib/logstash-logger/tagged_logging.rb

@@ -0,0 +1,40 @@
+module LogStashLogger
+  # Shamelessly copied from ActiveSupport::TaggedLogging
+  module TaggedLogging
+    def tagged(*tags)
+      formatter.tagged(*tags) { yield self }
+    end
+
+    def flush
+      formatter.clear_tags!
+      super if defined?(super)
+    end
+
+    module Formatter
+      def tagged(*tags)
+        new_tags = push_tags(*tags)
+        yield self
+      ensure
+        pop_tags(new_tags.size)
+      end
+
+      def push_tags(*tags)
+        tags.flatten.reject{ |t| t.nil? || t.empty? }.tap do |new_tags|
+          current_tags.concat new_tags
+        end
+      end
+
+      def pop_tags(size = 1)
+        current_tags.pop size
+      end
+
+      def clear_tags!
+        current_tags.clear
+      end
+
+      def current_tags
+        Thread.current[:logstash_logger_tags] ||= []
+      end
+    end
+  end
+end

data/lib/logstash-logger/version.rb

@@ -0,0 +1,3 @@
+module LogStashLogger
+  VERSION = "0.27.0"
+end

data/lib/logstash/event.rb

@@ -0,0 +1,272 @@
+require "yajl"
+require "time"
+require "date"
+require "logstash/namespace"
+require "logstash/util/fieldreference"
+
+# Use a custom serialization for jsonifying Time objects.
+# TODO(sissel): Put this in a separate file.
+class Time
+  def to_json(*args)
+    return iso8601(3).to_json(*args)
+  end
+
+  def inspect
+    return to_json
+  end
+end
+
+# the logstash event object.
+#
+# An event is simply a tuple of (timestamp, data).
+# The 'timestamp' is an ISO8601 timestamp. Data is anything - any message,
+# context, references, etc that are relevant to this event.
+#
+# Internally, this is represented as a hash with only two guaranteed fields.
+#
+# * "@timestamp" - an ISO8601 timestamp representing the time the event
+#   occurred at.
+# * "@version" - the version of the schema. Currently "1"
+#
+# They are prefixed with an "@" symbol to avoid clashing with your
+# own custom fields.
+#
+# When serialized, this is represented in JSON. For example:
+#
+#     {
+#       "@timestamp": "2013-02-09T20:39:26.234Z",
+#       "@version": "1",
+#       message: "hello world"
+#     }
+class LogStash::Event
+  class DeprecatedMethod < StandardError; end
+
+  public
+  def initialize(data={})
+    @cancelled = false
+
+    @data = data
+    if data.include?("@timestamp")
+      t = data["@timestamp"]
+      if t.is_a?(String)
+        data["@timestamp"] = Time.parse(t).gmtime
+      end
+    else
+      data["@timestamp"] = ::Time.now.utc
+    end
+    data["@version"] = "1" if !@data.include?("@version")
+  end # def initialize
+
+  # Add class methods on inclusion.
+  public
+  def self.included(klass)
+    klass.extend(ClassMethods)
+  end # def included
+
+  module ClassMethods
+    public
+    def from_json(json)
+      return self.new(Yajl.load(json))
+    end # def from_json
+  end
+
+  public
+  def cancel
+    @cancelled = true
+  end # def cancel
+
+  public
+  def uncancel
+    @cancelled = false
+  end # def uncancel
+
+  public
+  def cancelled?
+    return @cancelled
+  end # def cancelled?
+
+  # Create a deep-ish copy of this event.
+  public
+  def clone
+    copy = {}
+    @data.each do |k,v|
+      # TODO(sissel): Recurse if this is a hash/array?
+      copy[k] = v.clone
+    end
+    return self.class.new(copy)
+  end # def clone
+
+  if RUBY_ENGINE == "jruby"
+    public
+    def to_s
+      return self.sprintf("%{+yyyy-MM-dd'T'HH:mm:ss.SSSZ} %{host} %{message}")
+    end # def to_s
+  else
+    public
+    def to_s
+      return self.sprintf("#{self["@timestamp"].iso8601} %{host} %{message}")
+    end # def to_s
+  end
+
+  public
+  def timestamp; return @data["@timestamp"]; end # def timestamp
+  def timestamp=(val); return @data["@timestamp"] = val; end # def timestamp=
+
+  def unix_timestamp
+    raise DeprecatedMethod
+  end # def unix_timestamp
+
+  def ruby_timestamp
+    raise DeprecatedMethod
+  end # def unix_timestamp
+
+  # field-related access
+  public
+  def [](str)
+    if str[0,1] == "+"
+    else
+      return LogStash::Util::FieldReference.exec(str, @data)
+    end
+  end # def []
+
+  public
+  def []=(str, value)
+    r = LogStash::Util::FieldReference.exec(str, @data) do |obj, key|
+      obj[key] = value
+    end
+
+    # The assignment can fail if the given field reference (str) does not exist
+    # In this case, we'll want to set the value manually.
+    if r.nil?
+      # TODO(sissel): Implement this in LogStash::Util::FieldReference
+      if str[0,1] != "["
+        return @data[str] = value
+      end
+
+      # No existing element was found, so let's set one.
+      *parents, key = str.scan(/(?<=\[)[^\]]+(?=\])/)
+      obj = @data
+      parents.each do |p|
+        if obj.include?(p)
+          obj = obj[p]
+        else
+          obj[p] = {}
+          obj = obj[p]
+        end
+      end
+      obj[key] = value
+    end
+    return value
+  end # def []=
+
+  public
+  def fields
+    raise DeprecatedMethod
+  end
+
+  public
+  def to_json(*args)
+    return Yajl.dump(@data, *args)
+  end # def to_json
+
+  def to_hash
+    return @data
+  end # def to_hash
+
+  public
+  def overwrite(event)
+    @data = event.to_hash
+  end
+
+  public
+  def include?(key)
+    return !self[key].nil?
+  end # def include?
+
+  # Append an event to this one.
+  public
+  def append(event)
+    # non-destructively merge that event with ourselves.
+    LogStash::Util.hash_merge(@data, event.to_hash)
+  end # append
+
+  # Remove a field or field reference. Returns the value of that field when
+  # deleted
+  public
+  def remove(str)
+    return LogStash::Util::FieldReference.exec(str, @data) do |obj, key|
+      next obj.delete(key)
+    end
+  end # def remove
+
+  # sprintf. This could use a better method name.
+  # The idea is to take an event and convert it to a string based on
+  # any format values, delimited by %{foo} where 'foo' is a field or
+  # metadata member.
+  #
+  # For example, if the event has type == "foo" and source == "bar"
+  # then this string:
+  #   "type is %{type} and source is %{host}"
+  # will return
+  #   "type is foo and source is bar"
+  #
+  # If a %{name} value is an array, then we will join by ','
+  # If a %{name} value does not exist, then no substitution occurs.
+  #
+  # TODO(sissel): It is not clear what the value of a field that
+  # is an array (or hash?) should be. Join by comma? Something else?
+  public
+  def sprintf(format)
+    format = format.to_s
+    if format.index("%").nil?
+      return format
+    end
+
+    return format.gsub(/%\{[^}]+\}/) do |tok|
+      # Take the inside of the %{ ... }
+      key = tok[2 ... -1]
+
+      if key == "+%s"
+        # Got %{+%s}, support for unix epoch time
+        next @data["@timestamp"].to_i
+      elsif key[0,1] == "+"
+        t = @data["@timestamp"]
+        formatter = org.joda.time.format.DateTimeFormat.forPattern(key[1 .. -1])\
+          .withZone(org.joda.time.DateTimeZone::UTC)
+        #next org.joda.time.Instant.new(t.tv_sec * 1000 + t.tv_usec / 1000).toDateTime.toString(formatter)
+        # Invoke a specific Instant constructor to avoid this warning in JRuby
+        #  > ambiguous Java methods found, using org.joda.time.Instant(long)
+        org.joda.time.Instant.java_class.constructor(Java::long).new_instance(
+          t.tv_sec * 1000 + t.tv_usec / 1000
+        ).to_java.toDateTime.toString(formatter)
+      else
+        value = self[key]
+        case value
+          when nil
+            tok # leave the %{foo} if this field does not exist in this event.
+          when Array
+            value.join(",") # Join by ',' if value is an array
+          when Hash
+            value.to_json # Convert hashes to json
+          else
+            value # otherwise return the value
+        end # case value
+      end # 'key' checking
+    end # format.gsub...
+  end # def sprintf
+
+  # Shims to remove after event v1 is the default.
+  def tags=(value); self["tags"] = value; end
+  def tags; return self["tags"]; end
+  def message=(value); self["message"] = value; end
+  def source=(value); self["source"] = value; end
+  def type=(value); self["type"] = value; end
+  def type; return self["type"]; end
+  def fields; return self.to_hash; end
+
+  def tag(value)
+    # Generalize this method for more usability
+    self["tags"] ||= []
+    self["tags"] << value unless self["tags"].include?(value)
+  end
+end # class LogStash::Event

data/lib/logstash/namespace.rb

@@ -0,0 +1,15 @@
+#$: << File.join(File.dirname(__FILE__), "..", "..", "vendor", "bundle")
+
+module LogStash
+  module Inputs; end
+  module Outputs; end
+  module Filters; end
+  module Search; end
+  module Config; end
+  module File; end
+  module Web; end
+  module Util; end
+  module PluginMixins; end
+
+  SHUTDOWN = :shutdown
+end # module LogStash

data/lib/logstash/util.rb

@@ -0,0 +1,105 @@
+require "logstash/namespace"
+
+module LogStash::Util
+  UNAME = case RbConfig::CONFIG["host_os"]
+    when /^linux/; "linux"
+    else; RbConfig::CONFIG["host_os"]
+  end
+
+  PR_SET_NAME = 15
+  def self.set_thread_name(name)
+    if RUBY_ENGINE == "jruby"
+      # Keep java and ruby thread names in sync.
+      Java::java.lang.Thread.currentThread.setName(name)
+    end
+    Thread.current[:name] = name
+
+    if UNAME == "linux"
+      require "logstash/util/prctl"
+      # prctl PR_SET_NAME allows up to 16 bytes for a process name
+      # since MRI 1.9, JRuby, and Rubinius use system threads for this.
+      LibC.prctl(PR_SET_NAME, name[0..16], 0, 0, 0)
+    end
+  end # def set_thread_name
+
+  # Merge hash 'src' into 'dst' nondestructively
+  #
+  # Duplicate keys will become array values
+  #
+  # [ src["foo"], dst["foo"] ]
+  def self.hash_merge(dst, src)
+    src.each do |name, svalue|
+      if dst.include?(name)
+        dvalue = dst[name]
+        if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
+          dvalue = hash_merge(dvalue, svalue)
+        elsif svalue.is_a?(Array)
+          if dvalue.is_a?(Array)
+            # merge arrays without duplicates.
+            dvalue |= svalue
+          else
+            dvalue = [dvalue] | svalue
+          end
+        else
+          if dvalue.is_a?(Array)
+            dvalue << svalue unless dvalue.include?(svalue)
+          else
+            dvalue = [dvalue, svalue] unless dvalue == svalue
+          end
+        end
+
+        dst[name] = dvalue
+      else
+        # dst doesn't have this key, just set it.
+        dst[name] = svalue
+      end
+    end
+
+    return dst
+  end # def self.hash_merge
+
+  # Merge hash 'src' into 'dst' nondestructively
+  #
+  # Duplicate keys will become array values
+  # Arrays merged will simply be appended.
+  #
+  # [ src["foo"], dst["foo"] ]
+  def self.hash_merge_with_dups(dst, src)
+    src.each do |name, svalue|
+      if dst.include?(name)
+        dvalue = dst[name]
+        if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
+          dvalue = hash_merge(dvalue, svalue)
+        elsif svalue.is_a?(Array)
+          if dvalue.is_a?(Array)
+            # merge arrays without duplicates.
+            dvalue += svalue
+          else
+            dvalue = [dvalue] + svalue
+          end
+        else
+          if dvalue.is_a?(Array)
+            dvalue << svalue unless dvalue.include?(svalue)
+          else
+            dvalue = [dvalue, svalue] unless dvalue == svalue
+          end
+        end
+
+        dst[name] = dvalue
+      else
+        # dst doesn't have this key, just set it.
+        dst[name] = svalue
+      end
+    end
+
+    return dst
+  end # def self.hash_merge
+
+  def self.hash_merge_many(*hashes)
+    dst = {}
+    hashes.each do |hash|
+      hash_merge_with_dups(dst, hash)
+    end
+    return dst
+  end # def hash_merge_many
+end # module LogStash::Util