logstash-lite 0.2.20110206003603 → 0.2.20110329105411

data/bin/logstash-test

@@ -36,6 +36,8 @@ def check_libraries
             "needed for websocket output")
   results << check_lib("rack", "rack", true,
             "needed for logstash-web")
+  results << check_lib("thin", "thin", true,
+            "needed for logstash-web")
   results << check_lib("amqp", "amqp", true,
             "needed for AMQP input and output")
   results << check_lib("sinatra/async", "async_sinatra", true,
@@ -46,6 +48,8 @@ def check_libraries
             "improve logstash debug logging output")
   results << check_lib("eventmachine", "eventmachine", false,
             "required for logstash to function")
+  results << check_lib("json", "json", false,
+            "required for logstash to function")
 
   missing_required = results.count { |r| !r[:optional] and !r[:found] }
   if missing_required == 0
@@ -66,6 +70,8 @@ end
 
 def main(args)
   report_ruby_version
+  # TODO(sissel): Add a way to call out specific things to test, like
+  # logstash-web, elasticsearch, mongodb, syslog, etc.
   if !check_libraries
     puts "Library check failed."
     return 1

data/lib/logstash/filters/grok.rb

@@ -50,9 +50,21 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
 
     if match
       match.each_capture do |key, value|
+        match_type = nil
         if key.include?(":")
-          key = key.split(":")[1]
+          name, key, match_type = key.split(":")
         end
+
+        # http://code.google.com/p/logstash/issues/detail?id=45
+        # Permit typing of captures by giving an additional colon and a type,
+        # like: %{FOO:name:int} for int coercion.
+        case match_type
+          when "int"
+            value = value.to_i
+          when "float"
+            value = value.to_f
+        end
+
         if event.message == value
           # Skip patterns that match the entire line
           @logger.debug("Skipping capture '#{key}' since it matches the whole line.")
@@ -65,7 +77,9 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
           event.fields[key] = []
         end
 
-        if value && !value.empty?
+        # If value is not nil, or responds to empty and is not empty, add the
+        # value to the event.
+        if !value.nil? && (!value.empty? rescue true)
           event.fields[key] << value
         end
       end

data/lib/logstash/inputs/amqp.rb

@@ -3,9 +3,10 @@ require "logstash/inputs/base"
 require "logstash/namespace"
 require "mq" # rubygem 'amqp'
 require "uuidtools" # rubygem 'uuidtools'
+require "cgi"
 
 class LogStash::Inputs::Amqp < LogStash::Inputs::Base
-  MQTYPES = [ "fanout", "queue", "topic" ]
+  MQTYPES = [ "fanout", "direct", "topic" ]
 
   public
   def initialize(url, type, config={}, &block)
@@ -13,39 +14,47 @@ class LogStash::Inputs::Amqp < LogStash::Inputs::Base
 
     @mq = nil
 
-    # Handle path /<type>/<name>
-    unused, @mqtype, @name = @url.path.split("/", 3)
-    if @mqtype == nil or @name == nil
-      raise "amqp urls must have a path of /<type>/name where <type> is #{MQTYPES.join(", ")}"
+    # Handle path /<vhost>/<type>/<name> or /<type>/<name>
+    # vhost allowed to contain slashes
+    if @url.path =~ %r{^/((.*)/)?([^/]+)/([^/]+)}
+      unused, @vhost, @mqtype, @name = $~.captures
+    else
+      raise "amqp urls must have a path of /<type>/name or /vhost/<type>/name where <type> is #{MQTYPES.join(", ")}"
     end
 
     if !MQTYPES.include?(@mqtype)
-      raise "Invalid type '#{@mqtype}' must be one of #{MQTYPES.JOIN(", ")}"
+      raise "Invalid type '#{@mqtype}' must be one of #{MQTYPES.join(", ")}"
     end
   end # def initialize
 
   public
   def register
     @logger.info("Registering input #{@url}")
+    query_args = @url.query ? CGI.parse(@url.query) : {}
     amqpsettings = {
+      :vhost => (@vhost or "/"),
       :host => @url.host,
       :port => (@url.port or 5672),
     }
     amqpsettings[:user] = @url.user if @url.user
     amqpsettings[:pass] = @url.password if @url.password
+    amqpsettings[:logging] = query_args.include? "debug"
+    queue_name = ((@urlopts["queue"].nil? or @urlopts["queue"].empty?) ? "logstash-#{@name}" : @urlopts["queue"])
+    @logger.debug("Connecting with AMQP settings #{amqpsettings.inspect} to set up #{@mqtype.inspect} queue #{queue_name} on exchange #{@name.inspect}")
     @amqp = AMQP.connect(amqpsettings)
     @mq = MQ.new(@amqp)
     @target = nil
 
-    @target = @mq.queue(UUIDTools::UUID.timestamp_create)
+    @durable_exchange = @urlopts["durable_exchange"] ? true : false
+    @durable_queue = @urlopts["durable_queue"] ? true : false
+    @target = @mq.queue(queue_name, :durable => @durable_queue)
     case @mqtype
       when "fanout"
-        #@target.bind(MQ.fanout(@url.path, :durable => true))
-        @target.bind(@mq.fanout(@name))
+        @target.bind(@mq.fanout(@name, :durable => @durable_exchange))
       when "direct"
-        @target.bind(@mq.direct(@name))
+        @target.bind(@mq.direct(@name, :durable => @durable_exchange))
       when "topic"
-        @target.bind(@mq.topic(@name))
+        @target.bind(@mq.topic(@name, :durable => @durable_exchange))
     end # case @mqtype
 
     @target.subscribe(:ack => true) do |header, message|

data/lib/logstash/namespace.rb

@@ -2,4 +2,5 @@ module LogStash
   module Inputs; end
   module Outputs; end
   module Filters; end
+  module Search; end
 end # module LogStash

data/lib/logstash/outputs/amqp.rb

@@ -2,52 +2,72 @@ require "amqp" # rubygem 'amqp'
 require "logstash/outputs/base"
 require "logstash/namespace"
 require "mq" # rubygem 'amqp'
+require "cgi"
 
 class LogStash::Outputs::Amqp < LogStash::Outputs::Base
-  MQTYPES = [ "fanout", "queue", "topic" ]
+  MQTYPES = [ "fanout", "direct", "topic" ]
 
   public
   def initialize(url, config={}, &block)
     super
 
-    # Handle path /<type>/<name>
-    unused, @mqtype, @name = @url.path.split("/", 3)
-    if @mqtype == nil or @name == nil
-      raise "amqp urls must have a path of /<type>/name where <type> is #{MQTYPES.join(", ")}"
+    @mq = nil
+    @bulk_prefix = nil
+
+    # Handle path /<vhost>/<type>/<name> or /<type>/<name>
+    # vhost allowed to contain slashes
+    if @url.path =~ %r{^/((.*)/)?([^/]+)/([^/]+)}
+      unused, @vhost, @mqtype, @name = $~.captures
+    else
+      raise "amqp urls must have a path of /<type>/name or /vhost/<type>/name where <type> is #{MQTYPES.join(", ")}"
     end
 
     if !MQTYPES.include?(@mqtype)
-      raise "Invalid type '#{@mqtype}' must be one #{MQTYPES.join(", ")}"
+      raise "Invalid type '#{@mqtype}' must be one of #{MQTYPES.join(", ")}"
     end
   end # def initialize
 
   public
   def register
     @logger.info("Registering output #{@url}")
+    query_args = @url.query ? CGI.parse(@url.query) : {}
     amqpsettings = {
+      :vhost => (@vhost or "/"),
       :host => @url.host,
       :port => (@url.port or 5672),
     }
     amqpsettings[:user] = @url.user if @url.user
     amqpsettings[:pass] = @url.password if @url.password
+    amqpsettings[:logging] = query_args.include? "debug"
+    @logger.debug("Connecting with AMQP settings #{amqpsettings.inspect} to set up #{@mqtype.inspect} exchange #{@name.inspect}")
     @amqp = AMQP.connect(amqpsettings)
     @mq = MQ.new(@amqp)
     @target = nil
 
+    if @urlopts.include? "es_index" and @urlopts.include? "es_type"
+      @bulk_prefix = { "index" => { "_index" => @urlopts["es_index"], "_type" => @urlopts["es_type"] } }.to_json + "\n"
+      @logger.debug "Preparing ElasticSearch bulk API header for injection: #{@bulk_prefix.inspect}"
+    end
+
+    @durable = @urlopts["durable"] ? true : false
     case @mqtype
       when "fanout"
-        @target = @mq.fanout(@name)
-      when "queue"
-        @target = @mq.queue(@name, :durable => @urlopts["durable"] ? true : false)
+        @target = @mq.fanout(@name, :durable => @durable)
+      when "direct"
+        @target = @mq.direct(@name, :durable => @durable)
       when "topic"
-        @target = @mq.topic(@name)
+        @target = @mq.topic(@name, :durable => @durable)
     end # case @mqtype
   end # def register
 
   public
   def receive(event)
     @logger.debug(["Sending event", { :url => @url, :event => event }])
-    @target.publish(event.to_json)
+    if @bulk_prefix
+      @target.publish(@bulk_prefix + event.to_json + "\n")
+    else
+      @target.publish(event.to_json)
+    end
   end # def receive
 
   # This is used by the ElasticSearch AMQP/River output.

data/lib/logstash/outputs/elasticsearch.rb

@@ -2,6 +2,7 @@ require "em-http-request"
 require "logstash/namespace"
 require "logstash/outputs/amqp"
 require "logstash/outputs/base"
+require "cgi"
 
 class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
   public
@@ -41,6 +42,9 @@ class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
       }, # "settings"
     } # ES Index
 
+    #puts :waiting
+    puts @esurl.to_s
+    #sleep 10
     indexurl = @esurl.to_s
     indexmap_http = EventMachine::HttpRequest.new(indexurl)
     indexmap_req = indexmap_http.put :body => indexmap.to_json
@@ -49,41 +53,61 @@ class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
       ready(params)
     end
     indexmap_req.errback do
-      @logger.warn(["Failure configuring index", @esurl.to_s, indexmap])
+      @logger.warn(["Failure configuring index (http failed to connect?)",
+                    @esurl.to_s, indexmap])
+      @logger.warn([indexmap_req])
+      #sleep 30
       raise "Failure configuring index: #{@esurl.to_s}"
+      
     end
   end # def register
 
   public
   def ready(params)
-    case params["method"]
+    method = params.delete("method")
+    case method
     when "http"
       @logger.debug "ElasticSearch using http with URL #{@url.to_s}"
       @http = EventMachine::HttpRequest.new(@url.to_s)
       @callback = self.method(:receive_http)
     when "river"
-      params["port"] ||= 5672
-      auth = "#{params["user"] or "guest"}:#{params["pass"] or "guest"}"
-      mq_url = URI::parse("amqp://#{auth}@#{params["host"]}:#{params["port"]}/queue/#{params["queue"]}?durable=1")
+      river_type = params.delete("type") || "rabbitmq"
+      amqp_host = params.delete("host") || 'localhost'
+      amqp_port = params.delete("port") || 5672
+      amqp_exchange_type = params.delete("exchange_type") || "direct"
+      amqp_queue_name = params.delete("queue") || "es"
+      amqp_exchange_name = params.delete("exchange") || amqp_queue_name
+      amqp_exchange_durable = (params["durable"] || "false") =~ /^[ty1]/
+      amqp_user = params.delete("user") or "guest"
+      amqp_pass = params.delete("pass") or "guest"
+      amqp_vhost = params.delete("vhost") || "/"
+      vhost_str = (amqp_vhost == "/") ? "" : "/#{amqp_vhost}"
+      qs = params.map {|k,v| "#{CGI.escape(k)}=#{CGI.escape(v)}"}.join("&")
+      mq_url = URI::parse("amqp://#{amqp_user}:#{amqp_pass}@#{amqp_host}:#{amqp_port}#{vhost_str}/#{amqp_exchange_type}/#{amqp_exchange_name}?#{qs}")
       @mq = LogStash::Outputs::Amqp.new(mq_url.to_s)
       @mq.register
       @callback = self.method(:receive_river)
       em_url = URI.parse("http://#{@url.host}:#{@url.port}/_river/logstash#{@url.path.tr("/", "_")}/_meta")
       unused, @es_index, @es_type = @url.path.split("/", 3)
 
-      river_config = {"type" => params["type"],
-                      params["type"] => {"host" => params["host"],
-                                         "user" => params["user"],
-                                         "port" => params["port"],
-                                         "pass" => params["pass"],
-                                         "vhost" => params["vhost"],
-                                         "queue" => params["queue"],
-                                         "exchange" => params["queue"],
-                                        },
-                     "index" => {"bulk_size" => 100,
-                                 "bulk_timeout" => "10ms",
-                                },
-                     }
+      river_config = {
+        "type" => river_type,
+        river_type => {
+          "host" => amqp_host,
+          "user" => amqp_user,
+          "port" => amqp_port,
+          "pass" => amqp_pass,
+          "vhost" => amqp_vhost,
+          "queue" => amqp_queue_name,
+          "exchange" => amqp_exchange_name,
+          "exchange_durable" => amqp_exchange_durable ? "true" : "false",
+          "exchange_type" => amqp_exchange_type,
+        },
+        "index" => {
+          "bulk_size" => 100,
+          "bulk_timeout" => "10ms",
+        },
+      }
       @logger.debug(["ElasticSearch using river", river_config])
       http_setup = EventMachine::HttpRequest.new(em_url.to_s)
       req = http_setup.put :body => river_config.to_json
@@ -91,7 +115,7 @@ class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
         @logger.warn "Error setting up river: #{req.response}"
       end
       @callback = self.method(:receive_river)
-    else raise "unknown elasticsearch method #{params["method"].inspect}"
+    else raise "unknown elasticsearch method #{method.inspect}"
     end
 
     #receive(LogStash::Event.new({

data/lib/logstash/search/base.rb

@@ -0,0 +1,39 @@
+
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/event"
+
+class LogStash::Search::Base
+  # Do a search. 
+  #
+  # This method is async. You can expect a block and therefore
+  # should yield a result, not return one.
+  # 
+  # Implementations should yield a LogStash::Search::Result
+  # LogStash::Search::Result#events must be an array of LogStash::Event
+  def search(query)
+    raise "The class #{self.class.name} must implement the 'search' method."
+  end # def search
+
+  # Yields a histogram by field of a query.
+  #
+  # This method is async. You should expect a block to be passed and therefore
+  # should yield a result, not return one.
+  #
+  # Implementations should yield a LogStash::Search::FacetResult::Histogram
+  def histogram(query, field, interval=nil)
+    raise "The class #{self.class.name} must implement the 'histogram' method."
+  end
+
+  # Returns a list of popular terms from a query
+  # TODO(sissel): Implement
+  def popular_terms(query, fields, count=10)
+    raise "The class #{self.class.name} must implement the 'popular_terms' method."
+  end
+
+  # Count the results given by a query.
+  def count(query)
+    raise "The class #{self.class.name} must implement the 'count' method."
+  end
+
+end # class LogStash::Search::Base

data/lib/logstash/search/elasticsearch.rb

@@ -0,0 +1,196 @@
+
+require "em-http-request"
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/event"
+require "logstash/search/base"
+require "logstash/search/query"
+require "logstash/search/result"
+require "logstash/search/facetresult"
+require "logstash/search/facetresult/histogram"
+ 
+class LogStash::Search::ElasticSearch < LogStash::Search::Base
+  public
+  def initialize(settings={})
+    @host = (settings[:host] || "localhost")
+    @port = (settings[:port] || 9200).to_i
+    @logger = LogStash::Logger.new(STDOUT)
+  end
+
+  # See LogStash::Search;:Base#search
+  public
+  def search(query)
+    raise "No block given for search call." if !block_given?
+    if query.is_a?(String)
+      query = LogStash::Search::Query.parse(query)
+    end
+
+    # TODO(sissel): only search a specific index?
+    http = EventMachine::HttpRequest.new("http://#{@host}:#{@port}/_search")
+
+    @logger.info(["Query", query])
+    esreq = {
+      "sort" => [
+        { "@timestamp" => "desc" }
+      ],
+      "query" => {
+        "query_string" => { 
+           "query" => query.query_string,
+           "default_operator" => "AND"
+        } # query_string
+      }, # query
+      "from" => query.offset,
+      "size" => query.count
+    } # elasticsearch request
+
+    @logger.info("ElasticSearch Query: #{esreq.to_json}")
+    start_time = Time.now
+    req = http.get :body => esreq.to_json
+    result = LogStash::Search::Result.new
+    req.callback do
+      data = JSON.parse(req.response)
+      result.duration = Time.now - start_time
+
+      hits = data["hits"]["hits"] rescue nil
+
+      if hits.nil? or !data["error"].nil?
+        # Use the error message if any, otherwise, return the whole
+        # data object as json as the error message for debugging later.
+        result.error_message = (data["error"] rescue false) || data.to_json
+        yield result
+        next
+      end
+
+      @logger.info(["Got search results", 
+                   { :query => query.query_string, :duration => data["duration"],
+                     :result_count => hits.size }])
+      if req.response_header.status != 200
+        result.error_message = data["error"] || req.inspect
+        @error = data["error"] || req.inspect
+      end
+
+      # We want to yield a list of LogStash::Event objects.
+      hits.each do |hit|
+        result.events << LogStash::Event.new(hit["_source"])
+      end
+
+      # Total hits this search could find if not limited
+      result.total = data["hits"]["total"]
+      result.offset = query.offset
+
+      yield result
+    end
+
+    req.errback do 
+      @logger.warn(["Query failed", query, req, req.response])
+      result.duration = Time.now - start_time
+      result.error_message = req.response
+      #yield result
+
+      yield({ "error" => req.response })
+    end
+  end # def search
+
+  # See LogStash::Search;:Base#histogram
+  public
+  def histogram(query, field, interval=nil)
+    if query.is_a?(String)
+      query = LogStash::Search::Query.parse(query)
+    end
+
+    # TODO(sissel): only search a specific index?
+    http = EventMachine::HttpRequest.new("http://#{@host}:#{@port}/_search")
+
+    @logger.info(["Query", query])
+    histogram_settings = {
+      "field" => field
+    }
+
+    if !interval.nil? && interval.is_a?(Numeric)
+      histogram_settings["interval"] = interval
+    end
+
+    esreq = {
+      "query" => {
+        "query_string" => { 
+           "query" => query.query_string,
+           "default_operator" => "AND"
+        } # query_string
+      }, # query
+      "from" => 0,
+      "size" => 0,
+      "facets" => {
+        "amazingpants" => { # just a name for this histogram...
+          "histogram" => histogram_settings,
+        },
+      },
+    } # elasticsearch request
+
+    @logger.info("ElasticSearch Facet Query: #{esreq.to_json}")
+    start_time = Time.now
+    req = http.get :body => esreq.to_json
+    result = LogStash::Search::FacetResult.new
+    req.callback do
+      data = JSON.parse(req.response)
+      result.duration = Time.now - start_time
+
+      @logger.info(["Got search results", 
+                   { :query => query.query_string, :duration => data["duration"] }])
+      if req.response_header.status != 200
+        result.error_message = data["error"] || req.inspect
+        @error = data["error"] || req.inspect
+      end
+
+      entries = data["facets"]["amazingpants"]["entries"] rescue nil
+
+      if entries.nil? or !data["error"].nil?
+        # Use the error message if any, otherwise, return the whole
+        # data object as json as the error message for debugging later.
+        result.error_message = (data["error"] rescue false) || data.to_json
+        yield result
+        next
+      end
+      entries.each do |entry|
+        # entry is a hash of keys 'total', 'mean', 'count', and 'key'
+        hist_entry = LogStash::Search::FacetResult::Histogram.new
+        hist_entry.key = entry["key"]
+        hist_entry.count = entry["count"]
+        result.results << hist_entry
+      end # for each histogram result
+      yield result
+    end # request callback
+
+    req.errback do 
+      @logger.warn(["Query failed", query, req, req.response])
+      result.duration = Time.now - start_time
+      result.error_message = req.response
+      yield result
+      #yield({ "error" => req.response })
+    end
+  end
+
+  # Not used. Needs refactoring elsewhere.
+  private
+  def __anonymize
+    # TODO(sissel): Plugin-ify this (Search filters!)
+    # TODO(sissel): Implement
+    #  Search anonymization
+    #require "digest/md5"
+    #data["hits"]["hits"].each do |hit|
+    [].each do |hit|
+      event = LogStash::Event.new(hit["_source"])
+      event.to_hash.each do |key, value|
+        next unless value.is_a?(String)
+        value.gsub!(/[^ ]+\.loggly\.net/) { |match| "loggly-" + Digest::MD5.hexdigest(match)[0..6]  + ".example.com"}
+      end
+
+      event.fields.each do |key, value|
+        value = [value] if value.is_a?(String)
+        next unless value.is_a?(Array)
+        value.each do |v|
+          v.gsub!(/[^ ]+\.loggly\.net/) { |match| "loggly-" + Digest::MD5.hexdigest(match)[0..6]  + ".example.com"}
+        end # value.each
+      end # hit._source.@fields.each
+    end # data.hits.hits.each
+  end # def __anonymize
+end # class LogStash::Search::ElasticSearch