logstash-lite 0.2.20110206003603 → 0.2.20110329105411

data/lib/logstash/search/facetresult.rb

@@ -0,0 +1,25 @@
+
+require "logstash/namespace"
+require "logstash/logging"
+
+class LogStash::Search::FacetResult
+  # Array of LogStash::Search::FacetResult::Entry
+  attr_accessor :results
+
+  # How long this query took, in seconds (or fractions of).
+  attr_accessor :duration
+
+  # Error message, if any.
+  attr_accessor :error_message
+
+  def initialize(settings={})
+    @results = []
+    @duration = nil
+    @error_message = nil
+  end
+
+  def error?
+    return !@error_message.nil?
+  end
+end # class LogStash::Search::FacetResult
+

data/lib/logstash/search/facetresult/entry.rb

@@ -0,0 +1,6 @@
+
+require "logstash/search/facetresult"
+
+class LogStash::Search::FacetResult::Entry
+  # nothing here
+end # class LogStash::Search::FacetResult::Entry

data/lib/logstash/search/facetresult/histogram.rb

@@ -0,0 +1,21 @@
+
+require "json"
+require "logstash/search/facetresult/entry"
+
+class LogStash::Search::FacetResult::Histogram < LogStash::Search::FacetResult::Entry
+  # The name or key for this result.
+  attr_accessor :key
+  attr_accessor :mean
+  attr_accessor :total
+  attr_accessor :count
+
+  # sometimes a parent call to to_json calls us with args?
+  def to_json(*args)
+    return {
+      "key" => @key,
+      "mean" => @mean,
+      "total" => @total,
+      "count" => @count,
+    }.to_json
+  end
+end

data/lib/logstash/search/query.rb

@@ -0,0 +1,35 @@
+require "logstash/namespace"
+require "logstash/logging"
+
+class LogStash::Search::Query
+  # The query string
+  attr_accessor :query_string
+
+  # The offset to start at (like SQL's SELECT ... OFFSET n)
+  attr_accessor :offset
+
+  # The max number of results to return. (like SQL's SELECT ... LIMIT n)
+  attr_accessor :count
+
+  # New query object.
+  #
+  # 'settings' should be a hash containing:
+  # 
+  # * :query_string - a string query for searching
+  # * :offset - (optional, default 0) offset to search from
+  # * :count - (optional, default 50) max number of results to return
+  def initialize(settings)
+    @query_string = settings[:query_string]
+    @offset = settings[:offset] || 0
+    @count = settings[:count] || 50
+  end
+
+  # Class method. Parses a query string and returns
+  # a LogStash::Search::Query instance
+  def self.parse(query_string)
+    # TODO(sissel): I would prefer not to invent my own query language.
+    # Can we be similar to Lucene, SQL, or other query languages?
+    return self.new(:query_string => query_string)
+  end
+
+end # class LogStash::Search::Query

data/lib/logstash/search/result.rb

@@ -0,0 +1,39 @@
+require "logstash/namespace"
+require "logstash/logging"
+
+class LogStash::Search::Result
+  # Array of LogStash::Event of results
+  attr_accessor :events
+
+  # How long this query took, in seconds (or fractions of).
+  attr_accessor :duration
+
+  # Offset in search
+  attr_accessor :offset
+
+  # Total records matched by this query, regardless of offset/count in query.
+  attr_accessor :total
+
+  # Error message, if any.
+  attr_accessor :error_message
+
+  def initialize(settings={})
+    @events = []
+    @duration = nil
+    @error_message = nil
+  end
+
+  def error?
+    return !@error_message.nil?
+  end
+
+  def to_json
+    return {
+      "events" => @events,
+      "duration" => @duration,
+      "offset" => @offset,
+      "total" => @total,
+    }.to_json
+  end # def to_json
+end # class LogStash::Search::Result
+

data/lib/logstash/search/twitter.rb

@@ -0,0 +1,90 @@
+require "em-http-request"
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/event"
+require "logstash/search/base"
+require "logstash/search/query"
+require "logstash/search/result"
+require "logstash/search/facetresult"
+require "logstash/search/facetresult/histogram"
+ 
+class LogStash::Search::Twitter < LogStash::Search::Base
+  public
+  def initialize(settings={})
+    @host = (settings[:host] || "search.twitter.com")
+    @port = (settings[:port] || 80).to_i
+    @logger = LogStash::Logger.new(STDOUT)
+  end
+
+  public
+  def search(query)
+    raise "No block given for search call." if !block_given?
+    if query.is_a?(String)
+      query = LogStash::Search::Query.parse(query)
+    end
+
+    # TODO(sissel): only search a specific index?
+    http = EventMachine::HttpRequest.new("http://#{@host}:#{@port}/search.json?q=#{URI.escape(query.query_string)}&rpp=#{URI.escape(query.count) rescue query.count}")
+
+    @logger.info(["Query", query])
+
+    start_time = Time.now
+    req = http.get
+
+    result = LogStash::Search::Result.new
+    req.callback do
+      data = JSON.parse(req.response)
+      result.duration = Time.now - start_time
+
+      hits = (data["results"] || nil) rescue nil
+
+      if hits.nil? or !data["error"].nil?
+        # Use the error message if any, otherwise, return the whole
+        # data object as json as the error message for debugging later.
+        result.error_message = (data["error"] rescue false) || data.to_json
+        yield result
+        next
+      end
+
+      hits.each do |hit|
+        hit["@message"]  = hit["text"]
+        hit["@timestamp"] = hit["created_at"]
+        hit.delete("text")
+      end
+
+      @logger.info(["Got search results", 
+                   { :query => query.query_string, :duration => data["duration"],
+                     :result_count => hits.size }])
+
+      if req.response_header.status != 200
+        result.error_message = data["error"] || req.inspect
+        @error = data["error"] || req.inspect
+      end
+
+      # We want to yield a list of LogStash::Event objects.
+      hits.each do |hit|
+        result.events << LogStash::Event.new(hit)
+      end
+
+      # Total hits this search could find if not limited
+      result.total = hits.size
+      result.offset = 0
+
+      yield result
+    end
+
+    req.errback do 
+      @logger.warn(["Query failed", query, req, req.response])
+      result.duration = Time.now - start_time
+      result.error_message = req.response
+
+      yield result
+    end
+  end # def search
+
+  def histogram(query, field, interval=nil)
+    # Nothing to histogram.
+    result = LogStash::Search::FacetResult.new
+    yield result
+  end
+end # class LogStash::Search::ElasticSearch

data/lib/logstash/web/helpers/require_param.rb

@@ -0,0 +1,17 @@
+require "sinatra/base"
+
+module Sinatra
+  module RequireParam
+    def require_param(*fields)
+      missing = []
+      fields.each do |field|
+        if params[field].nil?
+          missing << field
+        end
+      end
+      return missing
+    end # def require_param
+  end # module RequireParam
+
+  helpers RequireParam
+end # module Sinatra

data/lib/logstash/web/public/js/logstash.js

@@ -1,4 +1,6 @@
 (function() {
+  // TODO(sissel): Write something that will use history.pushState and fall back
+  // to document.location.hash madness.
 
   var logstash = {
     params: { 
@@ -6,21 +8,79 @@
       count: 50,
     },
 
-    search: function(query) {
+    search: function(query, options) {
       if (query == undefined || query == "") {
         return;
       }
-      //console.log("Searching: " + query);
+
+      /* Default options */
+      if (typeof(options) == 'undefined') {
+        options = { graph: true };
+      }
 
       var display_query = query.replace("<", "&lt;").replace(">", "&gt;")
-      $("#querystatus").html("Loading query '" + display_query + "'")
+      $("#querystatus, #results h1").html("Loading query '" + display_query + "' (offset:" + logstash.params.offset + ", count:" + logstash.params.count + ") <img class='throbber' src='/media/construction.gif'>")
       //console.log(logstash.params)
       logstash.params.q = query;
       document.location.hash = escape(JSON.stringify(logstash.params));
-      $("#results").load("/search/ajax", logstash.params);
+
+      /* Load the search results */
+      $("#results").load("/api/search?format=html", logstash.params);
+
+      if (options.graph != false) {
+        /* Load the default histogram graph */
+        logstash.params.interval = 3600000; /* 1 hour, default */
+        logstash.histogram();
+      } /* if options.graph != false */
       $("#query").val(logstash.params.q);
     }, /* search */
 
+    histogram: function(tries) {
+      if (typeof(tries) == 'undefined') {
+        tries = 7;
+      }
+
+      /* GeoCities mode on the graph while waiting ...
+       * This won't likely survive 1.0, but it's fun for now... */
+      $("#visual").html("<center><img src='/media/truckconstruction.gif'><center>");
+
+      jQuery.getJSON("/api/histogram", logstash.params, function(histogram, text, jqxhr) {
+        /* Load the data into the graph */
+        var flot_data = [];
+        // histogram is an array of { "key": ..., "count": ... }
+        for (var i in histogram) {
+          flot_data.push([parseInt(histogram[i]["key"]), histogram[i]["count"]])
+        }
+        logstash.plot(flot_data, logstash.params.interval);
+        //console.log(histogram);
+
+        /* Try to be intelligent about how we choose the histogram interval.
+         * If there are too few data points, try a smaller interval.
+         * If there are too many data points, try a larger interval.
+         * Give up after a few tries and go with the last result. 
+         *
+         * This queries the backend several times, but should be reasonably
+         * speedy as this behaves roughly as a binary search. */
+        //if (flot_data.length < 6 && flot_data.length > 0 && tries > 0) {
+          //console.log("Histogram bucket " + logstash.params.interval + " has only " + flot_data.length + " data points, trying smaller...");
+          //logstash.params.interval /= 2;
+          //if (logstash.params.interval < 1000) {
+            //tries = 0; /* stop trying, too small... */
+            //logstash.plot(flot_data, logstash.params.interval);
+            //return;
+          //}
+          //logstash.histogram(tries - 1);
+        //} else if (flot_data.length > 50 && tries > 0) {
+          //console.log("Histogram bucket " + logstash.params.interval + " too many (" + flot_data.length + ") data points, trying larger interval...");
+          //logstash.params.interval *= 2;
+          //logstash.histogram(tries - 1);
+        //} else {
+          //console.log("Histo:" + logstash.params.interval);
+          //logstash.plot(flot_data, logstash.params.interval);
+        //}
+      });
+    },
+
     parse_params: function(href) {
       var query = href.replace(/^[^?]*\?/, "");
       if (query == href) {
@@ -48,14 +108,15 @@
       logstash.search(newquery.trim());
     }, /* appendquery */
 
-    plot: function(data) {
+    plot: function(data, interval) {
       var target = $("#visual");
+      target.css("display", "block");
       var plot = $.plot(target,
         [ {  /* data */
             data: data,
             bars: { 
               show: true,
-              barWidth: 3600000,
+              barWidth: interval,
             }
         } ],
         { /* options */
@@ -67,8 +128,12 @@
       target.bind("plotclick", function(e, pos, item) {
         if (item) {
           start = logstash.ms_to_iso8601(item.datapoint[0]);
-          end = logstash.ms_to_iso8601(item.datapoint[0] + 3600000);
+          end = logstash.ms_to_iso8601(item.datapoint[0] + interval);
 
+          /* Clicking on the graph means a new search, means
+           * we probably don't want to keep the old offset since
+           * the search results will change. */
+          logstash.params.offset = 0;
           logstash.appendquery("@timestamp:[" + start + " TO " + end + "]");
         }
       });
@@ -125,13 +190,16 @@
       for (var p in params) {
         logstash.params[p] = params[p];
       }
-      logstash.search(logstash.params.q)
+      logstash.search(logstash.params.q, { graph: false })
       return false;
     });
 
     var result_row_selector = "table.results tr.event";
     $(result_row_selector).live("click", function() {
-      var data = eval($("td.message", this).data("full"));
+      var data = $("td.message", this).data("full");
+      if (typeof(data) == "string") {
+        data = JSON.parse(data);
+      }
 
       /* Apply template to the dialog */
       var query = $("#query").val().replace(/^\s+|\s+$/g, "")
@@ -155,8 +223,8 @@
 
       /* TODO(sissel): recurse through the data */
       var fields = new Array();
-      for (var i in data._source["@fields"]) {
-        var value = data._source["@fields"][i]
+      for (var i in data["@fields"]) {
+        var value = data["@fields"][i]
         if (/^[, ]*$/.test(value)) {
           continue; /* Skip empty data fields */
         }
@@ -166,9 +234,9 @@
         fields.push( { type: "field", field: i, value: value })
       }
 
-      for (var i in data._source) {
+      for (var i in data) {
         if (i == "@fields") continue;
-        var value = data._source[i]
+        var value = data[i]
         if (!(value instanceof Array)) {
           value = [value];
         }

data/lib/logstash/web/server.rb

@@ -1,4 +1,7 @@
 #!/usr/bin/env ruby
+# I don't want folks to have to learn to use yet another tool (rackup)
+# just to launch logstash-web. So let's work like a standard ruby
+# executable.
 ##rackup -Ilib:../lib -s thin
 
 $:.unshift("%s/../lib" % File.dirname(__FILE__))
@@ -6,22 +9,49 @@ $:.unshift(File.dirname(__FILE__))
 
 require "eventmachine"
 require "json"
-require "lib/elasticsearch"
+require "logstash/search/elasticsearch"
+require "logstash/search/query"
 require "logstash/namespace"
 require "rack"
 require "rubygems"
 require "sinatra/async"
+require "logstash/web/helpers/require_param"
 
 class EventMachine::ConnectionError < RuntimeError; end
+module LogStash::Web; end
 
 class LogStash::Web::Server < Sinatra::Base
   register Sinatra::Async
+  helpers Sinatra::RequireParam # logstash/web/helpers/require_param
+
   set :haml, :format => :html5
   set :logging, true
   set :public, "#{File.dirname(__FILE__)}/public"
   set :views, "#{File.dirname(__FILE__)}/views"
-  elasticsearch = LogStash::Web::ElasticSearch.new
 
+  use Rack::CommonLogger
+  #use Rack::ShowExceptions
+
+  def initialize(settings={})
+    super
+    # TODO(sissel): Support alternate backends
+    backend_url = URI.parse(settings.backend_url)
+
+    case backend_url.scheme 
+      when "elasticsearch"
+        @backend = LogStash::Search::ElasticSearch.new(
+          :host => backend_url.host,
+          :port => backend_url.port
+        )
+      when "twitter"
+        require "logstash/search/twitter"
+        @backend = LogStash::Search::Twitter.new(
+          :host => backend_url.host,
+          :port => backend_url.port
+        )
+    end # backend_url.scheme
+  end # def initialize
+ 
   aget '/style.css' do
     headers "Content-Type" => "text/css; charset=utf8"
     body sass :style
@@ -32,8 +62,11 @@ class LogStash::Web::Server < Sinatra::Base
   end # '/'
 
   aget '/search' do
-    result_callback = proc do
+    result_callback = proc do |results|
       status 500 if @error
+      @results = results
+
+      p :got => results
 
       params[:format] ||= "html"
       case params[:format]
@@ -48,10 +81,10 @@ class LogStash::Web::Server < Sinatra::Base
         body erb :"search/results.txt", :layout => false
       when "json"
         headers({"Content-Type" => "text/plain" })
+        # TODO(sissel): issue/30 - needs refactoring here.
         hits = @hits.collect { |h| h["_source"] }
         response = {
           "hits" => hits,
-          "facets" => (@results["facets"] rescue nil),
         }
 
         response["error"] = @error if @error
@@ -63,43 +96,79 @@ class LogStash::Web::Server < Sinatra::Base
     # have javascript enabled, we need to show the results in
     # case a user doesn't have javascript.
     if params[:q] and params[:q] != ""
-      elasticsearch.search(params) do |results|
-        @results = results
-        @hits = (@results["hits"]["hits"] rescue [])
+      query = LogStash::Search::Query.new(
+        :query_string => params[:q],
+        :offset => params[:offset],
+        :count => params[:count]
+      )
+
+      @backend.search(query) do |results|
+        p :got => results
         begin
-          result_callback.call
+          result_callback.call results
         rescue => e
-          puts e
+          p :exception => e
         end
-      end # elasticsearch.search
+      end # @backend.search
     else
-      #@error = "No query given."
-      @hits = []
-      result_callback.call
+      results = LogStash::Search::Result.new(
+        :events => [],
+        :error_message => "No query given"
+      )
+      result_callback.call results
     end
   end # aget '/search'
 
-  apost '/search/ajax' do
+  apost '/api/search' do
+    api_search
+  end # apost /api/search
+
+  aget '/api/search' do
+    api_search
+  end # aget /api/search
+
+  def api_search
+
     headers({"Content-Type" => "text/html" })
     count = params["count"] = (params["count"] or 50).to_i
     offset = params["offset"] = (params["offset"] or 0).to_i
-    elasticsearch.search(params) do |results|
+    format = (params[:format] or "json")
+
+    query = LogStash::Search::Query.new(
+      :query_string => params[:q],
+      :offset => offset,
+      :count => count
+    )
+
+    @backend.search(query) do |results|
       @results = results
-      if @results.include?("error")
-        body haml :"search/error", :layout => !request.xhr?
+      if @results.error?
+        status 500
+        case format
+        when "html"
+          headers({"Content-Type" => "text/html" })
+          body haml :"search/error", :layout => !request.xhr?
+        when "text"
+          headers({"Content-Type" => "text/plain" })
+          body erb :"search/error.txt", :layout => false
+        when "txt"
+          headers({"Content-Type" => "text/plain" })
+          body erb :"search/error.txt", :layout => false
+        when "json"
+          headers({"Content-Type" => "text/plain" })
+          # TODO(sissel): issue/30 - needs refactoring here.
+          if @results.error?
+            body({ "error" => @results.error_message }.to_json)
+          else
+            body @results.to_json
+          end
+        end # case params[:format]
         next
       end
 
-      @hits = (@results["hits"]["hits"] rescue [])
-      @total = (@results["hits"]["total"] rescue 0)
-      @graphpoints = []
-      begin
-        @results["facets"]["by_hour"]["entries"].each do |entry|
-          @graphpoints << [entry["key"], entry["count"]]
-        end
-      rescue => e
-        puts e
-      end
+      @events = @results.events
+      @total = (@results.total rescue 0)
+      count = @results.events.size
 
       if count and offset
         if @total > (count + offset)
@@ -115,7 +184,7 @@ class LogStash::Web::Server < Sinatra::Base
         next_params["offset"] = [offset + count, @total - count].min
         @next_href = "?" +  next_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
         last_params = next_params.clone
-        last_params["offset"] = @total - offset
+        last_params["offset"] = @total - count
         @last_href = "?" +  last_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
       end
 
@@ -124,24 +193,83 @@ class LogStash::Web::Server < Sinatra::Base
         prev_params["offset"] = [offset - count, 0].max
         @prev_href = "?" +  prev_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
 
-        if prev_params["offset"] > 0
+        #if prev_params["offset"] > 0
           first_params = prev_params.clone
           first_params["offset"] = 0
           @first_href = "?" +  first_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
-        end
+        #end
       end
 
-      body haml :"search/ajax", :layout => !request.xhr?
-    end # elasticsearch.search
-  end # apost '/search/ajax'
+      # TODO(sissel): make a helper function taht goes hash -> cgi querystring
+      @refresh_href = "?" +  params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
+
+      case format
+      when "html"
+        headers({"Content-Type" => "text/html" })
+        body haml :"search/ajax", :layout => !request.xhr?
+      when "text"
+        headers({"Content-Type" => "text/plain" })
+        body erb :"search/results.txt", :layout => false
+      when "txt"
+        headers({"Content-Type" => "text/plain" })
+        body erb :"search/results.txt", :layout => false
+      when "json"
+        headers({"Content-Type" => "text/plain" })
+        # TODO(sissel): issue/30 - needs refactoring here.
+        response = @results
+        body response.to_json
+      end # case params[:format]
+    end # @backend.search
+  end # def api_search
+
+  aget '/api/histogram' do
+    headers({"Content-Type" => "text/plain" })
+    missing = require_param(:q)
+    if !missing.empty?
+      status 500
+      body({ "error" => "Missing requiremed parameters",
+             "missing" => missing }.to_json)
+      next
+    end # if !missing.empty?
+
+    format = (params[:format] or "json")            # default json
+    field = (params[:field] or "@timestamp")        # default @timestamp
+    interval = (params[:interval] or 3600000).to_i  # default 1 hour
+    @backend.histogram(params[:q], field, interval) do |results|
+      @results = results
+      if @results.error?
+        status 500
+        body({ "error" => @results.error_message }.to_json)
+        next
+      end
+
+      begin
+        a = results.results.to_json
+      rescue => e
+        status 500
+        body e.inspect
+        p :exception => e
+        p e
+        raise e
+      end
+      status 200
+      body a
+    end # @backend.search
+  end # aget '/api/histogram'
+
+  aget '/*' do
+    status 404 if @error
+    body "Invalid path."
+  end # aget /*
 end # class LogStash::Web::Server
 
 require "optparse"
-Settings = Struct.new(:daemonize, :logfile, :address, :port)
+Settings = Struct.new(:daemonize, :logfile, :address, :port, :backend_url)
 settings = Settings.new
 
-settings.address      = "0.0.0.0"
-settings.port      = 9292
+settings.address = "0.0.0.0"
+settings.port = 9292
+settings.backend_url = "elasticsearch://localhost:9200/"
 
 progname = File.basename($0)
 
@@ -163,6 +291,11 @@ opts = OptionParser.new do |opts|
   opts.on("-p", "--port PORT", "Port on which to start webserver. Default is 9292.") do |port|
     settings.port = port.to_i
   end
+
+  opts.on("-b", "--backend URL",
+          "The backend URL to use. Default is elasticserach://localhost:9200/") do |url|
+    settings.backend_url = url
+  end
 end
 
 opts.parse!
@@ -189,5 +322,5 @@ end
 Rack::Handler::Thin.run(
   Rack::CommonLogger.new( \
     Rack::ShowExceptions.new( \
-      LogStash::Web::Server.new)),
+      LogStash::Web::Server.new(settings))),
   :Port => settings.port, :Host => settings.address)