logstash-lite 0.2.20101222161646 → 0.2.20110112115019

data/lib/logstash/outputs/stomp.rb

@@ -1,20 +1,24 @@
 require "logstash/outputs/base"
+require "logstash/namespace"
 require "logstash/stomp/handler"
 
 class LogStash::Outputs::Stomp < LogStash::Outputs::Base
   attr_reader :url
 
+  public
   def initialize(url, config={}, &block)
     super
 
     @logger.debug(["Initialize", { :url => @url }])
   end # def initialize
 
+  public
   def register
     @logger.info(["Registering output", { :url => @url }])
     @connection = EventMachine::connect(@url.host, @url.port, LogStash::Stomp::Handler, self, @logger, @url)
   end # def register
 
+  public
   def receive(event)
     @logger.debug(["Sending event", { :url => @url, :event => event }])
     @connection.send(@url.path, event.to_json)

data/lib/logstash/outputs/tcp.rb

@@ -1,10 +1,8 @@
 require "logstash/outputs/base"
+require "logstash/namespace"
 
 class LogStash::Outputs::Tcp < LogStash::Outputs::Base
-  def initialize(url, config={}, &block)
-    super
-  end
-
+  public
   def register
     # TODO(sissel): Write generic validation methods
     if !@url.host or !@url.port
@@ -16,6 +14,7 @@ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
     @connection = EventMachine::connect(@url.host, @url.port)
   end # def register
 
+  public
   def receive(event)
     @connection.send_data(event.to_hash.to_json)
     @connection.send_data("\n")

data/lib/logstash/outputs/websocket.rb

@@ -1,11 +1,9 @@
-require "logstash/outputs/base"
 require "em-websocket" # rubygem 'em-websocket'
+require "logstash/namespace"
+require "logstash/outputs/base"
 
 class LogStash::Outputs::Websocket < LogStash::Outputs::Base
-  def initialize(url, config={}, &block)
-    super
-  end
-
+  public
   def register
     @channel = EventMachine::Channel.new
     @subscribers = 0
@@ -27,6 +25,7 @@ class LogStash::Outputs::Websocket < LogStash::Outputs::Base
     end
   end # def register
 
+  public
   def receive(event)
     # Only publish the event to websockets if there are subscribers
     # TODO(sissel): send a patch to eventmachine to fix this.
@@ -34,5 +33,5 @@ class LogStash::Outputs::Websocket < LogStash::Outputs::Base
       @logger.info("Sending event to websocket.")
       @channel.push event.to_json
     end
-  end # def event
+  end # def receive
 end # class LogStash::Outputs::Websocket

data/lib/logstash/ruby_fixes.rb

@@ -1,12 +1,10 @@
-
-
 # Ruby 1.8.7 added String#start_with? - monkeypatch the
 # String class if it isn't supported (<= ruby 1.8.6)
 if !String.instance_methods.include?("start_with?")
   class String
+    public
     def start_with?(str)
       return self[0 .. (str.length-1)] == str
     end
   end
 end
-

data/lib/logstash/stomp/handler.rb

@@ -1,3 +1,5 @@
+require "logstash/namespace"
+
 # Base of Stomp Handler
 # it handles connecting and subscribing to the stomp broker which
 # is used in both stomp input and output
@@ -5,29 +7,52 @@ class LogStash::Stomp
   class Handler < EventMachine::Connection
     include EM::Protocols::Stomp
 
+    attr_accessor :should_subscribe
+    attr_accessor :ready
+
+    public
     def initialize(*args)
       super
 
       @input = args[0]
       @logger = args[1]
       @url = args[2]
+      @should_subscribe = true
+      @ready = false
     end # def initialize
 
+    public
     def connection_completed
       @logger.debug("Connected")
       connect :login => @url.user, :passcode => @url.password
+      @ready = true
     end # def connection_completed
 
+    public
     def unbind
-      @logger.error(["Error when connecting to stomp broker", { :url => @url }])
+      if $EVENTMACHINE_STOPPING
+        @logger.debug(["Connection to stomp broker died (probably since we are exiting)",
+                      { :url => @url }])
+        return
+      end
+                    
+      @logger.error(["Connection to stomp broker died, retrying.", { :url => @url }])
+      @ready = false
+      EventMachine::Timer.new(1) do
+        reconnect(@url.host, @url.port)
+      end
     end # def unbind
 
+    public
     def receive_msg(message)
       @logger.debug(["receiving message", { :msg => message }])
       if message.command == "CONNECTED"
-        @logger.debug(["subscribing to", { :path => @url.path }])
-        subscribe @url.path
-        return
+        if @should_subscribe
+          @logger.debug(["subscribing to", { :path => @url.path }])
+          subscribe @url.path
+          return
+        end
+        @ready = true
       end
     end # def receive_msg
   end # class Handler

data/lib/logstash/web/lib/elasticsearch.rb

@@ -5,11 +5,14 @@ require "logstash/logging"
 require "logstash/event"
 
 module LogStash::Web; end
+
 class LogStash::Web::ElasticSearch
+  public
   def initialize
     @logger = LogStash::Logger.new(STDOUT)
   end
 
+  public
   def search(params)
     http = EventMachine::HttpRequest.new("http://localhost:9200/_search")
     params[:offset] ||= 0
@@ -37,7 +40,8 @@ class LogStash::Web::ElasticSearch
       "from" => params[:offset],
       "size" => params[:count],
     }
-    #@logger.info(["ElasticSearch Query", esreq])
+
+    @logger.info("ElasticSearch Query: #{esreq.to_json}")
     start_time = Time.now
     req = http.get :body => esreq.to_json
     req.callback do
@@ -69,13 +73,13 @@ class LogStash::Web::ElasticSearch
                    { :query => params[:q], :duration => data["duration"]}])
       #@logger.info(data)
       if req.response_header.status != 200
-        @error = data["error"]
+        @error = data["error"] || req.inspect
       end
       yield data
     end
     req.errback do 
-      @logger.warn(["Query failed", params, req.response])
+      @logger.warn(["Query failed", params, req, req.response])
       yield({ "error" => req.response })
     end
   end # def search
-end
+end # class LogStash::Web::ElasticSearch

data/lib/logstash/web/public/js/logstash.js

@@ -7,12 +7,28 @@
     },
 
     search: function(query) {
+      if (query == undefined || query == "") {
+        return;
+      }
+      var display_query = query.replace("<", "&lt;").replace(">", "&gt;")
+      $("#querystatus").html("Loading query '" + display_query + "'")
       logstash.params.q = query;
       document.location.hash = escape(JSON.stringify(logstash.params));
       $("#results").load("/search/ajax", logstash.params);
       $("#query").val(logstash.params.q);
     }, /* search */
 
+    parse_params: function(href) {
+      var params = href.replace(/^[^?]*\?/, "").split("&")
+      for (var p in params) {
+        var a = params[p].split("=");
+        var key = a[0]
+        var value = a[1]
+        logstash.params[key] = unescape(value)
+      }
+      return params;
+    },
+
     appendquery: function(query) {
       var newquery = $("#query").val();
       newquery += " " + query;
@@ -70,6 +86,13 @@
         // Do nothing 
       }
       logstash.search(logstash.params.q);
+    } else {
+      /* No hash. See if there's a query param. */
+      var params = logstash.parse_params(location.href);
+      for (var p in params) {
+        logstash.params[p] = params[p];
+      }
+      logstash.search(logstash.params.q)
     }
 
     $(window).hashchange(function() {
@@ -83,12 +106,9 @@
 
     $("a.pager").live("click", function() {
       var href = $(this).attr("href");
-      var params = href.replace(/^[^?]*\?/, "").split("&")
+      var params = logstash.parse_params(location.href);
       for (var p in params) {
-        var a = params[p].split("=");
-        var key = a[0]
-        var value = a[1]
-        logstash.params[key] = unescape(value)
+        logstash.params[p] = params[p];
       }
       logstash.search(logstash.params.q)
       return false;

data/lib/logstash/web/public/ws/index.html

@@ -57,16 +57,18 @@
           //console.log(["WebSocket open", ws])
         };
         ws.onmessage = function(event) {
-          eval("var data = " + event.data);
+          var data = JSON.parse(event.data); 
           var el = $("#message-template").tmpl(data, { "message": data["@message"], "timestamp": data["@timestamp"] });
+            //.css("display", "none")
+            //.fadeIn()
           el.addClass("message")
-            .css("display", "none")
             .appendTo($("#radiator"))
-            .fadeIn()
-            .delay(10000)
-            .fadeOut(2000, function() {
-              $(this).remove();
-            });
+            //.delay(10000)
+            //.hide(2000, function() {
+              //$(this).remove();
+            //});
+          setTimeout(function() { $(el).remove() }, 12000)
+            //.fadeOut(2000, function() {
         };
       });
     </script>

data/lib/logstash/web/server.rb

@@ -4,13 +4,13 @@
 $:.unshift("%s/../lib" % File.dirname(__FILE__))
 $:.unshift(File.dirname(__FILE__))
 
-require "rubygems"
-require "json"
 require "eventmachine"
-require "rack"
-require "sinatra/async"
+require "json"
 require "lib/elasticsearch"
 require "logstash/namespace"
+require "rack"
+require "rubygems"
+require "sinatra/async"
 
 class EventMachine::ConnectionError < RuntimeError; end
 
@@ -25,24 +25,58 @@ class LogStash::Web::Server < Sinatra::Base
   aget '/style.css' do
     headers "Content-Type" => "text/css; charset=utf8"
     body sass :style
-  end
+  end # /style.css
 
   aget '/' do
     redirect "/search"
   end # '/'
 
   aget '/search' do
-    headers({"Content-Type" => "text/html" })
+    result_callback = proc do 
+      status 500 if @error
+
+      params[:format] ||= "html"
+      case params[:format]
+      when "html"
+        headers({"Content-Type" => "text/html" })
+        body haml :"search/results", :layout => !request.xhr?
+      when "text"
+        headers({"Content-Type" => "text/plain" })
+        body erb :"search/results.txt", :layout => false
+      when "txt"
+        headers({"Content-Type" => "text/plain" })
+        body erb :"search/results.txt", :layout => false
+      when "json"
+        headers({"Content-Type" => "text/plain" })
+        hits = @hits.collect { |h| h["_source"] }
+        response = {
+          "hits" => hits,
+          "facets" => (@results["facets"] rescue nil),
+        }
+
+        response["error"] = @error if @error
+        body response.to_json
+      end # case params[:format]
+    end # proc result_callback
+
+    # We'll still do a search query here even though most users
+    # have javascript enabled, we need to show the results in
+    # case a user doesn't have javascript.
     if params[:q] and params[:q] != ""
       elasticsearch.search(params) do |@results|
         @hits = (@results["hits"]["hits"] rescue [])
-        body haml :"search/results", :layout => !request.xhr?
-      end
+        begin
+          result_callback.call
+        rescue => e
+          puts e
+        end
+      end # elasticsearch.search
     else
+      #@error = "No query given."
       @hits = []
-      body haml :"search/results", :layout => !request.xhr?
+      result_callback.call
     end
-  end
+  end # aget '/search'
 
   apost '/search/ajax' do
     headers({"Content-Type" => "text/html" })
@@ -99,10 +133,9 @@ class LogStash::Web::Server < Sinatra::Base
       body haml :"search/ajax", :layout => !request.xhr?
     end # elasticsearch.search
   end # apost '/search/ajax'
-end # class LogStashWeb
+end # class LogStash::Web::Server
 
 require "optparse"
-
 Settings = Struct.new(:daemonize, :logfile)
 settings = Settings.new
 progname = File.basename($0)
@@ -133,6 +166,11 @@ if settings.logfile
   logfile = File.open(settings.logfile, "w")
   STDOUT.reopen(logfile)
   STDERR.reopen(logfile)
+elsif settings.daemonize
+  # Write to /dev/null if 
+  devnull = File.open("/dev/null", "w")
+  STDOUT.reopen(devnull)
+  STDERR.reopen(devnull)
 end
 
 Rack::Handler::Thin.run(

data/lib/logstash/web/views/search/ajax.haml

@@ -31,9 +31,12 @@
           last
   - if @hits.length == 0 
     - if !params[:q]
-      No query given. How about <a href="?q=*" class="querychanger">this?</a>
+      / We default to a '+2 days' in the future  to capture 'today at 00:00'
+      / plus tomorrow, inclusive, in case you are 23 hours behind the international
+      / dateline.
+      %h3#querystatus No query given. How about <a href="?q=* @timestamp:[#{(Time.now - 7*24*60*60).strftime("%Y-%m-%d")} TO #{(Time.now + 2*24*60*60).strftime("%Y-%m-%d")}]" class="querychanger">this?</a>
     - else
-      No results for query '#{params[:q]}' 
+      %h3#querystatus No results for query '#{params[:q]}' 
   - else
     %table.results
       %tr

data/lib/logstash/web/views/search/results.txt.erb

@@ -0,0 +1,10 @@
+<% 
+  # Sinatra currently doesn't do ERB with newline trimming, so we
+  # have to write this funky mishmosh that is hard to read.
+if @error %>Error: <%= @error %><% else
+  @hits.each do |hit|
+    event = LogStash::Event.new(hit["_source"])
+%><%= event.message || event.to_hash.to_json %>
+<%   end
+   end 
+%>

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: logstash-lite
 version: !ruby/object:Gem::Version 
-  hash: 40202444323275
+  hash: 40220224230017
   prerelease: false
   segments: 
   - 0
   - 2
-  - 20101222161646
-  version: 0.2.20101222161646
+  - 20110112115019
+  version: 0.2.20110112115019
 platform: ruby
 authors: 
 - Jordan Sissel
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-22 00:00:00 -08:00
+date: 2011-01-12 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -95,8 +95,10 @@ files:
 - lib/logstash/inputs/internal.rb
 - lib/logstash/inputs/syslog.rb
 - lib/logstash/inputs/file.rb
+- lib/logstash/inputs/stdin.rb
 - lib/logstash/inputs/base.rb
 - lib/logstash/inputs/amqp.rb
+- lib/logstash/inputs/twitter.rb
 - lib/logstash/inputs/stomp.rb
 - lib/logstash/inputs/beanstalk.rb
 - lib/logstash/inputs/tcp.rb
@@ -209,6 +211,7 @@ files:
 - lib/logstash/web/views/style.sass
 - lib/logstash/web/views/main/index.haml
 - lib/logstash/web/views/layout.haml
+- lib/logstash/web/views/search/results.txt.erb
 - lib/logstash/web/views/search/ajax.haml
 - lib/logstash/web/views/search/results.haml
 - lib/logstash/web/views/search/error.haml