logstash-lite 0.2.20101120024757 → 0.2.20101123133737

data/examples/sample-agent-in-ruby.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+$: << "lib"
+
+require "rubygems"
+require "logstash/agent"
+require "yaml"
+
+collector_config = YAML.load <<"YAML"
+---
+inputs:
+  foo:
+  - internal:///
+outputs:
+- amqp://localhost/topic/logstash/testing
+YAML
+
+receiver_config = YAML.load <<"YAML"
+---
+inputs:
+  foo:
+  - amqp://localhost/topic/logstash/testing
+outputs:
+- internal:///
+YAML
+
+collector_agent = LogStash::Agent.new(collector_config)
+receiver_agent = LogStash::Agent.new(receiver_config)
+
+data = ["hello world", "foo", "bar"]
+
+EventMachine::run do
+  receiver_agent.register
+  collector_agent.register
+
+  EM::next_tick do
+    # Register output callback on the receiver
+    receiver_agent.outputs\
+        .find { |o| o.is_a?(LogStash::Outputs::Internal) }\
+        .callback do |event|
+      puts event
+      #puts expect.first == event.message
+      #expect.shift
+      #agent.stop
+    end
+
+    EM::next_tick do
+      # Send input to the collector
+      expect = data.clone
+      input = collector_agent.inputs\
+        .find { |i| i.is_a?(LogStash::Inputs::Internal) }
+      channel = input.channel
+      data.each { |message| channel.push(message) }
+    end
+  end
+end
+

data/lib/logstash/agent.rb

@@ -9,6 +9,9 @@ require "logstash/logging"
 # Collect logs, ship them out.
 class LogStash::Agent
   attr_reader :config
+  attr_reader :inputs
+  attr_reader :outputs
+  attr_reader :filters
 
   def initialize(config)
     @logger = LogStash::Logger.new(STDERR)
@@ -26,7 +29,7 @@ class LogStash::Agent
   # Register any event handlers with EventMachine
   # Technically, this agent could listen for anything (files, sockets, amqp,
   # stomp, etc).
-  protected
+  public
   def register
     # TODO(sissel): warn when no inputs and no outputs are defined.
     # TODO(sissel): Refactor this madness into a config lib
@@ -80,12 +83,20 @@ class LogStash::Agent
   end # def register
 
   public
-  def run
+  def run(&block)
     EventMachine.run do
       self.register
+      yield if block_given?
     end # EventMachine.run
   end # def run
 
+  public
+  def stop
+    # TODO(sissel): Stop inputs, fluch outputs, wait for finish,
+    # then stop the event loop
+    EventMachine.stop_event_loop
+  end
+
   protected
   def filter(event)
     @filters.each do |f|

data/lib/logstash/inputs.rb

@@ -1,5 +1,6 @@
 
 require "logstash/namespace"
+require "logstash/ruby_fixes"
 require "uri"
 
 module LogStash::Inputs

data/lib/logstash/inputs/amqp.rb

@@ -23,7 +23,7 @@ class LogStash::Inputs::Amqp < LogStash::Inputs::Base
   end
 
   def register
-    @logger.info("Registering #{@url}")
+    @logger.info("Registering input #{@url}")
     @amqp = AMQP.connect(:host => @url.host)
     @mq = MQ.new(@amqp)
     @target = nil

data/lib/logstash/inputs/internal.rb

@@ -0,0 +1,36 @@
+
+require "logstash/inputs/base"
+require "eventmachine-tail"
+require "socket" # for Socket.gethostname
+
+class LogStash::Inputs::Internal < LogStash::Inputs::Base
+  attr_reader :channel
+
+  def initialize(url, type, config={}, &block)
+    super
+
+    # Default host to the machine's hostname if it's not set
+    @url.host ||= Socket.gethostname
+    @channel = EventMachine::Channel.new
+  end
+
+  def register
+    @logger.info("Registering input #{@url}")
+    @channel.subscribe do |event|
+      receive(event)
+    end
+  end # def register
+
+  def receive(event)
+    if !event.is_a?(LogStash::Event)
+      event = LogStash::Event.new({
+        "@message" => event,
+        "@type" => @type,
+        "@tags" => @tags.clone,
+        "@source" => @url,
+      })
+    end
+    @logger.debug(["Got event", event])
+    @callback.call(event)
+  end # def receive
+end # class LogStash::Inputs::Internal

data/lib/logstash/logging.rb

@@ -1,4 +1,5 @@
 require "logstash/namespace"
+require "logstash/ruby_fixes"
 require "logger"
 
 class LogStash::Logger < Logger
@@ -18,16 +19,22 @@ class LogStash::Logger < Logger
 
     # Set default loglevel to WARN unless $DEBUG is set (run with 'ruby -d')
     self.level = $DEBUG ? Logger::DEBUG: Logger::INFO
+    if ENV["LOGSTASH_DEBUG"]
+      self.level = Logger::DEBUG
+    end
+
+    @formatter.progname = self.progname = File.basename($0)
 
     # Conditional support for awesome_print
     if !@@have_awesome_print && @@notify_awesome_print_load_failed
-      info [ "Failed: require 'ap' (aka awesome_print); some " \
-             "logging features may be disabled", 
-             @@notify_awesome_print_load_failed ]
+      debug [ "awesome_print not found, falling back to Object#inspect." \
+              "If you want prettier log output, run 'gem install "\
+              "awesome_print'", 
+              { :exception @@notify_awesome_print_load_failed }]
+
+      # Only show this once.
       @@notify_awesome_print_load_failed = nil
     end
-
-    @formatter.progname = self.progname = File.basename($0)
   end # def initialize
 
   def level=(level)

data/lib/logstash/outputs/amqp.rb

@@ -19,6 +19,7 @@ class LogStash::Outputs::Amqp < LogStash::Outputs::Base
   end # def initialize
 
   def register
+    @logger.info("Registering output #{@url}")
     @amqp = AMQP.connect(:host => @url.host)
     @mq = MQ.new(@amqp)
     @target = nil

data/lib/logstash/outputs/gelf.rb

@@ -26,6 +26,7 @@ class LogStash::Outputs::Gelf < LogStash::Outputs::Base
     gelf.file = event["@source_path"]
 
     event.fields.each do |name, value|
+      next if value == nil or value.empty?
       gelf.add_additional name, value
     end
     gelf.add_additional "event_timestamp", event.timestamp

data/lib/logstash/outputs/internal.rb

@@ -0,0 +1,30 @@
+require "logstash/outputs/base"
+
+class LogStash::Outputs::Internal < LogStash::Outputs::Base
+  def initialize(url, config={}, &block)
+    super
+    @callback = block
+  end
+
+  def register
+    @logger.info("Registering output #{@url}")
+  end # def register
+
+  def receive(event)
+    if !@callback
+      @logger.error("No callback for output #{@url}, cannot receive")
+      return
+    end
+    @callback.call(event)
+  end # def event
+
+  # Set the callback by passing a block of code
+  def callback(&block)
+    @callback = block
+  end
+
+  # Set the callback by passing a proc object
+  def callback=(proc_block)
+    @callback = proc_block
+  end
+end # class LogStash::Outputs::Internal

data/lib/logstash/outputs/tcp.rb

@@ -0,0 +1,23 @@
+require "logstash/outputs/base"
+
+class LogStash::Outputs::Tcp < LogStash::Outputs::Base
+  def initialize(url, config={}, &block)
+    super
+  end
+
+  def register
+    # TODO(sissel): Write generic validation methods
+    if !@url.host or !@url.port
+      @logger.fatal("No host or port given in #{self.class}: #{@url}")
+      # TODO(sissel): Make this an actual exception class
+      raise "configuration error"
+    end
+
+    @connection = EventMachine::connect(@url.host, @url.port)
+  end # def register
+
+  def receive(event)
+    @connection.send_data(event.to_hash.to_json)
+    @connection.send_data("\n")
+  end # def receive
+end # class LogStash::Outputs::Tcp

data/lib/logstash/ruby_fixes.rb

@@ -0,0 +1,12 @@
+
+
+# Ruby 1.8.7 added String#start_with? - monkeypatch the
+# String class if it isn't supported (<= ruby 1.8.6)
+if !String.instance_methods.include?("start_with?")
+  class String
+    def start_with?(str)
+      return self[0 .. (str.length-1)] == str
+    end
+  end
+end
+

data/lib/logstash/time.rb

@@ -1,3 +1,4 @@
+require "logstash/namespace"
 
 # Provide our own Time wrapper for ISO8601 support
 # Example:
@@ -6,7 +7,7 @@
 #
 #   >> LogStash::Time.now.utc.to_iso8601
 #   => "2010-10-17 07:25:26.788704Z"
-module LogStash; class Time < ::Time
+class LogStash::Time < ::Time
   ISO8601 = "%Y-%m-%dT%H:%M:%S"
 
   # Return a string that is this time in ISO8601 format.
@@ -24,4 +25,4 @@ module LogStash; class Time < ::Time
       raise "Can't convert object of type #{obj.class} (#{obj}) to iso8601."
     end
   end
-end; end # class LogStash::Time
+end # class LogStash::Time

data/lib/logstash/web/lib/elasticsearch.rb

@@ -75,7 +75,7 @@ class LogStash::Web::ElasticSearch
     end
     req.errback do 
       @logger.warn(["Query failed", params, req.response])
-      yield :failure
+      yield({ "error" => req.response })
     end
   end # def search
 end

data/lib/logstash/web/public/ws/index.html

@@ -0,0 +1,73 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.3/jquery.min.js"></script>
+    <script src="../js/jquery.tmpl.min.js"></script>
+    <style type="text/css">
+      #radiator {
+        margin-top: 1em;
+        padding-left:10px;
+        padding-right:10px;
+        font-size: 120%;
+        border-top: 1px solid black;
+        width: 100%;
+      }
+
+      #radiator .entry:nth-child(2n) {
+        background-color: #C4FFC6;
+      }
+
+      #radiator .timestamp {
+        white-space: nowrap;
+        text-align: left;
+        vertical-align: top;
+        font-family: monospace;
+        font-size: 75%;
+        padding-right: 1em;
+        width: 14em;
+        border-right: 1px solid grey;
+      }
+
+      #radiator .message {
+        font-family: monospace;
+        //text-indent: -2em;
+        //padding-left: 3em;
+      }
+    </style>
+  </head>
+  <body>
+    <h1> logstash information radiator </h1>
+
+    New log data will be streamed here in real-time as logstash
+    receives it.
+
+    <table id="radiator"></table>
+
+    <script id="message-template" type="text/x-jquery-tmpl"><![CDATA[
+      <tr>
+          <td class="timestamp">${received_timestamp}</td>
+          <td class="message">${message}</td>
+      </tr>
+    ]]></script>
+
+    <script>
+      $(document).ready(function() {
+        var ws = new WebSocket("ws://snack.home:3000");
+        ws.onmessage = function(event) {
+          eval("var data = " + event.data);
+          //var el = $("<div></div>")
+          //el.html(data.message)
+          var el = $("#message-template").tmpl(data);
+          el.addClass("message")
+            .css("display", "none")
+            .appendTo($("#radiator"))
+            .fadeIn()
+            .delay(10000)
+            .fadeOut(2000, function() {
+              $(this).remove();
+            });
+        }
+      });
+    </script>
+  </body>
+</html>

data/lib/logstash/web/server.rb

@@ -49,11 +49,21 @@ class LogStash::Web::Server < Sinatra::Base
     count = params["count"] = (params["count"] or 50).to_i
     offset = params["offset"] = (params["offset"] or 0).to_i
     elasticsearch.search(params) do |@results|
+      #p instance_variables
+      if @results.include?("error")
+        body haml :"search/error", :layout => !request.xhr?
+        next
+      end
+
       @hits = (@results["hits"]["hits"] rescue [])
       @total = (@results["hits"]["total"] rescue 0)
       @graphpoints = []
-      @results["facets"]["by_hour"]["entries"].each do |entry|
-        @graphpoints << [entry["key"], entry["count"]]
+      begin
+        @results["facets"]["by_hour"]["entries"].each do |entry|
+          @graphpoints << [entry["key"], entry["count"]]
+        end
+      rescue => e
+        puts e
       end
 
       if count and offset
@@ -69,12 +79,21 @@ class LogStash::Web::Server < Sinatra::Base
         next_params = params.clone
         next_params["offset"] = [offset + count, @total - count].min
         @next_href = "?" +  next_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
+        last_params = next_params.clone
+        last_params["offset"] = @total - offset
+        @last_href = "?" +  last_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
       end
 
       if offset > 0
         prev_params = params.clone
         prev_params["offset"] = [offset - count, 0].max
         @prev_href = "?" +  prev_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
+
+        if prev_params["offset"] > 0
+          first_params = prev_params.clone
+          first_params["offset"] = 0
+          @first_href = "?" +  first_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
+        end
       end
 
       body haml :"search/ajax", :layout => !request.xhr?

data/lib/logstash/web/views/search/ajax.haml

@@ -14,6 +14,9 @@
       %strong
         Results #{@result_start} - #{@result_end} of #{@total}
       |
+      - if @first_href
+        %a.pager{ :href => @first_href } first
+        |
       - if @prev_href
         %a.pager{ :href => @prev_href } 
           prev
@@ -22,6 +25,10 @@
       - if @next_href
         %a.pager{ :href => @next_href }
           next
+      - if @last_href
+        |
+        %a.pager{ :href => @last_href }
+          last
   - if @hits.length == 0 
     - if !params[:q]
       No query given. How about <a href="?q=*" class="querychanger">this?</a>

data/lib/logstash/web/views/search/error.haml

@@ -0,0 +1,3 @@
+#error
+  %h4 The query '#{params["q"]}' resulted the following error:
+  %pre&= @results["error"]

data/lib/logstash/web/views/style.sass

@@ -46,6 +46,8 @@ body
   background-color: pink
   border: 1px solid red
   padding: 3px
+  pre
+    white-space: pre-wrap
 #error h1
   font-size: 130%
   padding: 0

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: logstash-lite
 version: !ruby/object:Gem::Version 
-  hash: 40202240049533
+  hash: 40202246267461
   prerelease: false
   segments: 
   - 0
   - 2
-  - 20101120024757
-  version: 0.2.20101120024757
+  - 20101123133737
+  version: 0.2.20101123133737
 platform: ruby
 authors: 
 - Jordan Sissel
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-20 00:00:00 -08:00
+date: 2010-11-23 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -59,6 +59,8 @@ files:
 - lib/logstash.rb
 - lib/logstash/agent.rb
 - lib/logstash/inputs.rb
+- lib/logstash/ruby_fixes.rb
+- lib/logstash/inputs/internal.rb
 - lib/logstash/inputs/syslog.rb
 - lib/logstash/inputs/file.rb
 - lib/logstash/inputs/base.rb
@@ -66,11 +68,13 @@ files:
 - lib/logstash/inputs/tcp.rb
 - lib/logstash/outputs/gelf.rb
 - lib/logstash/outputs/elasticsearch.rb
+- lib/logstash/outputs/internal.rb
 - lib/logstash/outputs/mongodb.rb
 - lib/logstash/outputs/stdout.rb
 - lib/logstash/outputs/websocket.rb
 - lib/logstash/outputs/base.rb
 - lib/logstash/outputs/amqp.rb
+- lib/logstash/outputs/tcp.rb
 - lib/logstash/namespace.rb
 - lib/logstash/time.rb
 - lib/logstash/filters.rb
@@ -98,6 +102,7 @@ files:
 - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_55_fbf9ee_1x400.png
 - lib/logstash/web/public/css/smoothness/images/ui-bg_flat_75_ffffff_40x100.png
 - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_95_fef1ec_1x400.png
+- lib/logstash/web/public/ws/index.html
 - lib/logstash/web/public/js/jquery.livequery.js
 - lib/logstash/web/public/js/logstash.js
 - lib/logstash/web/public/js/jquery-hashchange-1.0.0.js
@@ -167,8 +172,10 @@ files:
 - lib/logstash/web/views/layout.haml
 - lib/logstash/web/views/search/ajax.haml
 - lib/logstash/web/views/search/results.haml
+- lib/logstash/web/views/search/error.haml
 - lib/logstash/web/views/header.haml
 - examples/test.rb
+- examples/sample-agent-in-ruby.rb
 - etc/tograylog.yaml
 - etc/logstash-elasticsearch-rabbitmq-river.yaml
 - etc/logstash-reader.yaml