logstash-lite 0.2.20101222161646 → 0.2.20110112115019

data/lib/logstash/logging.rb

@@ -4,7 +4,7 @@ require "logger"
 
 class LogStash::Logger < Logger
   # Try to load awesome_print, if it fails, log it later
-  # but otherwise we should continue to operate as normal.
+  # but otherwise we will continue to operate as normal.
   begin 
     require "ap"
     @@have_awesome_print = true
@@ -13,6 +13,7 @@ class LogStash::Logger < Logger
     @@notify_awesome_print_load_failed = e
   end
 
+  public
   def initialize(*args)
     super(*args)
     @formatter = LogStash::Logger::Formatter.new
@@ -37,6 +38,7 @@ class LogStash::Logger < Logger
     end
   end # def initialize
 
+  public
   def level=(level)
     super(level)
     @formatter.level = level
@@ -48,6 +50,7 @@ class LogStash::Logger::Formatter < Logger::Formatter
   attr_accessor :level
   attr_accessor :progname
 
+  public
   def call(severity, timestamp, who, object)
     # override progname to be the caller if the log level threshold is DEBUG
     # We only do this if the logger level is DEBUG because inspecting the

data/lib/logstash/namespace.rb

@@ -1,4 +1,3 @@
-
 module LogStash
   module Inputs; end
   module Outputs; end

data/lib/logstash/outputs.rb

@@ -1,8 +1,8 @@
-
 require "logstash/namespace"
 require "uri"
 
 module LogStash::Outputs
+  public
   def self.from_url(url, &block)
     uri = URI.parse(url)
     # TODO(sissel): Add error handling

data/lib/logstash/outputs/amqp.rb

@@ -1,9 +1,12 @@
-require "logstash/outputs/base"
 require "amqp" # rubygem 'amqp'
+require "logstash/outputs/base"
+require "logstash/namespace"
 require "mq" # rubygem 'amqp'
 
 class LogStash::Outputs::Amqp < LogStash::Outputs::Base
   MQTYPES = [ "fanout", "queue", "topic" ]
+
+  public
   def initialize(url, config={}, &block)
     super
 
@@ -18,9 +21,10 @@ class LogStash::Outputs::Amqp < LogStash::Outputs::Base
     end
   end # def initialize
 
+  public
   def register
     @logger.info("Registering output #{@url}")
-    @amqp = AMQP.connect(:host => @url.host)
+    @amqp = AMQP.connect(:host => @url.host, :port => (@url.port or 5672))
     @mq = MQ.new(@amqp)
     @target = nil
 
@@ -34,11 +38,14 @@ class LogStash::Outputs::Amqp < LogStash::Outputs::Base
     end # case @mqtype
   end # def register
 
+  public
   def receive(event)
     @logger.debug(["Sending event", { :url => @url, :event => event }])
     @target.publish(event.to_json)
   end # def receive
 
+  # This is used by the ElasticSearch AMQP/River output.
+  public
   def receive_raw(raw)
     if @target == nil
       raise "had trouble registering AMQP URL #{@url.to_s}, @target is nil"

data/lib/logstash/outputs/base.rb

@@ -1,11 +1,13 @@
-require "logstash/namespace"
+require "cgi"
 require "logstash/event"
 require "logstash/logging"
-require "cgi"
+require "logstash/namespace"
 require "uri"
 
 class LogStash::Outputs::Base
   attr_accessor :logger
+
+  public
   def initialize(url, config={}, &block)
     @url = url
     @url = URI.parse(url) if url.is_a? String
@@ -20,11 +22,13 @@ class LogStash::Outputs::Base
     end
   end
 
+  public
   def register
     raise "#{self.class}#register must be overidden"
   end # def register
 
+  public
   def receive(event)
     raise "#{self.class}#receive must be overidden"
-  end
+  end # def receive
 end # class LogStash::Outputs::Base

data/lib/logstash/outputs/beanstalk.rb

@@ -1,7 +1,9 @@
 require "logstash/outputs/base"
+require "logstash/namespace"
 require "em-jack"
 
 class LogStash::Outputs::Beanstalk < LogStash::Outputs::Base
+  public
   def initialize(url, config={}, &block)
     super
 
@@ -11,6 +13,7 @@ class LogStash::Outputs::Beanstalk < LogStash::Outputs::Base
     end
   end
 
+  public
   def register
     tube = @url.path[1..-1] # Skip leading '/'
     port = @url.port || 11300
@@ -19,6 +22,7 @@ class LogStash::Outputs::Beanstalk < LogStash::Outputs::Base
                                         :tube => tube)
   end # def register
 
+  public
   def receive(event)
     @beanstalk.put(event.to_json, :ttr => @ttr)
   end # def receive

data/lib/logstash/outputs/elasticsearch.rb

@@ -1,36 +1,78 @@
-require "logstash/outputs/base"
-require "logstash/outputs/amqp"
 require "em-http-request"
+require "logstash/namespace"
+require "logstash/outputs/amqp"
+require "logstash/outputs/base"
 
 class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
-  def initialize(url, config={}, &block)
-    super
-  end
-
+  public
   def register
+    @pending = []
     # Port?
     # Authentication?
-    @httpurl = @url.clone
-    @httpurl.scheme = "http"
+    @esurl = @url.clone
+    @esurl.scheme = "http"
+    @esurl.path = "/" + @url.path.split("/")[1]
     defaults = {"method" => "http"}
     params = defaults.merge(@urlopts)
 
+    # Describe this index to elasticsearch
+    indexmap = {
+      # The name of the index
+      "settings" => { 
+        @url.path.split("/")[-1] => {
+          "mappings" => {
+            "@source" => { "type" => "string" },
+            "@source_host" => { "type" => "string" },
+            "@source_path" => { "type" => "string" },
+            "@timestamp" => { "type" => "date" },
+            "@tags" => { "type" => "string" },
+            "@message" => { "type" => "string" },
+
+            # TODO(sissel): Hack for now until this bug is resolved:
+            # https://github.com/elasticsearch/elasticsearch/issues/issue/604
+            "@fields" => { 
+              "type" => "object",
+              "properties" => {
+                "HOSTNAME" => { "type" => "string" },
+              },
+            }, # "@fields"
+          }, # "properties"
+        }, # index map for this index type.
+      }, # "settings"
+    } # ES Index
+
+    indexurl = @esurl.to_s
+    indexmap_http = EventMachine::HttpRequest.new(indexurl)
+    indexmap_req = indexmap_http.put :body => indexmap.to_json
+    indexmap_req.callback do
+      @logger.info(["Done configuring index", indexurl, indexmap])
+      ready(params)
+    end
+    indexmap_req.errback do
+      @logger.warn(["Failure configuring index", @esurl.to_s, indexmap])
+    end
+  end # def register
+
+  public
+  def ready(params)
     case params["method"]
     when "http"
-      @logger.debug "ElasticSearch using http with URL #{@httpurl.to_s}"
-      @http = EventMachine::HttpRequest.new(@httpurl.to_s)
+      @logger.debug "ElasticSearch using http with URL #{@url.to_s}"
+      @http = EventMachine::HttpRequest.new(@url.to_s)
       @callback = self.method(:receive_http)
     when "river"
-      mq_url = URI::parse("amqp://#{params["host"]}/queue/#{params["queue"]}?durable=1")
+      params["port"] ||= 5672
+      mq_url = URI::parse("amqp://#{params["host"]}:#{params["port"]}/queue/#{params["queue"]}?durable=1")
       @mq = LogStash::Outputs::Amqp.new(mq_url.to_s)
       @mq.register
       @callback = self.method(:receive_river)
-      em_url = URI.parse("http://#{@httpurl.host}:#{@httpurl.port}/_river/logstash#{@httpurl.path.tr("/", "_")}/_meta")
-      unused, @es_index, @es_type = @httpurl.path.split("/", 3)
+      em_url = URI.parse("http://#{@url.host}:#{@url.port}/_river/logstash#{@url.path.tr("/", "_")}/_meta")
+      unused, @es_index, @es_type = @url.path.split("/", 3)
 
       river_config = {"type" => params["type"],
                       params["type"] => {"host" => params["host"],
                                          "user" => params["user"],
+                                         "port" => params["port"],
                                          "pass" => params["pass"],
                                          "vhost" => params["vhost"],
                                          "queue" => params["queue"],
@@ -46,21 +88,47 @@ class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
       req.errback do
         @logger.warn "Error setting up river: #{req.response}"
       end
+      @callback = self.method(:receive_river)
     else raise "unknown elasticsearch method #{params["method"].inspect}"
     end
-  end # def register
 
+    receive(LogStash::Event.new({
+      "@source" => "@logstashinit",
+      "@type" => "@none",
+      "@message" => "Starting logstash output to elasticsearch",
+      "@fields" => {
+        "HOSTNAME" => Socket.gethostname
+      },
+    }))
+
+    pending = @pending
+    @pending = []
+    pending.each do |event|
+      receive(event)
+    end
+  end # def ready
+
+  public
   def receive(event)
-    @callback.call(event)
+    if @callback
+      @callback.call(event)
+    else
+      @pending << event
+    end
   end # def receive
 
-  def receive_http(event)
+  public
+  def receive_http(event, tries=5)
     req = @http.post :body => event.to_json
     req.errback do
-      $stderr.puts "Request to index to #{@httpurl.to_s} failed. Event was #{event.to_s}"
+      $stderr.puts "Request to index to #{@url.to_s} failed (will retry, #{tries} tries left). Event was #{event.to_s}"
+      EventMachine::add_timer(2) do
+        receive_http(event, tries - 1)
+      end
     end
   end # def receive_http
 
+  public
   def receive_river(event)
     # bulk format; see http://www.elasticsearch.com/docs/elasticsearch/river/rabbitmq/
     index_message = {"index" => {"_index" => @es_index, "_type" => @es_type}}.to_json + "\n"
@@ -68,4 +136,4 @@ class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
     index_message += event.to_hash.to_json + "\n"
     @mq.receive_raw(index_message)
   end # def receive_river
-end # class LogStash::Outputs::Websocket
+end # class LogStash::Outputs::Elasticsearch

data/lib/logstash/outputs/gelf.rb

@@ -4,18 +4,17 @@
 # This class doesn't currently use 'eventmachine'-style code, so it may
 # block things. Whatever, we can fix that later ;)
 
-require "gelf"
+require "gelf" # rubygem 'gelf'
+require "logstash/namespace"
 require "logstash/outputs/base"
 
 class LogStash::Outputs::Gelf < LogStash::Outputs::Base
-  def initialize(url, config={}, &block)
-    super
-  end
-
+  public
   def register
     # nothing to do
   end # def register
 
+  public
   def receive(event)
     # TODO(sissel): Use Gelf::Message instead
     gelf = Gelf.new(@url.host, (@url.port or 12201))
@@ -31,5 +30,5 @@ class LogStash::Outputs::Gelf < LogStash::Outputs::Base
     end
     gelf.add_additional "event_timestamp", event.timestamp
     gelf.send
-  end # def event
+  end # def receive
 end # class LogStash::Outputs::Gelf

data/lib/logstash/outputs/internal.rb

@@ -1,15 +1,19 @@
+require "logstash/namespace"
 require "logstash/outputs/base"
 
 class LogStash::Outputs::Internal < LogStash::Outputs::Base
+  public
   def initialize(url, config={}, &block)
     super
     @callback = block
-  end
+  end # def initialize
 
+  public
   def register
     @logger.info("Registering output #{@url}")
   end # def register
 
+  public
   def receive(event)
     if !@callback
       @logger.error("No callback for output #{@url}, cannot receive")
@@ -19,11 +23,13 @@ class LogStash::Outputs::Internal < LogStash::Outputs::Base
   end # def event
 
   # Set the callback by passing a block of code
+  public
   def callback(&block)
     @callback = block
   end
 
   # Set the callback by passing a proc object
+  public
   def callback=(proc_block)
     @callback = proc_block
   end

data/lib/logstash/outputs/mongodb.rb

@@ -1,19 +1,19 @@
 require "logstash/outputs/base"
+require "logstash/namespace"
 require "em-mongo"
 
 class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
-  def initialize(url, config={}, &block)
-    super
-  end
-
+  public
   def register
-    # Port?
-    # Authentication?
-    db = @url.path[1..-1] # Skip leading '/'
-    @mongodb = EventMachine::Mongo::Connection.new(@url.host).db(db)
+    # TODO(sissel): Port?
+    # TODO(sissel): Authentication?
+    # db and collection are mongodb://.../db/collection
+    unused, @db, @collection = @url.path.split("/", 3)
+    @mongodb = EventMachine::Mongo::Connection.new(@url.host).db(@db)
   end # def register
 
+  public
   def receive(event)
-    @mongodb.collection("events").insert(event.to_hash)
-  end # def event
+    @mongodb.collection(@collection).insert(event.to_hash)
+  end # def receive
 end # class LogStash::Outputs::Mongodb

data/lib/logstash/outputs/nagios.rb

@@ -1,9 +1,11 @@
+require "logstash/namespace"
 require "logstash/outputs/base"
 
 class LogStash::Outputs::Nagios < LogStash::Outputs::Base
   NAGIOS_CRITICAL = 2
   NAGIOS_WARN = 1
 
+  public
   def initialize(url, config={}, &block)
     super
 
@@ -12,12 +14,14 @@ class LogStash::Outputs::Nagios < LogStash::Outputs::Base
     else
       @cmdfile = @url.path
     end
-  end
+  end # def initialize
 
+  public
   def register
     # nothing to do
   end # def register
 
+  public
   def receive(event)
     if !File.exists?(@cmdfile)
       @logger.warn(["Skipping nagios output; command file is missing",
@@ -68,5 +72,5 @@ class LogStash::Outputs::Nagios < LogStash::Outputs::Base
                    {"error" => $!, "cmdfile" => @cmdfile,
                     "missed_event" => event}])
     end
-  end # def event
+  end # def receive
 end # class LogStash::Outputs::Nagios

data/lib/logstash/outputs/stdout.rb

@@ -1,14 +1,13 @@
 require "logstash/outputs/base"
+require "logstash/namespace"
 
 class LogStash::Outputs::Stdout < LogStash::Outputs::Base
-  def initialize(url, config={}, &block)
-    super
-  end
-
+  public
   def register
     # nothing to do
   end # def register
 
+  public
   def receive(event)
     puts event
   end # def event