logstash-lib 1.3.2

data/lib/logstash/inputs/drupal_dblog/jdbcconnection.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+require "java"
+require "rubygems"
+require "jdbc/mysql"
+
+java_import "com.mysql.jdbc.Driver"
+
+# A JDBC mysql connection class.
+# The interface is compatible with the mysql2 API.
+class LogStash::DrupalDblogJavaMysqlConnection
+
+  def initialize(host, username, password, database, port = nil)
+    port ||= 3306
+
+    address = "jdbc:mysql://#{host}:#{port}/#{database}"
+    @connection = java.sql.DriverManager.getConnection(address, username, password)
+  end # def initialize
+
+  def query(sql)
+    if sql =~ /select/i
+      return select(sql)
+    else
+      return update(sql)
+    end
+  end # def query
+
+  def select(sql)
+    stmt = @connection.createStatement
+    resultSet = stmt.executeQuery(sql)
+
+    meta = resultSet.getMetaData
+    column_count = meta.getColumnCount
+
+    rows = []
+
+    while resultSet.next
+      res = {}
+
+      (1..column_count).each do |i|
+        name = meta.getColumnName(i)
+        case meta.getColumnType(i)
+        when java.sql.Types::INTEGER
+          res[name] = resultSet.getInt(name)
+        else
+          res[name] = resultSet.getString(name)
+        end
+      end
+
+      rows << res
+    end
+
+    stmt.close
+    return rows
+  end # def select
+
+  def update(sql)
+    stmt = @connection.createStatement
+    stmt.execute_update(sql)
+    stmt.close
+  end # def update
+
+  def close
+    @connection.close
+  end # def close
+
+end # class LogStash::DrupalDblogJavaMysqlConnection

data/lib/logstash/inputs/elasticsearch.rb

@@ -0,0 +1,140 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "logstash/util/socket_peer"
+
+# Read from elasticsearch.
+#
+# This is useful for replay testing logs, reindexing, etc.
+#
+# Example:
+#
+#     input {
+#       # Read all documents from elasticsearch matching the given query
+#       elasticsearch {
+#         host => "localhost"
+#         query => "ERROR"
+#       }
+#     }
+#
+# * TODO(sissel): configurable scroll timeout
+# * TODO(sissel): Option to keep the index, type, and doc id so we can do reindexing?
+class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
+  config_name "elasticsearch"
+  milestone 1
+
+  default :codec, "json"
+
+  # The address of your elasticsearch server
+  config :host, :validate => :string, :required => true
+
+  # The http port of your elasticsearch server's REST interface
+  config :port, :validate => :number, :default => 9200
+
+  # The index to search
+  config :index, :validate => :string, :default => "logstash-*"
+
+  # The query to use
+  config :query, :validate => :string, :default => "*"
+
+  # Enable the scan search_type.
+  # This will disable sorting but increase speed and performance.
+  config :scan, :validate => :boolean, :default => true
+
+  # This allows you to set the number of items you get back per scroll
+  config :size, :validate => :number, :default => 1000
+
+  # this parameter controls the keep alive time of the scrolling request and initiates the scrolling process.
+  # The timeout applies per round trip (i.e. between the previous scan scroll request, to the next).
+  config :scroll, :validate => :string, :default => "1m"
+
+  public
+  def register
+    require "ftw"
+    @agent = FTW::Agent.new
+    params = {
+      "q" => @query,
+      "scroll" => @scroll,
+      "size" => "#{@size}",
+    }
+
+    params['search_type'] = "scan" if @scan
+
+    @url = "http://#{@host}:#{@port}/#{@index}/_search?#{encode(params)}"
+  end # def register
+
+  private
+  def encode(hash)
+    return hash.collect do |key, value|
+      CGI.escape(key) + "=" + CGI.escape(value)
+    end.join("&")
+  end # def encode
+
+  public
+  def run(output_queue)
+
+    # Execute the search request
+    response = @agent.get!(@url)
+    json = ""
+    response.read_body { |c| json << c }
+    result = JSON.parse(json)
+    scroll_id = result["_scroll_id"]
+
+    # When using the search_type=scan we don't get an initial result set.
+    # So we do it here.
+    if @scan
+
+      scroll_params = {
+        "scroll_id" => scroll_id,
+        "scroll" => @scroll
+      }
+
+      scroll_url = "http://#{@host}:#{@port}/_search/scroll?#{encode(scroll_params)}"
+      response = @agent.get!(scroll_url)
+      json = ""
+      response.read_body { |c| json << c }
+      result = JSON.parse(json)
+
+    end
+
+    while true
+      break if result.nil?
+      hits = result["hits"]["hits"]
+      break if hits.empty?
+
+      hits.each do |hit|
+        event = hit["_source"]
+
+        # Hack to make codecs work
+        @codec.decode(event.to_json) do |event|
+          decorate(event)
+          output_queue << event
+        end
+      end
+
+      # Get the scroll id from the previous result set and use it for getting the next data set
+      scroll_id = result["_scroll_id"]
+
+      # Fetch the next result set
+      scroll_params = {
+        "scroll_id" => scroll_id,
+        "scroll" => @scroll
+      }
+      scroll_url = "http://#{@host}:#{@port}/_search/scroll?#{encode(scroll_params)}"
+
+      response = @agent.get!(scroll_url)
+      json = ""
+      response.read_body { |c| json << c }
+      result = JSON.parse(json)
+
+      if result["error"]
+        @logger.warn(result["error"], :request => scroll_url)
+        # TODO(sissel): raise an error instead of breaking
+        break
+      end
+
+    end
+  rescue LogStash::ShutdownSignal
+    # Do nothing, let us quit.
+  end # def run
+end # class LogStash::Inputs::Elasticsearch

data/lib/logstash/inputs/eventlog.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "socket"
+
+# Pull events from a Windows Event Log
+#
+# To collect Events from the System Event Log, use a config like:
+#
+#     input {
+#       eventlog {
+#         type  => 'Win32-EventLog'
+#         logfile  => 'System'
+#       }
+#     }
+class LogStash::Inputs::EventLog < LogStash::Inputs::Base
+
+  config_name "eventlog"
+  milestone 2
+
+  default :codec, "plain"
+
+  # Event Log Name
+  config :logfile, :validate => :array, :default => [ "Application", "Security", "System" ]
+
+  public
+  def register
+
+    # wrap specified logfiles in suitable OR statements
+    @logfiles = @logfile.join("' OR TargetInstance.LogFile = '")
+
+    @hostname = Socket.gethostname
+    @logger.info("Registering input eventlog://#{@hostname}/#{@logfile}")
+
+    if RUBY_PLATFORM == "java"
+      require "logstash/inputs/eventlog/racob_fix"
+      require "jruby-win32ole"
+    else
+      require "win32ole"
+    end
+  end # def register
+
+  public
+  def run(queue)
+    @wmi = WIN32OLE.connect("winmgmts://")
+
+    wmi_query = "Select * from __InstanceCreationEvent Where TargetInstance ISA 'Win32_NTLogEvent' And (TargetInstance.LogFile = '#{@logfiles}')"
+
+    begin
+      @logger.debug("Tailing Windows Event Log '#{@logfile}'")
+
+      events = @wmi.ExecNotificationQuery(wmi_query)
+
+      while
+        notification = events.NextEvent
+        event = notification.TargetInstance
+
+        timestamp = to_timestamp(event.TimeGenerated)
+
+        e = LogStash::Event.new(
+          "host" => @hostname,
+          "path" => @logfile,
+          "type" => @type,
+          "@timestamp" => timestamp
+        )
+
+        %w{Category CategoryString ComputerName EventCode EventIdentifier
+            EventType Logfile Message RecordNumber SourceName
+            TimeGenerated TimeWritten Type User
+        }.each{
+            |property| e[property] = event.send property 
+        }
+
+        if RUBY_PLATFORM == "java"
+          # unwrap jruby-win32ole racob data
+          e["InsertionStrings"] = unwrap_racob_variant_array(event.InsertionStrings)
+          data = unwrap_racob_variant_array(event.Data)
+          # Data is an array of signed shorts, so convert to bytes and pack a string
+          e["Data"] = data.map{|byte| (byte > 0) ? byte : 256 + byte}.pack("c*")
+        else
+          # win32-ole data does not need to be unwrapped
+          e["InsertionStrings"] = event.InsertionStrings
+          e["Data"] = event.Data
+        end
+
+        e["message"] = event.Message
+
+        decorate(e)
+        queue << e
+
+      end # while
+
+    rescue Exception => ex
+      @logger.error("Windows Event Log error: #{ex}\n#{ex.backtrace}")
+      sleep 1
+      retry
+    end # rescue
+
+  end # def run
+
+  private
+  def unwrap_racob_variant_array(variants)
+    variants ||= []
+    variants.map {|v| (v.respond_to? :getValue) ? v.getValue : v}
+  end # def unwrap_racob_variant_array
+
+  # the event log timestamp is a utc string in the following format: yyyymmddHHMMSS.xxxxxx±UUU
+  # http://technet.microsoft.com/en-us/library/ee198928.aspx
+  private
+  def to_timestamp(wmi_time)
+    result = ""
+    # parse the utc date string
+    /(?<w_date>\d{8})(?<w_time>\d{6})\.\d{6}(?<w_sign>[\+-])(?<w_diff>\d{3})/ =~ wmi_time
+    result = "#{w_date}T#{w_time}#{w_sign}"
+    # the offset is represented by the difference, in minutes, 
+    # between the local time zone and Greenwich Mean Time (GMT).
+    if w_diff.to_i > 0
+      # calculate the timezone offset in hours and minutes
+      h_offset = w_diff.to_i / 60
+      m_offset = w_diff.to_i - (h_offset * 60)
+      result.concat("%02d%02d" % [h_offset, m_offset])
+    else
+      result.concat("0000")
+    end
+  
+    return DateTime.strptime(result, "%Y%m%dT%H%M%S%z").iso8601
+  end
+end # class LogStash::Inputs::EventLog
+

data/lib/logstash/inputs/eventlog/racob_fix.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+# The jruby-win32ole gem uses 'java.lang.System.set_property' to
+# tell java(?) where to find the racob dll.
+#
+# However, it fails when it tries to load the racob dll from the jar
+# (UnsatisfiedLinkError).
+#
+# So easy fix, right? Monkeypatch the set_property to do two things:
+#   - extract the racob dll somewhere
+#   - set the property to the extracted path
+#
+
+require "fileutils"
+require "tmpdir"
+
+class << java.lang.System
+  alias_method :set_property_seriously, :set_property
+  def set_property(key, value)
+    if key == "racob.dll.path" && value =~ /file:.*\.jar!\//
+      # Path is set in a jar, we'll need to extract it to a
+      # temporary location, then load it.
+      filename = File.basename(value)
+      extracted_path = File.join(Dir.tmpdir, filename)
+      # FileUtils.cp (and copy, and copy_file) are broken
+      # when copying this file. I have not debugged it other
+      # than to see it doesn't work. racob_x86.dll is 105kb,
+      # but FileUtils.cp only copies 4kb of it.
+      
+      # open both files in 'binary' mode (sets encoding to BINARY aka
+      # ASCII-8BIT).
+      input = File.new(value, "rb") 
+      output = File.new(extracted_path, "wb")
+      while chunk = input.read(16384)
+        output.write(chunk) 
+      end
+      input.close
+      output.close
+
+      return set_property_seriously(key, extracted_path)
+    else
+      return set_property_seriously(key, value)
+    end
+  end
+end

data/lib/logstash/inputs/exec.rb

@@ -0,0 +1,69 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "socket" # for Socket.gethostname
+
+# Run command line tools and capture the whole output as an event.
+#
+# Notes:
+#
+# * The '@source' of this event will be the command run.
+# * The '@message' of this event will be the entire stdout of the command
+#   as one event.
+#
+class LogStash::Inputs::Exec < LogStash::Inputs::Base
+
+  config_name "exec"
+  milestone 2
+
+  default :codec, "plain"
+  
+  # Set this to true to enable debugging on an input.
+  config :debug, :validate => :boolean, :default => false
+  
+  # Command to run. For example, "uptime"
+  config :command, :validate => :string, :required => true
+  
+  # Interval to run the command. Value is in seconds.
+  config :interval, :validate => :number, :required => true
+  
+  public
+  def register
+    @logger.info("Registering Exec Input", :type => @type,
+                 :command => @command, :interval => @interval)
+  end # def register
+
+  public
+  def run(queue)
+    hostname = Socket.gethostname
+    loop do
+      start = Time.now
+      @logger.info("Running exec", :command => @command) if @debug
+      out = IO.popen(@command)
+      # out.read will block until the process finishes.
+      @codec.decode(out.read) do |event|
+        decorate(event)
+        event["host"] = hostname
+        event["command"] = @command
+        queue << event
+      end
+      
+      duration = Time.now - start
+      if @debug
+        @logger.info("Command completed", :command => @command,
+                     :duration => duration)
+      end
+
+      # Sleep for the remainder of the interval, or 0 if the duration ran
+      # longer than the interval.
+      sleeptime = [0, @interval - duration].max
+      if sleeptime == 0
+        @logger.warn("Execution ran longer than the interval. Skipping sleep.",
+                     :command => @command, :duration => duration,
+                     :interval => @interval)
+      else
+        sleep(sleeptime)
+      end
+    end # loop
+  end # def run
+end # class LogStash::Inputs::Exec