logstash-lib 1.3.2

data/lib/logstash/inputs/collectd.rb

@@ -0,0 +1,306 @@
+# encoding utf-8
+require "date"
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "socket"
+require "tempfile"
+require "time"
+
+# Read events from the connectd binary protocol over the network via udp.
+# See https://collectd.org/wiki/index.php/Binary_protocol
+#
+# Configuration in your Logstash configuration file can be as simple as:
+#     input {
+#       collectd {}
+#     }
+#
+# A sample collectd.conf to send to Logstash might be:
+#
+#     Hostname    "host.example.com"
+#     LoadPlugin interface
+#     LoadPlugin load
+#     LoadPlugin memory
+#     LoadPlugin network
+#     <Plugin interface>
+#         Interface "eth0"
+#         IgnoreSelected false
+#     </Plugin>
+#     <Plugin network>
+#         <Server "10.0.0.1" "25826">
+#         </Server>
+#     </Plugin>
+#
+# Be sure to replace "10.0.0.1" with the IP of your Logstash instance.
+#
+
+#
+class LogStash::Inputs::Collectd < LogStash::Inputs::Base
+  config_name "collectd"
+  milestone 1
+
+  # File path(s) to collectd types.db to use.
+  # The last matching pattern wins if you have identical pattern names in multiple files.
+  # If no types.db is provided the included types.db will be used (currently 5.4.0).
+  config :typesdb, :validate => :array
+
+  # The address to listen on.  Defaults to all available addresses.
+  config :host, :validate => :string, :default => "0.0.0.0"
+
+  # The port to listen on.  Defaults to the collectd expected port of 25826.
+  config :port, :validate => :number, :default => 25826
+  
+  # Prune interval records.  Defaults to true.
+  config :prune_intervals, :validate => :boolean, :default => true
+
+  # Buffer size. 1452 is the collectd default for v5+
+  config :buffer_size, :validate => :number, :default => 1452
+
+  public
+  def initialize(params)
+    super
+    BasicSocket.do_not_reverse_lookup = true
+    @idbyte = 0
+    @length = 0
+    @prev_typenum = 0
+    @header = []; @body = []
+    @timestamp = Time.now().utc
+    @collectd = {}
+    @types = {}
+  end # def initialize
+
+  public
+  def register
+    @udp = nil
+    if @typesdb.nil?
+      if __FILE__ =~ /^file:\/.+!.+/
+        begin
+          # Running from a jar, assume types.db is at the root.
+          jar_path = [__FILE__.split("!").first, "/types.db"].join("!")
+          @typesdb = [jar_path]
+        rescue => ex
+          raise "Failed to cache, due to: #{ex}\n#{ex.backtrace}"
+        end
+      else
+        if File.exists?("types.db")
+          @typesdb = ["types.db"]
+        elsif File.exists?("vendor/collectd/types.db")
+          @typesdb = ["vendor/collectd/types.db"]
+        else
+          raise "You must specify 'typesdb => ...' in your collectd input"
+        end
+      end
+    end
+    @logger.info("Using internal types.db", :typesdb => @typesdb.to_s)
+  end # def register
+
+  public
+  def run(output_queue)
+    begin
+      # get types
+      get_types(@typesdb)
+      # collectd server
+      collectd_listener(output_queue)
+    rescue LogStash::ShutdownSignal
+      # do nothing, shutdown was requested.
+    rescue => e
+      @logger.warn("Collectd listener died", :exception => e, :backtrace => e.backtrace)
+      sleep(5)
+      retry
+    end # begin
+  end # def run
+
+  public
+  def get_types(paths)
+    # Get the typesdb
+    paths.each do |path|
+      @logger.info("Getting Collectd typesdb info", :typesdb => path.to_s)
+      File.open(path, 'r').each_line do |line|
+        typename, *line = line.strip.split
+        next if typename.nil? || if typename[0,1] != '#' # Don't process commented or blank lines
+          v = line.collect { |l| l.strip.split(":")[0] }
+          @types[typename] = v
+        end
+      end
+    end
+  @logger.debug("Collectd Types", :types => @types.to_s)
+  end # def get_types
+
+  public
+  def type_map(id)
+    case id
+      when 0;   return "host"
+      when 1,8; return "@timestamp"
+      when 2;   return "plugin"
+      when 3;   return "plugin_instance"
+      when 4;   return "collectd_type"
+      when 5;   return "type_instance"
+      when 6;   return "values"
+      when 9;   return "interval"
+      when 100; return "message"
+      when 101; return "severity"
+    end
+  end # def type_map
+
+  public
+  def vt_map(id)
+    case id
+      when 0; return "COUNTER"
+      when 1; return "GAUGE"
+      when 2; return "DERIVE"
+      when 3; return "ABSOLUTE"
+      else;   return 'UNKNOWN'
+    end
+  end
+
+  public
+  def get_values(id, body)
+    retval = ''
+    case id
+      when 0,2,3,4,5,100 #=> String types
+        retval = body.pack("C*")
+        retval = retval[0..-2]
+      when 1 # Time
+        # Time here, in bit-shifted format.  Parse bytes into UTC.
+        byte1, byte2 = body.pack("C*").unpack("NN")
+        retval = Time.at(( ((byte1 << 32) + byte2))).utc
+      when 7,101 #=> Numeric types
+        retval = body.slice!(0..7).pack("C*").unpack("E")[0]
+      when 8 # Time, Hi-Res
+        # Time here, in bit-shifted format.  Parse bytes into UTC.
+        byte1, byte2 = body.pack("C*").unpack("NN")
+        retval = Time.at(( ((byte1 << 32) + byte2) * (2**-30) )).utc
+      when 9 # Interval, Hi-Res
+        byte1, byte2 = body.pack("C*").unpack("NN")
+        retval = (((byte1 << 32) + byte2) * (2**-30)).to_i
+      when 6 # Values
+        val_bytes = body.slice!(0..1)
+        val_count = val_bytes.pack("C*").unpack("n")
+        if body.length % 9 == 0 # Should be 9 fields
+          count = 0
+          retval = []
+          types = body.slice!(0..((body.length/9)-1))
+          while body.length > 0
+            vtype = vt_map(types[count])
+            case types[count]
+              when 0, 3; v = body.slice!(0..7).pack("C*").unpack("Q>")[0]
+              when 1;    v = body.slice!(0..7).pack("C*").unpack("E")[0]
+              when 2;    v = body.slice!(0..7).pack("C*").unpack("q>")[0]
+              else;      v = 0
+            end
+            retval << v
+            count += 1
+          end
+        else
+          @logger.error("Incorrect number of data fields for collectd record", :body => body.to_s)
+        end
+    end
+    # Populate some state variables based on their type...
+    case id
+      when 2
+        if @plugin != retval      # Zero-out @plugin_instance when @plugin changes
+          @plugin_instance = ''
+          @collectd.delete('plugin_instance')
+        end
+        @plugin = retval
+      when 0;   @cdhost = retval        
+      when 3;   @plugin_instance = retval
+      when 4;   @cdtype = retval
+      when 5;   @type_instance = retval
+      when 1,8; @timestamp = retval
+    end 
+    return retval
+  end # def get_values
+
+  private
+  def generate_event(data, output_queue)
+    # Prune these *specific* keys if they exist and are empty.
+    # This is better than looping over all keys every time.
+    data.delete('type_instance') if data['type_instance'] == ""
+    data.delete('plugin_instance') if data['plugin_instance'] == ""              
+    # As crazy as it sounds, this is where we actually send our events to the queue!
+    event = LogStash::Event.new
+    data.each {|k, v| event[k] = data[k]}
+    decorate(event)
+    output_queue << event
+  end # def generate_event
+  
+  private
+  def collectd_listener(output_queue)
+
+    @logger.info("Starting Collectd listener", :address => "#{@host}:#{@port}")
+
+    if @udp && ! @udp.closed?
+      @udp.close
+    end
+
+    @udp = UDPSocket.new(Socket::AF_INET)
+    @udp.bind(@host, @port)
+
+    loop do
+      payload, client = @udp.recvfrom(@buffer_size)
+      payload.each_byte do |byte|
+        # According to the documentation for the binary protocol
+        # it takes 4 bytes to define the header:
+        # The first 2 bytes are the type number,
+        # the second 2 bytes are the length of the message.
+        # So, until we have looped 4 times (@idbyte is our counter)
+        # append the byte to the @header
+        if @idbyte < 4
+          @header << byte
+        # Now that we have looped exactly 4 times...
+        elsif @idbyte == 4
+          @typenum = (@header[0] << 1) + @header[1] # @typenum gets the first 2 bytes
+          @length  = (@header[2] << 1) + @header[3] # @length gets the second 2 bytes
+          @body << byte                             # @body begins with the current byte
+        # And if we've looped more than 4, up until the length of the message (now defined)
+        elsif @idbyte > 4 && @idbyte < @length
+          @body << byte                             # append the current byte to @body
+        end
+        # So long as we have @length and we've reached it, it's time to parse
+        if @length > 0 && @idbyte == @length-1
+          field = type_map(@typenum)              # Get the field name based on type            
+          if @typenum < @prev_typenum             # We've started over, generate an event
+            if @prune_intervals
+              generate_event(@collectd, output_queue) unless @prev_typenum == 7 or @prev_typenum == 9
+            else
+              generate_event(@collectd, output_queue)
+            end
+            @collectd.clear                     # Empty @collectd
+            @collectd['host'] = @cdhost         # Reset these from state
+            @collectd['collectd_type'] = @cdtype
+            @collectd['plugin'] = @plugin       
+            @collectd['plugin_instance'] = @plugin_instance
+            @collectd['@timestamp'] = @timestamp
+          end
+          # Here is where we actually fill @collectd
+          values = get_values(@typenum, @body)
+          if values.kind_of?(Array)
+            if values.length > 1                  # Only do this iteration on multi-value arrays
+              values.each_with_index {|value, x| @collectd[@types[@collectd['collectd_type']][x]] = values[x]}
+            else                                  # Otherwise it's a single value
+              @collectd['value'] = values[0]      # So name it 'value' accordingly
+            end
+          elsif field != nil                      # Not an array, make sure it's non-empty
+            @collectd[field] = values             # Append values to @collectd under key field
+          end
+          @prev_typenum = @typenum
+          # All bytes in the collectd event have now been processed.  Reset counters, header & body.
+          @idbyte = 0; @length = 0; @header.clear; @body.clear;
+        else # Increment the byte positional counter
+          @idbyte += 1
+        end # End of if @length > 0 && @idbyte == @length-1
+      end   # End of payload.each_byte do |byte| loop
+    end     # End of loop do, payload, client = @udp.recvfrom(@buffer_size)  
+  ensure
+    if @udp
+      @udp.close_read rescue nil
+      @udp.close_write rescue nil
+    end
+  end # def collectd_listener
+
+  public
+  def teardown
+    @udp.close if @udp && !@udp.closed?
+  end
+
+end # class LogStash::Inputs::Collectd

data/lib/logstash/inputs/drupal_dblog.rb

@@ -0,0 +1,323 @@
+# encoding: utf-8
+require "date"
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+# Retrieve watchdog log events from a Drupal installation with DBLog enabled.
+# The events are pulled out directly from the database.
+# The original events are not deleted, and on every consecutive run only new
+# events are pulled.
+#
+# The last watchdog event id that was processed is stored in the Drupal
+# variable table with the name "logstash_last_wid". Delete this variable or
+# set it to 0 if you want to re-import all events.
+#
+# More info on DBLog: http://drupal.org/documentation/modules/dblog
+#
+class LogStash::Inputs::DrupalDblog < LogStash::Inputs::Base
+  config_name "drupal_dblog"
+  milestone 1
+
+  default :codec, "plain"
+
+  # Specify all drupal databases that you whish to import from.
+  # This can be as many as you whish.
+  # The format is a hash, with a unique site name as the key, and a databse
+  # url as the value.
+  #
+  # Example:
+  # [
+  #   "site1", "mysql://user1:password@host1.com/databasename",
+  #   "other_site", "mysql://user2:password@otherhost.com/databasename",
+  #   ...
+  # ]
+  config :databases, :validate => :hash
+
+  # By default, the event only contains the current user id as a field.
+  # If you whish to add the username as an additional field, set this to true.
+  config :add_usernames, :validate => :boolean, :default => false
+
+  # Time between checks in minutes.
+  config :interval, :validate => :number, :default => 10
+
+  # The amount of log messages that should be fetched with each query.
+  # Bulk fetching is done to prevent querying huge data sets when lots of
+  # messages are in the database.
+  config :bulksize, :validate => :number, :default => 5000
+
+  # Label this input with a type.
+  # Types are used mainly for filter activation.
+  #
+  #
+  # If you create an input with type "foobar", then only filters
+  # which also have type "foobar" will act on them.
+  #
+  # The type is also stored as part of the event itself, so you
+  # can also use the type to search for in the web interface.
+  config :type, :validate => :string, :default => 'watchdog'
+
+  public
+  def register
+    require "php_serialize"
+
+    if RUBY_PLATFORM == 'java'
+      require "logstash/inputs/drupal_dblog/jdbcconnection"
+    else
+      require "mysql2"
+    end
+  end # def register
+
+  public
+  def config_init(params)
+    super
+
+    dbs = {}
+    valid = true
+
+    @databases.each do |name, rawUri|
+      uri = URI(rawUri)
+
+      dbs[name] = {
+        "site" => name,
+        "scheme" => uri.scheme,
+        "host" => uri.host,
+        "user" => uri.user,
+        "password" => uri.password,
+        "database" => uri.path.sub('/', ''),
+        "port" => uri.port.to_i
+      }
+
+      if not (
+        uri.scheme and not uri.scheme.empty?\
+        and uri.host and not uri.host.empty?\
+        and uri.user and not uri.user.empty?\
+        and uri.password\
+        and uri.path and not uri.path.sub('/', '').empty?
+      )
+        @logger.error("Drupal DBLog: Invalid database URI for #{name} : #{rawUri}")
+        valid = false
+      end
+      if not uri.scheme == 'mysql'
+        @logger.error("Drupal DBLog: Only mysql databases are supported.")
+        valid = false
+      end
+    end
+
+    if not valid
+      @logger.error("Config validation failed.")
+      exit 1
+    end
+
+    @databases = dbs
+  end #def config_init
+
+  public
+  def run(output_queue)
+    @logger.info("Initializing drupal_dblog")
+
+    loop do
+      @logger.debug("Drupal DBLog: Starting to fetch new watchdog entries")
+      start = Time.now.to_i
+
+      @databases.each do |name, db|
+        @logger.debug("Drupal DBLog: Checking database #{name}")
+        check_database(output_queue, db)
+        @logger.info("Drupal DBLog: Retrieved all new watchdog messages from #{name}")
+      end
+
+      timeTaken = Time.now.to_i - start
+      @logger.info("Drupal DBLog: Fetched all new watchdog entries in #{timeTaken} seconds")
+
+      # If fetching of all databases took less time than the interval,
+      # sleep a bit.
+      sleepTime = @interval * 60 - timeTaken
+      if sleepTime > 0
+        @logger.debug("Drupal DBLog: Sleeping for #{sleepTime} seconds")
+        sleep(sleepTime)
+      end
+    end # loop
+  end # def run
+
+  private
+  def initialize_client(db)
+    if db["scheme"] == 'mysql'
+
+      if not db["port"] > 0
+        db["port"] = 3306
+      end
+
+      if RUBY_PLATFORM == 'java'
+        @client = LogStash::DrupalDblogJavaMysqlConnection.new(
+            db["host"],
+            db["user"],
+            db["password"],
+            db["database"],
+            db["port"]
+        )
+      else
+        @client = Mysql2::Client.new(
+            :host => db["host"],
+            :port => db["port"],
+            :username => db["user"],
+            :password => db["password"],
+            :database => db["database"]
+        )
+      end
+    end
+  end #def get_client
+
+  private
+  def check_database(output_queue, db)
+
+    begin
+      # connect to the MySQL server
+      initialize_client(db)
+    rescue Exception => e
+      @logger.error("Could not connect to database: " + e.message)
+      return
+    end #begin
+
+    begin
+      @sitename = db["site"]
+
+      @usermap = @add_usernames ? get_usermap : nil
+
+      # Retrieve last pulled watchdog entry id
+      initialLastWid = get_last_wid
+      lastWid = nil
+
+
+      if initialLastWid == false
+        lastWid = 0
+        set_last_wid(0, true)
+      else
+        lastWid = initialLastWid
+      end
+
+      # Fetch new entries, and create the event
+      while true
+        results = get_db_rows(lastWid)
+        if results.length() < 1
+          break
+        end
+
+        @logger.debug("Fetched " + results.length().to_s + " database rows")
+
+        results.each do |row|
+          event = build_event(row)
+          if event
+            decorate(event)
+            output_queue << event
+            lastWid = row['wid'].to_s
+          end
+        end
+
+        set_last_wid(lastWid, false)
+      end
+    rescue Exception => e
+      @logger.error("Error while fetching messages: ", :error => e.message)
+    end # begin
+
+    # Close connection
+    @client.close
+  end # def check_database
+
+  def get_db_rows(lastWid)
+    query = 'SELECT * from watchdog WHERE wid > ' + lastWid.to_s + " ORDER BY wid asc LIMIT " + @bulksize.to_s
+    return @client.query(query)
+  end # def get_db_rows
+
+  private
+  def update_sitename
+    if @sitename == ""
+      result = @client.query('SELECT value FROM variable WHERE name="site_name"')
+      if result.first()
+        @sitename = PHP.unserialize(result.first()['value'])
+      end
+    end
+  end # def update_sitename
+
+  private
+  def get_last_wid
+    result = @client.query('SELECT value FROM variable WHERE name="logstash_last_wid"')
+    lastWid = false
+
+    if result.count() > 0
+      tmp = result.first()["value"].gsub("i:", "").gsub(";", "")
+      lastWid = tmp.to_i.to_s == tmp ? tmp : "0"
+    end
+
+    return lastWid
+  end # def get_last_wid
+
+  private
+  def set_last_wid(wid, insert)
+    wid = PHP.serialize(wid.to_i)
+
+    # Update last import wid variable
+    if insert
+      # Does not exist yet, so insert
+      @client.query('INSERT INTO variable (name, value) VALUES("logstash_last_wid", "' + wid + '")')
+    else
+      @client.query('UPDATE variable SET value="' + wid + '" WHERE name="logstash_last_wid"')
+    end
+  end # def set_last_wid
+
+  private
+  def get_usermap
+    map = {}
+
+    @client.query("SELECT uid, name FROM users").each do |row|
+      map[row["uid"]] = row["name"]
+    end
+
+    map[0] = "guest"
+    return map
+  end # def get_usermap
+
+  private
+  def build_event(row)
+    # Convert unix timestamp
+    timestamp = Time.at(row["timestamp"]).to_datetime.iso8601
+
+    msg = row["message"]
+    vars = {}
+
+    # Unserialize the variables, and construct the message
+    if row['variables'] != 'N;'
+      vars = PHP.unserialize(row["variables"])
+
+      if vars.is_a?(Hash)
+        vars.each_pair do |k, v|
+          if msg.scan(k).length() > 0
+            msg = msg.gsub(k.to_s, v.to_s)
+          else
+            # If not inside the message, add var as an additional field
+            row["variable_" + k] = v
+          end
+        end
+      end
+    end
+
+    row.delete("message")
+    row.delete("variables")
+    row.delete("timestamp")
+
+    row["severity"] = row["severity"].to_i
+
+    if @add_usernames and @usermap.has_key?(row["uid"])
+      row["user"] = @usermap[row["uid"]]
+    end
+
+    entry = {
+      "@timestamp" => timestamp,
+      "tags" => [],
+      "type" => "watchdog",
+      "site" => @sitename,
+      "message" => msg
+    }.merge(row)
+
+    return LogStash::Event.new(entry)
+  end # def build_event
+
+end # class LogStash::Inputs::DrupalDblog