logstash-input-drupal_dblog 0.1.1-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ab1d14e9119a017b64d778d9b8f84b76898556ad
+  data.tar.gz: 74b57346691258168e9832c9ae3d8731986d0569
+SHA512:
+  metadata.gz: d823b5c58ef1f8c6574d6c0fdf7ce9fef18e3d0ce533c5ee523161b0942a86c87ed9a2dfb984ec4a3d5d993fa30c03d00bfe864b84bc0999701a67b3536d97ff
+  data.tar.gz: c576c6d730e8f697091edbb9d9a6529cdbfb652f959f09ecd94b202534be3d55e603db8ee09ff60d8e6387dbd5d73fab129b9dccb040670f2eb8ddef73015cfe

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+Gemfile.lock
+.bundle
+vendor

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/inputs/drupal_dblog.rb

@@ -0,0 +1,324 @@
+# encoding: utf-8
+require "date"
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+
+# Retrieve watchdog log events from a Drupal installation with DBLog enabled.
+# The events are pulled out directly from the database.
+# The original events are not deleted, and on every consecutive run only new
+# events are pulled.
+#
+# The last watchdog event id that was processed is stored in the Drupal
+# variable table with the name "logstash_last_wid". Delete this variable or
+# set it to 0 if you want to re-import all events.
+#
+# More info on DBLog: http://drupal.org/documentation/modules/dblog
+#
+class LogStash::Inputs::DrupalDblog < LogStash::Inputs::Base
+  config_name "drupal_dblog"
+  milestone 1
+
+  default :codec, "plain"
+
+  # Specify all drupal databases that you whish to import from.
+  # This can be as many as you whish.
+  # The format is a hash, with a unique site name as the key, and a databse
+  # url as the value.
+  #
+  # Example:
+  # [
+  #   "site1", "mysql://user1:password@host1.com/databasename",
+  #   "other_site", "mysql://user2:password@otherhost.com/databasename",
+  #   ...
+  # ]
+  config :databases, :validate => :hash
+
+  # By default, the event only contains the current user id as a field.
+  # If you whish to add the username as an additional field, set this to true.
+  config :add_usernames, :validate => :boolean, :default => false
+
+  # Time between checks in minutes.
+  config :interval, :validate => :number, :default => 10
+
+  # The amount of log messages that should be fetched with each query.
+  # Bulk fetching is done to prevent querying huge data sets when lots of
+  # messages are in the database.
+  config :bulksize, :validate => :number, :default => 5000
+
+  # Label this input with a type.
+  # Types are used mainly for filter activation.
+  #
+  #
+  # If you create an input with type "foobar", then only filters
+  # which also have type "foobar" will act on them.
+  #
+  # The type is also stored as part of the event itself, so you
+  # can also use the type to search for in the web interface.
+  config :type, :validate => :string, :default => 'watchdog'
+
+  public
+  def register
+    require "php_serialize"
+
+    if RUBY_PLATFORM == 'java'
+      require "logstash/inputs/drupal_dblog/jdbcconnection"
+    else
+      require "mysql2"
+    end
+  end # def register
+
+  public
+  def config_init(params)
+    super
+
+    dbs = {}
+    valid = true
+
+    @databases.each do |name, rawUri|
+      uri = URI(rawUri)
+
+      dbs[name] = {
+        "site" => name,
+        "scheme" => uri.scheme,
+        "host" => uri.host,
+        "user" => uri.user,
+        "password" => uri.password,
+        "database" => uri.path.sub('/', ''),
+        "port" => uri.port.to_i
+      }
+
+      if not (
+        uri.scheme and not uri.scheme.empty?\
+        and uri.host and not uri.host.empty?\
+        and uri.user and not uri.user.empty?\
+        and uri.password\
+        and uri.path and not uri.path.sub('/', '').empty?
+      )
+        @logger.error("Drupal DBLog: Invalid database URI for #{name} : #{rawUri}")
+        valid = false
+      end
+      if not uri.scheme == 'mysql'
+        @logger.error("Drupal DBLog: Only mysql databases are supported.")
+        valid = false
+      end
+    end
+
+    if not valid
+      @logger.error("Config validation failed.")
+      exit 1
+    end
+
+    @databases = dbs
+  end #def config_init
+
+  public
+  def run(output_queue)
+    @logger.info("Initializing drupal_dblog")
+
+    loop do
+      @logger.debug("Drupal DBLog: Starting to fetch new watchdog entries")
+      start = Time.now.to_i
+
+      @databases.each do |name, db|
+        @logger.debug("Drupal DBLog: Checking database #{name}")
+        check_database(output_queue, db)
+        @logger.info("Drupal DBLog: Retrieved all new watchdog messages from #{name}")
+      end
+
+      timeTaken = Time.now.to_i - start
+      @logger.info("Drupal DBLog: Fetched all new watchdog entries in #{timeTaken} seconds")
+
+      # If fetching of all databases took less time than the interval,
+      # sleep a bit.
+      sleepTime = @interval * 60 - timeTaken
+      if sleepTime > 0
+        @logger.debug("Drupal DBLog: Sleeping for #{sleepTime} seconds")
+        sleep(sleepTime)
+      end
+    end # loop
+  end # def run
+
+  private
+  def initialize_client(db)
+    if db["scheme"] == 'mysql'
+
+      if not db["port"] > 0
+        db["port"] = 3306
+      end
+
+      if RUBY_PLATFORM == 'java'
+        @client = LogStash::DrupalDblogJavaMysqlConnection.new(
+            db["host"],
+            db["user"],
+            db["password"],
+            db["database"],
+            db["port"]
+        )
+      else
+        @client = Mysql2::Client.new(
+            :host => db["host"],
+            :port => db["port"],
+            :username => db["user"],
+            :password => db["password"],
+            :database => db["database"]
+        )
+      end
+    end
+  end #def get_client
+
+  private
+  def check_database(output_queue, db)
+
+    begin
+      # connect to the MySQL server
+      initialize_client(db)
+    rescue Exception => e
+      @logger.error("Could not connect to database: " + e.message)
+      return
+    end #begin
+
+    begin
+      @sitename = db["site"]
+
+      @usermap = @add_usernames ? get_usermap : nil
+
+      # Retrieve last pulled watchdog entry id
+      initialLastWid = get_last_wid
+      lastWid = nil
+
+
+      if initialLastWid == false
+        lastWid = 0
+        set_last_wid(0, true)
+      else
+        lastWid = initialLastWid
+      end
+
+      # Fetch new entries, and create the event
+      while true
+        results = get_db_rows(lastWid)
+        if results.length() < 1
+          break
+        end
+
+        @logger.debug("Fetched " + results.length().to_s + " database rows")
+
+        results.each do |row|
+          event = build_event(row)
+          if event
+            decorate(event)
+            output_queue << event
+            lastWid = row['wid'].to_s
+          end
+        end
+
+        set_last_wid(lastWid, false)
+      end
+    rescue Exception => e
+      @logger.error("Error while fetching messages: ", :error => e.message)
+    end # begin
+
+    # Close connection
+    @client.close
+  end # def check_database
+
+  def get_db_rows(lastWid)
+    query = 'SELECT * from watchdog WHERE wid > ' + lastWid.to_s + " ORDER BY wid asc LIMIT " + @bulksize.to_s
+    return @client.query(query)
+  end # def get_db_rows
+
+  private
+  def update_sitename
+    if @sitename == ""
+      result = @client.query('SELECT value FROM variable WHERE name="site_name"')
+      if result.first()
+        @sitename = PHP.unserialize(result.first()['value'])
+      end
+    end
+  end # def update_sitename
+
+  private
+  def get_last_wid
+    result = @client.query('SELECT value FROM variable WHERE name="logstash_last_wid"')
+    lastWid = false
+
+    if result.count() > 0
+      tmp = result.first()["value"].gsub("i:", "").gsub(";", "")
+      lastWid = tmp.to_i.to_s == tmp ? tmp : "0"
+    end
+
+    return lastWid
+  end # def get_last_wid
+
+  private
+  def set_last_wid(wid, insert)
+    wid = PHP.serialize(wid.to_i)
+
+    # Update last import wid variable
+    if insert
+      # Does not exist yet, so insert
+      @client.query('INSERT INTO variable (name, value) VALUES("logstash_last_wid", "' + wid + '")')
+    else
+      @client.query('UPDATE variable SET value="' + wid + '" WHERE name="logstash_last_wid"')
+    end
+  end # def set_last_wid
+
+  private
+  def get_usermap
+    map = {}
+
+    @client.query("SELECT uid, name FROM users").each do |row|
+      map[row["uid"]] = row["name"]
+    end
+
+    map[0] = "guest"
+    return map
+  end # def get_usermap
+
+  private
+  def build_event(row)
+    # Convert unix timestamp
+    timestamp = Time.at(row["timestamp"])
+
+    msg = row["message"]
+    vars = {}
+
+    # Unserialize the variables, and construct the message
+    if row['variables'] != 'N;'
+      vars = PHP.unserialize(row["variables"])
+
+      if vars.is_a?(Hash)
+        vars.each_pair do |k, v|
+          if msg.scan(k).length() > 0
+            msg = msg.gsub(k.to_s, v.to_s)
+          else
+            # If not inside the message, add var as an additional field
+            row["variable_" + k] = v
+          end
+        end
+      end
+    end
+
+    row.delete("message")
+    row.delete("variables")
+    row.delete("timestamp")
+
+    row["severity"] = row["severity"].to_i
+
+    if @add_usernames and @usermap.has_key?(row["uid"])
+      row["user"] = @usermap[row["uid"]]
+    end
+
+    entry = {
+      "@timestamp" => timestamp,
+      "tags" => [],
+      "type" => "watchdog",
+      "site" => @sitename,
+      "message" => msg
+    }.merge(row)
+
+    return LogStash::Event.new(entry)
+  end # def build_event
+
+end # class LogStash::Inputs::DrupalDblog

data/lib/logstash/inputs/drupal_dblog/jdbcconnection.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+require "java"
+require "rubygems"
+require "jdbc/mysql"
+
+java_import "com.mysql.jdbc.Driver"
+
+# A JDBC mysql connection class.
+# The interface is compatible with the mysql2 API.
+class LogStash::DrupalDblogJavaMysqlConnection
+
+  def initialize(host, username, password, database, port = nil)
+    port ||= 3306
+
+    address = "jdbc:mysql://#{host}:#{port}/#{database}"
+    @connection = java.sql.DriverManager.getConnection(address, username, password)
+  end # def initialize
+
+  def query(sql)
+    if sql =~ /select/i
+      return select(sql)
+    else
+      return update(sql)
+    end
+  end # def query
+
+  def select(sql)
+    stmt = @connection.createStatement
+    resultSet = stmt.executeQuery(sql)
+
+    meta = resultSet.getMetaData
+    column_count = meta.getColumnCount
+
+    rows = []
+
+    while resultSet.next
+      res = {}
+
+      (1..column_count).each do |i|
+        name = meta.getColumnName(i)
+        case meta.getColumnType(i)
+        when java.sql.Types::INTEGER
+          res[name] = resultSet.getInt(name)
+        else
+          res[name] = resultSet.getString(name)
+        end
+      end
+
+      rows << res
+    end
+
+    stmt.close
+    return rows
+  end # def select
+
+  def update(sql)
+    stmt = @connection.createStatement
+    stmt.execute_update(sql)
+    stmt.close
+  end # def update
+
+  def close
+    @connection.close
+  end # def close
+
+end # class LogStash::DrupalDblogJavaMysqlConnection

data/logstash-input-drupal_dblog.gemspec

@@ -0,0 +1,41 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-input-drupal_dblog'
+  s.version         = '0.1.1'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Retrieve watchdog log events from a Drupal installation with DBLog enabled"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elasticsearch"]
+  s.email           = 'info@elasticsearch.com'
+  s.homepage        = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Jar dependencies
+  s.requirements << "jar 'mysql:mysql-connector-java', '5.1.33'"
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency 'jar-dependencies'
+
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'php_serialize'
+
+  if RUBY_PLATFORM == 'java'
+    s.platform = RUBY_PLATFORM
+    s.add_runtime_dependency 'jdbc-mysql'
+  else
+    s.add_runtime_dependency 'mysql2'
+  end
+
+  s.add_development_dependency 'logstash-devutils'
+end
+

data/spec/inputs/drupal_dblog_spec.rb

@@ -0,0 +1 @@
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,145 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-drupal_dblog
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: java
+authors:
+- Elasticsearch
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jar-dependencies
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: php_serialize
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jdbc-mysql
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: info@elasticsearch.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- Rakefile
+- lib/logstash/inputs/drupal_dblog.rb
+- lib/logstash/inputs/drupal_dblog/jdbcconnection.rb
+- logstash-input-drupal_dblog.gemspec
+- spec/inputs/drupal_dblog_spec.rb
+homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar 'mysql:mysql-connector-java', '5.1.33'
+rubyforge_project:
+rubygems_version: 2.2.2
+signing_key:
+specification_version: 4
+summary: Retrieve watchdog log events from a Drupal installation with DBLog enabled
+test_files:
+- spec/inputs/drupal_dblog_spec.rb