logstash-filter-jdbc_static 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f5052dc6f6f29b7919706257f1f72df70243ef4c8ab3e8c9995fc5fff8603156
+  data.tar.gz: 4ea4e75413cecd1bf5dbb44ac954810ab7369001448f9d23eaefe29631b0fd00
+SHA512:
+  metadata.gz: '086d5fd949f32b807b78562e36ecefd1952464775e81be53bd81cd4a11f612df65043defdc6368107368f21d196ce19de307cd4a2bd05ef6110ab46c131a857c'
+  data.tar.gz: 0b27db6ac28a7a9b831fe3a5e7369b3a4dbf69ca220fd5e33c968b372d7d7e69a6054d7668919ad06070ec02b9b902cdba3d09da6f8c58eb5eddd6c8e1f7ff51

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 1.0.0
+ - Initial commit

data/CONTRIBUTORS

@@ -0,0 +1,22 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Christian Paredes (cparedes)
+* Colin Surprenant (colinsurprenant)
+* Jay Luker (lbjay)
+* John E. Vincent (lusis)
+* Jordan Sissel (jordansissel)
+* João Duarte (jsvd)
+* Laust Rud Jacobsen (rud)
+* Nikolay Bryskin (nikicat)
+* Pete Fritchman (fetep)
+* Philippe Weber (wiibaa)
+* Pier-Hugues Pellerin (ph)
+* Suyog Rao (suyograo)
+* Guy Boertje (guyboertje)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2018 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,94 @@
+# logstash-filter-jdbc_static
+Lookup plugin which overwrites a loader query's result set into a local in-memory db and then lookup queries add records to a target field using properties from the event. 
+
+# Logstash Plugin
+
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-filter-jdbc_static.svg)](https://travis-ci.org/logstash-plugins/logstash-filter-jdbc_static)
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Need Help?
+
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
+
+## Developing
+
+### 1. Plugin Development and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run tests
+
+```sh
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone
+
+- Edit Logstash `Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+```
+- Install plugin
+```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
+```
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'filter {awesome {}}'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- Build your plugin gem
+```sh
+gem build logstash-filter-awesome.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash-filter-jdbc_static_jars.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+require 'logstash/environment'
+
+root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")

data/lib/logstash/filters/jdbc/basic_database.rb

@@ -0,0 +1,117 @@
+require "sequel"
+require "sequel/adapters/jdbc"
+require "java"
+require "logstash/util/loggable"
+
+module LogStash module Filters module Jdbc
+  EMBEDDED_DERBY_DRIVER_CLASS = "org.apache.derby.jdbc.EmbeddedDriver".freeze
+  MEMORY_DERBY_LOCAL_CONNECTION_STRING = "jdbc:derby:memory:____;create=true".freeze
+  CONNECTION_ERRORS = [::Sequel::DatabaseConnectionError, ::Sequel::DatabaseDisconnectError, ::Sequel::PoolTimeout]
+
+  class LookupJdbcException < ::StandardError; end
+  class LoaderJdbcException < ::StandardError; end
+  class ConnectionJdbcException < ::StandardError; end
+
+  class BasicDatabase
+    include LogStash::Util::Loggable
+
+    def self.wrap_error(new_error_class, exception, message = nil)
+      error = new_error_class.new(message || exception.message)
+      error.set_backtrace(exception.backtrace)
+      error
+    end
+
+    def self.create(
+      connection_string = MEMORY_DERBY_LOCAL_CONNECTION_STRING,
+      driver_class = EMBEDDED_DERBY_DRIVER_CLASS,
+      driver_library = nil,
+      user = nil,
+      password = nil)
+      instance = new
+      instance.post_create(connection_string, driver_class, driver_library, user, password)
+      instance
+    end
+
+    def self.random_name(length = 10)
+      SecureRandom.hex(length)
+    end
+
+    attr_reader :unique_db_name
+
+    def initialize()
+      @options_hash = {}
+      post_initialize
+    end
+
+    def connect(err_message)
+      begin
+        @db = ::Sequel.connect(@connection_string, @options_hash)
+      rescue *CONNECTION_ERRORS => err
+        # we do not raise an error when there is a connection error, we hope that the connection works next time
+        logger.error(err_message, :exception => err.message, :backtrace => err.backtrace.take(8))
+      end
+    end
+
+    def disconnect(err_message)
+      return if @db.nil?
+      begin
+        @db.disconnect
+      rescue *CONNECTION_ERRORS => err
+        # we do not raise an error when there is a connection error, we hope that the connection works next time
+        logger.error(err_message, :exception => err.message, :backtrace => err.backtrace.take(8))
+      ensure
+        @db = nil
+      end
+    end
+
+    def connected?
+      !@db.nil?
+    end
+
+    def empty_record_set
+      []
+    end
+
+    def post_create(connection_string, driver_class, driver_library, user, password)
+      raise NotImplementedError.new("#{self.class.name} is abstract, you must subclass it and implement #post_create()")
+    end
+
+    private
+
+    def verify_connection(connection_string, driver_class, driver_library, user, password)
+      begin
+        require driver_library if driver_library
+      rescue LoadError => e
+        msg = "The driver library cannot be loaded. The system error was: '#{e.message}'."
+        raise wrap_error(ConnectionJdbcException, e, msg)
+      end
+      begin
+        db = nil
+        ::Sequel::JDBC.load_driver(driver_class)
+        @connection_string = connection_string
+        if user
+          @options_hash[:user] = user
+        end
+        if password
+          @options_hash[:password] = password.value
+        end
+        # test the connection as early as possible
+        db = ::Sequel.connect(@connection_string, {:test => true}.merge(@options_hash))
+      rescue ::Sequel::AdapterNotFound => anf
+        raise wrap_error(ConnectionJdbcException, anf)
+      rescue ::Sequel::DatabaseConnectionError => dce
+        raise wrap_error(ConnectionJdbcException, dce)
+      ensure
+        db.disconnect unless db.nil?
+      end
+    end
+
+    def post_initialize()
+      @unique_db_name = SecureRandom.hex(12)
+    end
+
+    def wrap_error(new_error_class, exception, message = nil)
+      self.class.wrap_error(new_error_class, exception, message)
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/column.rb

@@ -0,0 +1,38 @@
+require_relative "validatable"
+
+module LogStash module Filters module Jdbc
+  class Column < Validatable
+    attr_reader :name, :datatype
+
+    private
+
+    def post_initialize
+      if valid?
+        @name = @name.to_sym
+        @datatype = @datatype.to_sym
+      end
+    end
+
+    def to_array
+      [@name.to_s, @datatype.to_s]
+    end
+
+    def parse_options
+      unless @options.is_a?(Array)
+        @option_errors << "The column options must be an array"
+      end
+
+      @name, @datatype = @options
+
+      unless @name && @name.is_a?(String)
+        @option_errors << "The first column option is the name and must be a string"
+      end
+
+      unless @datatype && @datatype.is_a?(String)
+        @option_errors << "The second column option is the datatype and must be a string"
+      end
+
+      @valid = @option_errors.empty?
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/db_object.rb

@@ -0,0 +1,103 @@
+require_relative "validatable"
+require_relative "column"
+
+module LogStash module Filters module Jdbc
+
+  TEMP_TABLE_PREFIX = "temp_".freeze
+
+  class DbObject < Validatable
+    #   {name => "servers", index_columns => ["ip"], columns => [["ip", "text"], ["name", "text"], ["location", "text"]]},
+
+    attr_reader :name, :columns, :preserve_existing, :index_columns
+
+    def build(db)
+      return unless valid?
+      schema_gen = db.create_table_generator()
+      @columns.each {|col| schema_gen.column(col.name, col.datatype)}
+      schema_gen.index(@index_columns)
+      options = {:generator => schema_gen}
+      if @preserve_existing
+        db.create_table?(@name, options)
+      else
+        db.create_table(@name, options)
+      end
+    end
+
+    def <=>(other)
+      @name <=> other.name
+    end
+
+    def as_temp_table_opts
+      {"name" => "#{TEMP_TABLE_PREFIX}#{@name}", "preserve_existing" => @preserve_existing, "index_columns" => @index_columns.map(&:to_s), "columns" => @columns.map(&:to_array)}
+    end
+
+    def to_s
+      inspect
+    end
+
+    def inspect
+      "<LogStash::Filters::Jdbc::DbObject name: #{@name}, columns: #{@columns.inspect}>"
+    end
+
+    private
+
+    def post_initialize
+      if valid?
+        @name = @name.to_sym
+      end
+    end
+
+    def parse_options
+      if !@options.is_a?(Hash)
+        @option_errors << "DbObject options must be a Hash"
+        @valid = false
+        return
+      end
+
+      @name = @options["name"]
+      unless @name && @name.is_a?(String)
+        @option_errors << "DbObject options must include a 'name' string"
+        @name = "unnamed"
+      end
+
+      @preserve_existing = @options.fetch("preserve_existing", false)
+      @preserve_existing = true if @preserve_existing == "true"
+
+      @columns_options = @options["columns"]
+      @columns = []
+      temp_column_names = []
+      if @columns_options && @columns_options.is_a?(Array)
+        sizes = @columns_options.map{|option| option.size}.uniq
+        if sizes == [2]
+          @columns_options.each do |option|
+            column = Column.new(option)
+            if column.valid?
+              @columns << column
+              temp_column_names << column.name
+            else
+              @option_errors << column.formatted_errors
+            end
+          end
+        else
+          @option_errors << "The columns array for '#{@name}' is not uniform, it should contain arrays of two strings only"
+        end
+      else
+        @option_errors << "DbObject options for '#{@name}' must include a 'columns' array"
+      end
+
+      @index_column_options = @options["index_columns"]
+      @index_columns = []
+      if @index_column_options && @index_column_options.is_a?(Array)
+        @index_column_options.each do |option|
+          if option.is_a?(String) && temp_column_names.member?(option.to_sym)
+            @index_columns << option.to_sym
+          else
+            @option_errors << "The index_columns element: '#{option}' must be a column defined in the columns array"
+          end
+        end
+      end
+
+      @valid = @option_errors.empty?
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/loader.rb

@@ -0,0 +1,114 @@
+# encoding: utf-8
+require_relative "validatable"
+require_relative "db_object"
+require_relative "read_only_database"
+require "logstash/util/loggable"
+
+module LogStash module Filters module Jdbc
+  class Loader < Validatable
+    include LogStash::Util::Loggable
+
+    CONNECTION_ERROR_MSG = "Remote DB connection error when executing loader Jdbc query"
+
+    attr_reader :id, :table, :temp_table, :query, :max_rows
+    attr_reader :connection_string, :driver_library, :driver_class
+    attr_reader :user, :password
+
+    def build_remote_db
+      @remote = ReadOnlyDatabase.create(connection_string, driver_class, driver_library, user, password)
+    end
+
+    def fetch
+      @remote.connect(CONNECTION_ERROR_MSG)
+      row_count = @remote.count(query)
+      if row_count.zero?
+        logger.warn? && logger.warn("Query returned no results", :lookup_id => @id, :query => query)
+        return @remote.empty_record_set
+      end
+      if row_count > max_rows
+        logger.warn? && logger.warn("Query returned more than max_rows results", :lookup_id => @id, :query => query, :count => row_count, :max_rows => max_rows)
+        return @remote.empty_record_set
+      end
+      @remote.query(query)
+    ensure
+      @remote.disconnect(CONNECTION_ERROR_MSG)
+    end
+
+    def close
+      @remote.disconnect(CONNECTION_ERROR_MSG)
+    end
+
+    private
+
+    def pre_initialize(options)
+      @table = options["local_table"]
+    end
+
+    def post_initialize
+      if valid?
+        @temp_table = "#{TEMP_TABLE_PREFIX}#{@table}".to_sym
+        @table = @table.to_sym
+      end
+    end
+
+    def parse_options
+      unless @table && @table.is_a?(String)
+        @option_errors << "The options must include a 'local_table' string"
+      end
+
+      @id = @options.fetch("id", @table)
+
+      @query = @options["query"]
+      unless @query && @query.is_a?(String)
+        @option_errors << "The options for '#{@table}' must include a 'query' string"
+      end
+
+      @max_rows = @options["max_rows"]
+      if @max_rows
+        if !@max_rows.respond_to?(:to_i)
+          @option_errors << "The 'max_rows' option for '#{@table}' must be an integer"
+        else
+          @max_rows = @max_rows.to_i
+        end
+      else
+        @max_rows = 1_000_000
+      end
+
+      @driver_library = @options["jdbc_driver_library"]
+      if @driver_library
+        if !@driver_library.is_a?(String)
+          @option_errors << "The 'jdbc_driver_library' option for '#{@table}' must be a string"
+        end
+        if !::File.exists?(@driver_library)
+          @option_errors << "The 'jdbc_driver_library' option for '#{@table}' must be a file that can be opened: #{driver_library}"
+        end
+      end
+
+      @driver_class = @options["jdbc_driver_class"]
+      if @driver_class && !@driver_class.is_a?(String)
+        @option_errors << "The 'jdbc_driver_class' option for '#{@table}' must be a string"
+      end
+
+      @connection_string = @options["jdbc_connection_string"]
+      if @connection_string && !@connection_string.is_a?(String)
+        @option_errors << "The 'jdbc_connection_string' option for '#{@table}' must be a string"
+      end
+
+      @user = @options["jdbc_user"]
+      if @user && !@user.is_a?(String)
+        @option_errors << "The 'jdbc_user' option for '#{@table}' must be a string"
+      end
+
+      @password = @options["jdbc_password"]
+      case @password
+      when String
+        @password = LogStash::Util::Password.new(@password)
+      when LogStash::Util::Password, nil
+        # this is OK
+      else
+        @option_errors << "The 'jdbc_password' option for '#{@table}' must be a string"
+      end
+      @valid = @option_errors.empty?
+    end
+  end
+end end end