logstash-filter-jdbc_static 1.0.0

data/lib/logstash/filters/jdbc/repeating_load_runner.rb

@@ -0,0 +1,11 @@
+require_relative "single_load_runner"
+
+module LogStash module Filters module Jdbc
+  class RepeatingLoadRunner < SingleLoadRunner
+   # info - attr_reader :local, :loaders, :preloaders
+
+    def repeated_load
+      local.repopulate_all(loaders)
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/single_load_runner.rb

@@ -0,0 +1,43 @@
+require_relative 'db_object'
+
+module LogStash module Filters module Jdbc
+  class SingleLoadRunner
+
+    attr_reader :local, :loaders, :preloaders
+
+    def initialize(local, loaders, preloaders)
+      @local = local
+      @loaders = loaders
+      @preloaders = []
+      preloaders.map do |pre|
+        dbo = DbObject.new(pre)
+        @preloaders << dbo
+        hash = dbo.as_temp_table_opts
+        _dbo = DbObject.new(hash)
+        @preloaders << _dbo if _dbo.valid?
+      end
+      @preloaders.sort!
+    end
+
+    def initial_load
+      do_preload
+      local.populate_all(loaders)
+    end
+
+    def repeated_load
+    end
+
+    def call
+      repeated_load
+    end
+
+    private
+
+    def do_preload
+      preloaders.each do |db_object|
+        local.build_db_object(db_object)
+      end
+    end
+  end
+
+end end end

data/lib/logstash/filters/jdbc/validatable.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+
+module LogStash module Filters module Jdbc
+  class Validatable
+    def self.find_validation_errors(array_of_options)
+      if !array_of_options.is_a?(Array)
+        return "The options must be an Array"
+      end
+      errors = []
+      array_of_options.each do |options|
+        instance = new(options)
+        unless instance.valid?
+          errors << instance.formatted_errors
+        end
+      end
+      return nil if errors.empty?
+      errors.join("; ")
+    end
+
+    def initialize(options)
+      pre_initialize(options)
+      @options = options
+      @valid = false
+      @option_errors = []
+      parse_options
+      post_initialize
+    end
+
+    def valid?
+      @valid
+    end
+
+    def formatted_errors
+      @option_errors.join(", ")
+    end
+
+    private
+
+    def pre_initialize(options)
+    end
+
+    def post_initialize
+    end
+
+    def parse_options
+      raise "Subclass must implement 'parse_options'"
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc_static.rb

@@ -0,0 +1,216 @@
+# encoding: utf-8
+require "logstash-filter-jdbc_static_jars"
+require "logstash/filters/base"
+require "logstash/namespace"
+require_relative "jdbc/loader"
+require_relative "jdbc/loader_schedule"
+require_relative "jdbc/repeating_load_runner"
+require_relative "jdbc/lookup_processor"
+
+# This filter can do multiple enhancements to an event in one pass.
+# Define multiple loader sources and multiple lookup targets.
+# Currently only one remote database connection is supported.
+# [source,ruby]
+
+#
+module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
+  config_name "jdbc_static"
+
+  # Define the loaders, an Array of Hashes, to fetch remote data and create local tables.
+  # the fetched data will be inserted into the local tables. Make sure that the
+  # local table name, columns and datatypes correspond to the shape of the remote data
+  # being fetched. The default for max_rows is 1 million rows. You may provide an `id`
+  # For example:
+  # loaders => [
+  #   {
+  #     id => "country_details"
+  #     query => "select code, name from WORLD.COUNTRY"
+  #     max_rows => 2000
+  #     local_table => "country"
+  #   },
+  #   {
+  #     id => "servers_load"
+  #     query => "select id, ip, name, location from INTERNAL.SERVERS"
+  #     local_table => "servers"
+  #   }
+  # ]
+  # This is optional. You can provide a pre-populated local database server then no initial loaders are needed.
+  config :loaders, :required => false, :default => [], :validate => [LogStash::Filters::Jdbc::Loader]
+
+  # Define an array of Database Objects to create when the plugin first starts.
+  # These will usually be the definitions to setup the local in-memory tables.
+  # For example:
+  # local_db_objects => [
+  #   {name => "servers", preserve_existing => true, index_columns => ["ip"], columns => [["id", "INTEGER"], ["ip", "varchar(64)"], ["name", "varchar(64)"], ["location", "varchar(64)"]]},
+  # ]
+  # NOTE: Important! use `preserve_existing => true` to keep a table created and filled in a previous Logstash session. It will default to false and is unneeded if the database is not persistent.
+  # NOTE: Important! Tables created here must have the same names as those used in the `loaders` and
+  # `local_lookups` configuration options
+  config :local_db_objects, :required => false, :default => [], :validate => [LogStash::Filters::Jdbc::DbObject]
+
+  # Define the list (Array) of enhancement local_lookups to be applied to an event
+  # Each entry is a hash of the query string, the target field and value and a
+  # parameter hash. Target is overwritten if existing. Target is optional,
+  # if omitted the lookup results will be written to the root of the event like this:
+  # event.set(<column name (or alias)>, <column value>)
+  # Use parameters to have this plugin put values from the event into the query.
+  # The parameter maps the symbol used in the query string to the field name in the event.
+  # NOTE: when using a query string that includes the LIKE keyword make sure that
+  # you provide a Logstash Event sprintf pattern with added wildcards.
+  # For example:
+  # local_lookups => [
+  #   {
+  #     "query" => "select * from country WHERE code = :code",
+  #     "parameters" => {"code" => "country_code"}
+  #     "target" => "country_details"
+  #   },
+  #   {
+  #     "query" => "select ip, name from servers WHERE ip LIKE :ip",
+  #     "parameters" => {"ip" => "%{[response][ip]}%"}
+  #     "target" => "servers"
+  #   }
+  # ]
+  config :local_lookups, :required => true, :validate => [LogStash::Filters::Jdbc::LookupProcessor]
+
+  # Schedule of when to periodically run loaders, in Cron format
+  # for example: "* * * * *" (execute query every minute, on the minute)
+  #
+  # There is no schedule by default. If no schedule is given, then the loaders are run
+  # exactly once.
+  config :loader_schedule, :validate => [LogStash::Filters::Jdbc::LoaderSchedule]
+
+  # Append values to the `tags` field if sql error occured
+  # Alternatively, individual `tag_on_failure` arrays can be added to each lookup hash
+  config :tag_on_failure, :validate => :array, :default => ["_jdbcstaticfailure"]
+
+  # Append values to the `tags` field if no record was found and default values were used
+  config :tag_on_default_use, :validate => :array, :default => ["_jdbcstaticdefaultsused"]
+
+  # Remote Load DB Jdbc driver library path to third party driver library.
+  config :jdbc_driver_library, :validate => :path
+
+  # Remote Load DB Jdbc driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver"
+  config :jdbc_driver_class, :validate => :string, :required => true
+
+  # Remote Load DB Jdbc connection string
+  config :jdbc_connection_string, :validate => :string, :required => true
+
+  # Remote Load DB Jdbc user
+  config :jdbc_user, :validate => :string
+
+  # Remote Load DB Jdbc password
+  config :jdbc_password, :validate => :password
+
+  # NOTE: For the initial release, we are not allowing the user to specify their own local lookup JDBC DB settings.
+  # In the near future we have to consider identical config running in multiple pipelines stomping over each other
+  # when the database names are common across configs because there is only one Derby server in memory per JVM.
+
+  # Local Lookup DB Jdbc driver class to load, for example "org.apache.derby.jdbc.ClientDriver"
+  # config :lookup_jdbc_driver_class, :validate => :string, :required => false
+
+  # Local Lookup DB Jdbc driver library path to third party driver library.
+  # config :lookup_jdbc_driver_library, :validate => :path, :required => false
+
+  # Local Lookup DB Jdbc connection string
+  # config :lookup_jdbc_connection_string, :validate => :string, :required => false
+
+  class << self
+    alias_method :old_validate_value, :validate_value
+
+    def validate_value(value, validator)
+      if validator.is_a?(Array) && validator.first.respond_to?(:find_validation_errors)
+        validation_errors = validator.first.find_validation_errors(value)
+        unless validation_errors.nil?
+          return false, validation_errors
+        end
+      else
+        return old_validate_value(value, validator)
+      end
+      [true, value]
+    end
+  end
+
+  public
+
+  def register
+    prepare_data_dir
+    prepare_runner
+    @loader_runner.initial_load
+  end
+
+  def filter(event)
+    enhancement_states = @processor.enhance(event)
+    filter_matched(event) if enhancement_states.all?
+  end
+
+  def stop
+    @scheduler.stop if @scheduler
+    @parsed_loaders.each(&:close)
+    @processor.close
+  end
+
+  private
+
+  def prepare_data_dir
+    # later, when local persistent databases are allowed set this property to LS_HOME/data/jdbc-static/
+    # must take multi-pipelines into account and more than one config using the same jdbc-static settings
+    java.lang.System.setProperty("derby.system.home", ENV["HOME"])
+  end
+
+  def prepare_runner
+    @parsed_loaders = @loaders.map do |options|
+      add_plugin_configs(options)
+      loader = Jdbc::Loader.new(options)
+      loader.build_remote_db
+      loader
+    end
+    runner_args = [@parsed_loaders, @local_db_objects]
+    @processor = Jdbc::LookupProcessor.new(@local_lookups, global_lookup_options)
+    runner_args.unshift(@processor.local)
+    if @loader_schedule
+      require "rufus/scheduler"
+      args = []
+      @loader_runner = Jdbc::RepeatingLoadRunner.new(*runner_args)
+
+      cronline = Jdbc::LoaderSchedule.new(@loader_schedule)
+      rufus_args = {:max_work_threads => 1, :frequency => cronline.schedule_frequency}
+
+      @scheduler = Rufus::Scheduler.new(rufus_args)
+      @scheduler.cron(cronline.loader_schedule, @loader_runner)
+      @scheduler.join
+    else
+      @loader_runner = Jdbc::SingleLoadRunner.new(*runner_args)
+    end
+  end
+
+  def global_lookup_options(options = Hash.new)
+    if @tag_on_failure && !@tag_on_failure.empty? && !options.key?("tag_on_failure")
+      options["tag_on_failure"] = @tag_on_failure
+    end
+    if @tag_on_default_use && !@tag_on_default_use.empty? && !options.key?("tag_on_default_use")
+      options["tag_on_default_use"] = @tag_on_default_use
+    end
+    options["lookup_jdbc_driver_class"] = @lookup_jdbc_driver_class
+    options["lookup_jdbc_driver_library"] = @lookup_jdbc_driver_library
+    options["lookup_jdbc_connection_string"] = @lookup_jdbc_connection_string
+    options
+  end
+
+  def add_plugin_configs(options)
+    if @jdbc_driver_library
+      options["jdbc_driver_library"] = @jdbc_driver_library
+    end
+    if @jdbc_driver_class
+      options["jdbc_driver_class"] = @jdbc_driver_class
+    end
+    if @jdbc_connection_string
+      options["jdbc_connection_string"] = @jdbc_connection_string
+    end
+    if @jdbc_user
+      options["jdbc_user"] = @jdbc_user
+    end
+    if @jdbc_password
+      options["jdbc_password"] = @jdbc_password
+    end
+  end
+end end end

data/logstash-filter-jdbc_static.gemspec

@@ -0,0 +1,38 @@
+Gem::Specification.new do |s|
+  s.name            = 'logstash-filter-jdbc_static'
+  s.version         = '1.0.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "This filter executes a SQL query to fetch a SQL query result, store it locally then use a second SQL query to update an event."
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elastic"]
+  s.email           = 'info@elastic.co'
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  # to fool jar_dependencies to mimic our other plugin gradle vendor script behaviour
+  # the rake vendor task removes jars dir and jars downloaded to it
+  s.require_paths   = ["lib", "jars"]
+
+  # Files
+  s.files = Dir['lib/**/*','vendor/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  derby_version = "10.14.1.0"
+  s.requirements << "jar 'org.apache.derby:derby', '#{derby_version}'"
+  s.requirements << "jar 'org.apache.derby:derbyclient', '#{derby_version}'"
+  # we may need 'org.apache.derby:derbynet' in the future, marking this here
+
+  s.add_development_dependency 'jar-dependencies', '~> 0.3'
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'sequel'
+  s.add_runtime_dependency 'tzinfo'
+  s.add_runtime_dependency 'tzinfo-data'
+  s.add_runtime_dependency 'rufus-scheduler'
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency "childprocess"
+end

data/spec/filters/env_helper.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+
+# use the rspec --require command line option to have this file evaluated before rspec runs
+# it i
+
+GEM_BASE_DIR = ::File.expand_path("../../..", __FILE__)
+BASE_DERBY_DIR = ::File.join(GEM_BASE_DIR, "spec", "helpers")
+ENV["HOME"] = GEM_BASE_DIR
+ENV["TEST_DEBUG"] = "true"
+java.lang.System.setProperty("ls.logs", "console")

data/spec/filters/jdbc/column_spec.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc/column"
+
+describe LogStash::Filters::Jdbc::Column  do
+  let(:invalid_messages) do
+    [
+      "The column options must be an array",
+      "The first column option is the name and must be a string",
+      "The second column option is the datatype and must be a string"
+    ]
+  end
+
+  context "various invalid non-array arguments" do
+    it "a nil does not validate" do
+      instance = described_class.new(nil)
+      expect(instance.valid?).to be_falsey
+      expect(instance.formatted_errors).to eq(invalid_messages.join(", "))
+    end
+
+    it "a string does not validate" do
+      instance = described_class.new("foo")
+      expect(instance.valid?).to be_falsey
+      expect(instance.formatted_errors).to eq(invalid_messages.values_at(0,2).join(", "))
+    end
+
+    it "a number does not validate" do
+      instance = described_class.new(42)
+      expect(instance.valid?).to be_falsey
+      expect(instance.formatted_errors).to eq(invalid_messages.join(", "))
+    end
+  end
+
+  context "various invalid array arguments" do
+    it "a single string element does not validate" do
+      instance = described_class.new(["foo"])
+      expect(instance.valid?).to be_falsey
+      expect(instance.formatted_errors).to eq(invalid_messages.last)
+    end
+    [ [], [1, 2] ].each do |arg|
+      it "do not validate" do
+        instance = described_class.new(arg)
+        expect(instance.valid?).to be_falsey
+        expect(instance.formatted_errors).to eq(invalid_messages.values_at(1,2).join(", "))
+      end
+    end
+    [ ["foo", 3], ["foo", nil] ].each do |arg|
+      it "do not validate" do
+        instance = described_class.new(arg)
+        expect(instance.valid?).to be_falsey
+        expect(instance.formatted_errors).to eq(invalid_messages.last)
+      end
+    end
+    [ [3, "foo"], [nil, "foo"] ].each do |arg|
+      it "do not validate" do
+        instance = described_class.new(arg)
+        expect(instance.valid?).to be_falsey
+        expect(instance.formatted_errors).to eq(invalid_messages[1])
+      end
+    end
+  end
+
+  context "a valid array argument" do
+    it "does validate" do
+      instance = described_class.new(["foo", "varchar2"])
+      expect(instance.valid?).to be_truthy
+      expect(instance.formatted_errors).to eq("")
+    end
+  end
+end

data/spec/filters/jdbc/db_object_spec.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc/db_object"
+
+describe LogStash::Filters::Jdbc::DbObject  do
+  context "various invalid non-hash arguments" do
+    it "a nil does not validate" do
+      instance = described_class.new(nil)
+      expect(instance.valid?).to be_falsey
+      expect(instance.formatted_errors).to eq("DbObject options must be a Hash")
+    end
+
+    it "a string does not validate" do
+      instance = described_class.new("foo")
+      expect(instance.valid?).to be_falsey
+      expect(instance.formatted_errors).to eq("DbObject options must be a Hash")
+    end
+
+    it "a number does not validate" do
+      instance = described_class.new(42)
+      expect(instance.valid?).to be_falsey
+      expect(instance.formatted_errors).to eq("DbObject options must be a Hash")
+    end
+  end
+
+  context "various invalid hash arguments" do
+    let(:error_messages) do
+      [
+        "DbObject options must include a 'name' string",
+        "DbObject options for 'foo' must include a 'columns' array"
+      ]
+    end
+    "DbObject options must include a 'name' string, DbObject options for 'unnamed' must include a 'columns' array"
+    it "an empty hash does not validate" do
+      instance = described_class.new({})
+      expect(instance.valid?).to be_falsey
+      expect(instance.formatted_errors).to eq(error_messages.values_at(0,1).join(", ").gsub('foo', 'unnamed'))
+    end
+
+    it "a name key value only" do
+      instance = described_class.new({"name" => "foo"})
+      expect(instance.valid?).to be_falsey
+      expect(instance.formatted_errors).to eq(error_messages[1])
+    end
+
+    it "a name and bad columns" do
+      instance = described_class.new({"name" => "foo", "columns" => 42})
+      expect(instance.valid?).to be_falsey
+      expect(instance.formatted_errors).to eq(error_messages[1])
+    end
+
+    it "a name and bad columns - empty array" do
+      instance = described_class.new({"name" => "foo", "columns" => []})
+      expect(instance.valid?).to be_falsey
+      msg = "The columns array for 'foo' is not uniform, it should contain arrays of two strings only"
+      expect(instance.formatted_errors).to eq(msg)
+    end
+
+    it "a name and bad columns - irregular arrays" do
+      instance = described_class.new({"name" => "foo", "columns" => [["ip", "text"], ["name"], ["a", "b", "c"]]})
+      expect(instance.valid?).to be_falsey
+      msg = "The columns array for 'foo' is not uniform, it should contain arrays of two strings only"
+      expect(instance.formatted_errors).to eq(msg)
+    end
+
+    it "a name, good columns and bad index_column" do
+      instance = described_class.new({"name" => "foo_index", "index_columns" => ["bar"], "columns" => [["ip", "text"], ["name", "text"]]})
+      expect(instance.valid?).to be_falsey
+      msg = "The index_columns element: 'bar' must be a column defined in the columns array"
+      expect(instance.formatted_errors).to eq(msg)
+    end
+  end
+
+  context "a valid hash argument" do
+    it "does validate" do
+      instance = described_class.new({"name" => "foo", "index_columns" => ["ip"], "columns" => [["ip", "text"], ["name", "text"]]})
+      expect(instance.formatted_errors).to eq("")
+      expect(instance.valid?).to be_truthy
+    end
+  end
+end