logstash-integration-jdbc 5.0.0.alpha1

data/spec/filters/jdbc/read_write_database_spec.rb

@@ -0,0 +1,90 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash-integration-jdbc_jars"
+require "logstash/util/password"
+require "logstash/filters/jdbc/db_object"
+require "logstash/filters/jdbc/read_write_database"
+require "stud/temporary"
+
+module LogStash module Filters module Jdbc
+  describe ReadWriteDatabase do
+    let(:db) { Sequel.connect('mock://mydb') }
+    let(:connection_string_regex) { /jdbc:derby:memory:\w+;create=true/ }
+    let(:temp_import_path_plugin) { Stud::Temporary.pathname }
+    subject(:read_write_db) { described_class.create }
+
+    describe "basic operations" do
+      context "connecting  to a db" do
+        it "connects with defaults" do
+          expect(::Sequel::JDBC).to receive(:load_driver).once.with("org.apache.derby.jdbc.EmbeddedDriver")
+          # two calls to connect because ReadWriteDatabase does verify_connection and connect
+          expect(::Sequel).to receive(:connect).once.with(connection_string_regex, {:test => true}).and_return(db)
+          expect(::Sequel).to receive(:connect).once.with(connection_string_regex, {}).and_return(db)
+          expect(read_write_db.empty_record_set).to eq([])
+        end
+
+        it "connects with fully specified arguments" do
+          connection_str = "a connection string"
+          user = "a user"
+          password = Util::Password.new("secret")
+          expect(::Sequel::JDBC).to receive(:load_driver).once.with("a driver class")
+          expect(::Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password => password.value, :test => true}).and_return(db)
+          expect(::Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password => password.value}).and_return(db)
+          described_class.create(connection_str, "a driver class", nil, user, password)
+        end
+      end
+
+      describe "methods" do
+        let(:dataset) { double("Sequel::Dataset") }
+        let(:loaders) { [] }
+        let(:loader)  { double("Loader") }
+        let(:table_name) { "users" }
+        let(:random_table_name)  { "foobarbaz" }
+        let(:multi_insert_sql) { "INSERT VALUES (1, 2, 3)" }
+
+        before(:each) do
+          FileUtils.mkdir_p(temp_import_path_plugin)
+          allow(::Sequel::JDBC).to receive(:load_driver)
+          # allow(::Sequel).to receive(:connect).twice.and_return(db)
+          allow(::Sequel).to receive(:connect).twice.and_return(db)
+          allow(loader).to receive(:fetch).and_return([{:a => 1, :b => 2, :c => 3}])
+          allow(loader).to receive(:table).and_return(table_name)
+          allow(loader).to receive(:id).and_return("some_id")
+          allow(loader).to receive(:staging_directory).and_return(temp_import_path_plugin)
+          allow(described_class).to receive(:random_name).and_return(random_table_name)
+          loaders.push(loader)
+        end
+
+        it "the populate_all method fills a local_db from the dataset" do
+          expect(db).to receive(:[]).with(loader.table).once.and_return(dataset)
+          expect(dataset).to receive(:literal).and_return(1.to_s, 2.to_s, 3.to_s)
+          import_file_path = ::File.join(temp_import_path_plugin, loader.table)
+          expect(db).to receive(:execute_ddl).once.with("CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE (null,'#{loader.table.upcase}','#{import_file_path}',null,'''',null,1)")
+          read_write_db.populate_all(loaders)
+        end
+
+        it "the repopulate_all method fills a local_db from the dataset" do
+          expect(db).to receive(:[]).with(loader.table).once.and_return(dataset)
+          expect(dataset).to receive(:literal).and_return(1.to_s, 2.to_s, 3.to_s)
+          import_file_path = ::File.join(temp_import_path_plugin, loader.table)
+          expect(db).to receive(:execute_ddl).once.with("CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE (null,'#{loader.table.upcase}','#{import_file_path}',null,'''',null,1)")
+          read_write_db.repopulate_all(loaders)
+        end
+
+        it "the fetch method executes a parameterised SQL statement on the local db" do
+          statement = "select 1 from dual"
+          parameters = 42
+          expect(db).to receive(:[]).with(statement, parameters).once.and_return(dataset)
+          expect(dataset).to receive(:all).once.and_return([1,2,3])
+          read_write_db.fetch(statement, parameters)
+        end
+
+        it "lends the local db to a DbObject build instance method" do
+          db_object = DbObject.new("type" => "index", "name" => "servers_idx", "table" => "servers", "columns" => ["ip"])
+          expect(db_object).to receive(:build).once.with(db)
+          read_write_db.build_db_object(db_object)
+        end
+      end
+    end
+  end
+end end end

data/spec/filters/jdbc/repeating_load_runner_spec.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require_relative "../shared_helpers"
+require "logstash/filters/jdbc/repeating_load_runner"
+
+module LogStash module Filters module Jdbc
+  describe RepeatingLoadRunner  do
+    let(:local_db) { double("local_db") }
+    let(:loaders) { Object.new }
+    let(:local_db_objects) { [] }
+    subject(:runner) { described_class.new(local_db, loaders, local_db_objects) }
+
+    it_behaves_like "a single load runner"
+
+    context "when repeating" do
+      it "repopulates the local db" do
+        expect(local_db).to receive(:populate_all).once.with(loaders)
+        expect(local_db).to receive(:repopulate_all).once.with(loaders)
+        runner.initial_load
+        subject.call
+      end
+    end
+  end
+end end end

data/spec/filters/jdbc/single_load_runner_spec.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require_relative "../shared_helpers"
+
+require "logstash/filters/jdbc/single_load_runner"
+
+module LogStash module Filters module Jdbc
+  describe SingleLoadRunner  do
+    let(:local_db) { double("local_db") }
+    let(:loaders) { Object.new }
+    let(:local_db_objects) { [] }
+    subject(:runner) { described_class.new(local_db, loaders, local_db_objects) }
+
+    it_behaves_like "a single load runner"
+  end
+end end end

data/spec/filters/jdbc_static_file_local_spec.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+require_relative "env_helper"
+require_relative "remote_server_helper"
+
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc_static"
+require "sequel"
+require "sequel/adapters/jdbc"
+
+module LogStash module Filters
+  describe JdbcStatic, :skip => "temporary disable lookup_jdbc_* settings" do
+    let(:db1) { ::Sequel.connect("jdbc:derby:memory:testdb;create=true", :user=> nil, :password=> nil) }
+    let(:test_loader) { "SELECT * FROM reference_table" }
+    let(:test_records) { db1[test_loader].all }
+    let(:lookup_db) { "lookupdb" }
+
+    let(:local_db_objects) do
+      [
+        {"name" => "servers", "preserve_existing" => true, "index_columns" => ["ip"], "columns" => [["ip", "varchar(64)"], ["name", "varchar(64)"], ["location", "varchar(64)"]]},
+      ]
+    end
+
+    let(:settings) do
+      {
+        "loaders" => [
+          {
+            "id" =>"servers",
+            "query" => "select ip, name, location from reference_table",
+            "local_table" => "servers"
+          }
+        ],
+        "local_db_objects" => local_db_objects,
+        "local_lookups" => [
+          {
+            "query" => "select * from servers WHERE ip LIKE :ip",
+            "parameters" => {"ip" => "%%{[ip]}"},
+            "target" => "server"
+          }
+        ]
+      }
+    end
+
+    let(:client_jar_path) { ::File.join(BASE_DERBY_DIR, "derbyclient.jar") }
+
+    let(:mixin_settings) do
+      { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+        "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true",
+        "lookup_jdbc_driver_class" => "org.apache.derby.jdbc.ClientDriver",
+        "lookup_jdbc_driver_library" => nil,
+        "lookup_jdbc_connection_string" => "jdbc:derby://localhost:1527/#{lookup_db};create=true" }
+    end
+    let(:plugin) { JdbcStatic.new(mixin_settings.merge(settings)) }
+
+    after do
+      plugin.stop
+      ServerProcessHelpers.jdbc_static_stop_derby_server(lookup_db)
+    end
+
+    before do
+      ServerProcessHelpers.jdbc_static_start_derby_server
+      db1.drop_table(:reference_table) rescue nil
+      db1.create_table :reference_table do
+        String :ip
+        String :name
+        String :location
+      end
+      db1[:reference_table].insert(:ip => "10.1.1.1", :name => "ldn-server-1", :location => "LDN-2-3-4")
+      db1[:reference_table].insert(:ip => "10.2.1.1", :name => "nyc-server-1", :location => "NYC-5-2-8")
+      db1[:reference_table].insert(:ip => "10.3.1.1", :name => "mv-server-1", :location => "MV-9-6-4")
+
+      plugin.register
+    end
+
+    let(:event)      { ::LogStash::Event.new("message" => "some text", "ip" => ipaddr) }
+
+    let(:ipaddr) { ".3.1.1" }
+
+    it "enhances an event" do
+      plugin.filter(event)
+      expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
+    end
+  end
+end end

data/spec/filters/jdbc_static_spec.rb

@@ -0,0 +1,162 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc_static"
+require "sequel"
+require "sequel/adapters/jdbc"
+require "stud/temporary"
+require "timecop"
+
+# LogStash::Logging::Logger::configure_logging("WARN")
+
+module LogStash module Filters
+  describe JdbcStatic do
+
+    before(:all) do
+      @thread_abort = Thread.abort_on_exception
+      Thread.abort_on_exception = true
+    end
+
+    let(:jdbc_connection_string) { "jdbc:derby:memory:jdbc_static_testdb;create=true" }
+    let(:db1) { ::Sequel.connect(jdbc_connection_string, :user=> nil, :password=> nil) }
+    let(:loader_statement) { "SELECT ip, name, location FROM reference_table" }
+    let(:lookup_statement) { "SELECT * FROM servers WHERE ip LIKE :ip" }
+    let(:parameters_rhs) { "%%{[ip]}" }
+    let(:temp_import_path_plugin) { Stud::Temporary.pathname }
+    let(:temp_import_path_rspec) { Stud::Temporary.pathname }
+
+    let(:local_db_objects) do
+      [
+        {"name" => "servers", "index_columns" => ["ip"], "columns" => [["ip", "varchar(64)"], ["name", "varchar(64)"], ["location", "varchar(64)"]]},
+      ]
+    end
+
+    let(:settings) do
+      {
+        "loaders" => [
+          {
+            "id" =>"servers",
+            "query" => loader_statement,
+            "local_table" => "servers"
+          }
+        ],
+        "local_db_objects" => local_db_objects,
+        "local_lookups" => [
+          {
+            "query" => lookup_statement,
+            "parameters" => {"ip" => parameters_rhs},
+            "target" => "server"
+          }
+        ]
+      }
+    end
+
+    let(:mixin_settings) do
+      { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+        "jdbc_connection_string" => "#{jdbc_connection_string}",
+        "staging_directory" => temp_import_path_plugin
+      }
+    end
+
+    let(:add_records) do
+      lambda do |fd|
+        fd.puts "'10.1.1.1', 'ldn-server-1', 'LDN-2-3-4'"
+        fd.puts "'10.2.1.1', 'nyc-server-1', 'NYC-5-2-8'"
+        fd.puts "'10.3.1.1', 'mv-serv''r-1', 'MV-9-6-4'"
+      end
+    end
+
+    let(:plugin) { JdbcStatic.new(mixin_settings.merge(settings)) }
+
+    before do
+      db1.drop_table(:reference_table) rescue nil
+      db1.create_table(:reference_table) do
+        String :ip
+        String :name
+        String :location
+      end
+      ::File.open(temp_import_path_rspec, "w") do |fd|
+        add_records.call(fd)
+      end
+      import_cmd = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE (null,'REFERENCE_TABLE','#{temp_import_path_rspec}',null,'''',null,1)"
+      db1.execute_ddl(import_cmd)
+    end
+
+    let(:event)      { ::LogStash::Event.new("message" => "some text", "ip" => ipaddr) }
+
+    let(:ipaddr) { ".3.1.1" }
+
+    describe "non scheduled operation" do
+      after { plugin.close }
+
+      context "under normal conditions" do
+        it "enhances an event" do
+          plugin.register
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-serv'r-1", "location"=>"MV-9-6-4"}])
+        end
+      end
+
+      context "when the loader query returns no results" do
+        let(:loader_statement) { "SELECT ip, name, location FROM reference_table WHERE ip LIKE '20%'" }
+        it "add an empty array to the target field" do
+          plugin.register
+          plugin.filter(event)
+          expect(event.get("server")).to eq([])
+        end
+      end
+
+      context "when the loader query returns a large recordset, local db is filled in chunks" do
+        let(:add_records) do
+          lambda do |fd|
+            256.times do |octet3|
+              256.times do |octet4|
+                fd.puts "'10.4.#{octet3}.#{octet4}', 'server-#{octet3}-#{octet4}', 'MV-10-#{octet3}-#{octet4}'"
+              end
+            end
+          end
+        end
+        let(:ipaddr) { "10.4.254.255" }
+        let(:lookup_statement) { "SELECT * FROM servers WHERE ip = :ip" }
+        let(:parameters_rhs) { "ip" }
+        it "enhances an event" do
+          plugin.register
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"ip"=>ipaddr, "name"=>"server-254-255", "location"=>"MV-10-254-255"}])
+        end
+      end
+
+      context "under normal conditions when index_columns is not specified" do
+        let(:local_db_objects) do
+          [
+            {"name" => "servers", "columns" => [["ip", "varchar(64)"], ["name", "varchar(64)"], ["location", "varchar(64)"]]},
+          ]
+        end
+        it "enhances an event" do
+          plugin.register
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-serv'r-1", "location"=>"MV-9-6-4"}])
+        end
+      end
+    end
+
+    describe "scheduled operation" do
+      context "given a loader_schedule" do
+        it "should properly schedule" do
+          settings["loader_schedule"] = "*/10 * * * * * UTC"
+          Timecop.travel(Time.now.utc - 3600)
+          Timecop.scale(60)
+          static_filter = JdbcStatic.new(mixin_settings.merge(settings))
+          runner = Thread.new(static_filter) do |filter|
+            filter.register
+          end
+          sleep 3
+          static_filter.filter(event)
+          expect(static_filter.loader_runner.reload_count).to be > 1
+          static_filter.close
+          Timecop.return
+          expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-serv'r-1", "location"=>"MV-9-6-4"}])
+        end
+      end
+    end
+  end
+end end