logstash-filter-jdbc_static 1.0.0

data/spec/filters/jdbc/loader_spec.rb

@@ -0,0 +1,76 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc/loader"
+
+describe LogStash::Filters::Jdbc::Loader do
+  let(:local_table) { "servers" }
+  let(:options) do
+    {
+      "jdbc_driver_class" => "org.postgresql.Driver",
+      "jdbc_connection_string" => "jdbc:postgres://user:password@remotedb/infra",
+      "query" => "select ip, name, location from INTERNAL.SERVERS",
+      "jdbc_user" => "bob",
+      "jdbc_password" => "letmein",
+      "max_rows" => 2000,
+      "local_table" => local_table
+    }
+  end
+  subject { described_class.new(options) }
+
+  context "when correct options are given" do
+    it "validation succeeds" do
+      expect(subject.valid?).to be_truthy
+      expect(subject.formatted_errors).to eq("")
+    end
+  end
+
+  context "when incorrect options are given" do
+    let(:options) do
+      {
+        "jdbc_driver_class" => 42,
+        "jdbc_connection_string" => 42,
+        "max_rows" => ["2000"]
+      }
+    end
+
+    it "validation fails" do
+      expect(subject.valid?).to be_falsey
+      expect(subject.formatted_errors).to eq("The options must include a 'local_table' string, The options for '' must include a 'query' string, The 'max_rows' option for '' must be an integer, The 'jdbc_driver_class' option for '' must be a string, The 'jdbc_connection_string' option for '' must be a string")
+    end
+  end
+
+  context "attr_reader methods" do
+    it "#table" do
+      expect(subject.table).to eq(:servers)
+    end
+
+    it "#query" do
+      expect(subject.query).to eq("select ip, name, location from INTERNAL.SERVERS")
+    end
+
+    it "#max_rows" do
+      expect(subject.max_rows).to eq(2000)
+    end
+
+    it "#connection_string" do
+      expect(subject.connection_string).to eq("jdbc:postgres://user:password@remotedb/infra")
+    end
+
+    it "#driver_class" do
+      expect(subject.driver_class).to eq("org.postgresql.Driver")
+    end
+
+    it "#driver_library" do
+      expect(subject.driver_library).to be_nil
+    end
+
+    it "#user" do
+      expect(subject.user).to eq("bob")
+    end
+
+    it "#password" do
+      expect(subject.password.value).to eq("letmein")
+    end
+  end
+
+
+end

data/spec/filters/jdbc/lookup_processor_spec.rb

@@ -0,0 +1,132 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc/lookup_processor"
+
+module LogStash module Filters module Jdbc
+  describe LookupProcessor do
+    describe "class method find_validation_errors" do
+      context "when supplied with an invalid arg" do
+        it "nil as arg, fails validation" do
+          result = described_class.find_validation_errors(nil)
+          expect(result).to eq("The options must be an Array")
+        end
+
+        it "hash as arg, fails validation" do
+          result = described_class.find_validation_errors({})
+          expect(result).to eq("The options must be an Array")
+        end
+
+        it "array of lookup hash without query key as arg, fails validation" do
+          lookup_hash = {
+            "parameters" => {"ip" => "%%{[ip]}"},
+            "target" => "server"
+          }
+          result = described_class.find_validation_errors([lookup_hash])
+          expect(result).to eq("The options for 'lookup-1' must include a 'query' string")
+        end
+
+        it "array of lookup hash with bad parameters value as arg, fails validation" do
+          lookup_hash = {
+            "query" => "select * from servers WHERE ip LIKE :ip",
+            "parameters" => %w(ip %%{[ip]}),
+            "target" => "server"
+          }
+          result = described_class.find_validation_errors([lookup_hash])
+          expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
+        end
+
+        it "array of lookup hash with bad parameters value as arg and no target, fails validation" do
+          lookup_hash = {
+            "query" => "select * from servers WHERE ip LIKE :ip",
+            "parameters" => %w(ip %%{[ip]})
+          }
+          result = described_class.find_validation_errors([lookup_hash])
+          expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
+        end
+
+        it "array of lookup hashes with invalid settings, fails validation with messages from each lookup" do
+          lookup_hash1 = {
+            "query" => "select * from servers WHERE ip LIKE :ip",
+            "parameters" => ["ip", "%%{[ip]}"],
+            "target" => "server"
+          }
+          lookup_hash2 = {
+            "parameters" => {"id" => "%%{[id]}"},
+            "default_hash" => {},
+            "target" => "server"
+          }
+          result = described_class.find_validation_errors([lookup_hash1, lookup_hash2])
+          expected = ["The 'parameters' option for 'lookup-1' must be a Hash"]
+          expected << "The options for 'lookup-2' must include a 'query' string"
+          expected << "Target setting must be different across all lookups, 'lookup-1', 'lookup-2' have the same target field setting"
+          expect(result).to eq(expected.join("; "))
+        end
+
+        it "array of lookup hashes with same id, fails validation with messages from each lookup" do
+          lookup_hash1 = {
+            "id" => "L1",
+            "query" => "select * from servers WHERE ip LIKE :ip",
+            "parameters" => ["ip", "%%{[ip]}"],
+          }
+          lookup_hash2 = {
+            "id" => "L1",
+            "parameters" => {"id" => "%%{[id]}"},
+            "default_hash" => {},
+          }
+          result = described_class.find_validation_errors([lookup_hash1, lookup_hash2])
+          expected = ["The 'parameters' option for 'L1' must be a Hash"]
+          expected << "The options for 'L1' must include a 'query' string"
+          expected << "Id setting must be different across all lookups, 'L1' is specified multiple times"
+          expect(result).to eq(expected.join("; "))
+        end
+
+        it "array of valid lookup hashes as arg with the same target, fails validation" do
+          lookup_hash1 = {
+            "id" => "L1",
+            "query" => "select * from servers WHERE ip LIKE :ip",
+            "parameters" => {"ip" => "%%{[ip]}"},
+            "target" => "server"
+          }
+          lookup_hash2 = {
+            "id" => "L2",
+            "query" => "select * from users WHERE id LIKE :id",
+            "parameters" => {"id" => "%%{[id]}"},
+            "target" => "server"
+          }
+          lookup_hash3 = {
+            "id" => "L3",
+            "query" => "select * from table1 WHERE ip LIKE :ip",
+            "parameters" => {"ip" => "%%{[ip]}"},
+            "target" => "somefield"
+          }
+          lookup_hash4 = {
+            "id" => "L4",
+            "query" => "select * from table2 WHERE id LIKE :id",
+            "parameters" => {"id" => "%%{[id]}"},
+            "target" => "somefield"
+          }
+          result = described_class.find_validation_errors([lookup_hash1, lookup_hash2, lookup_hash3, lookup_hash4])
+          expect(result).to eq("Target setting must be different across all lookups, 'L1', 'L2' have the same target field setting, 'L3', 'L4' have the same target field setting")
+        end
+      end
+
+      context "when supplied with a valid arg" do
+        it "empty array as arg, passes validation" do
+          result = described_class.find_validation_errors([])
+          expect(result).to eq(nil)
+        end
+
+        it "array of valid lookup hash as arg, passes validation" do
+          lookup_hash = {
+            "query" => "select * from servers WHERE ip LIKE :ip",
+            "parameters" => {"ip" => "%%{[ip]}"},
+            "target" => "server"
+          }
+          result = described_class.find_validation_errors([lookup_hash])
+          expect(result).to eq(nil)
+        end
+      end
+    end
+  end
+end end end
+

data/spec/filters/jdbc/lookup_spec.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+require_relative "../env_helper"
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc/lookup"
+
+module LogStash module Filters module Jdbc
+  describe Lookup do
+    describe "class method find_validation_errors" do
+      context "when supplied with an invalid arg" do
+        it "nil as arg, fails validation" do
+          result = described_class.find_validation_errors(nil)
+          expect(result).to eq("The options must be an Array")
+        end
+
+        it "hash as arg, fails validation" do
+          result = described_class.find_validation_errors({})
+          expect(result).to eq("The options must be an Array")
+        end
+
+        it "array of lookup hash without query key as arg, fails validation" do
+          lookup_hash = {
+          "parameters" => {"ip" => "%%{[ip]}"},
+          "target" => "server"
+          }
+          result = described_class.find_validation_errors([lookup_hash])
+          expect(result).to eq("The options for 'lookup-1' must include a 'query' string")
+        end
+
+        it "array of lookup hash with bad parameters value as arg, fails validation" do
+          lookup_hash = {
+          "query" => "select * from servers WHERE ip LIKE :ip",
+          "parameters" => %w(ip %%{[ip]}),
+          "target" => "server"
+          }
+          result = described_class.find_validation_errors([lookup_hash])
+          expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
+        end
+
+        it "array of lookup hash with bad parameters value as arg and no target, fails validation" do
+          lookup_hash = {
+          "query" => "select * from servers WHERE ip LIKE :ip",
+          "parameters" => %w(ip %%{[ip]})
+          }
+          result = described_class.find_validation_errors([lookup_hash])
+          expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
+        end
+      end
+
+      context "when supplied with a valid arg" do
+        it "empty array as arg, passes validation" do
+          result = described_class.find_validation_errors([])
+          expect(result).to eq(nil)
+        end
+
+        it "array of valid lookup hash as arg, passes validation" do
+          lookup_hash = {
+            "query" => "select * from servers WHERE ip LIKE :ip",
+            "parameters" => {"ip" => "%%{[ip]}"},
+            "target" => "server"
+          }
+          result = described_class.find_validation_errors([lookup_hash])
+          expect(result).to eq(nil)
+        end
+      end
+    end
+
+    describe "abnormal operations" do
+      let(:local_db) { double("local_db") }
+      let(:lookup_hash) do
+        {
+        "query" => "select * from servers WHERE ip LIKE :ip",
+        "parameters" => {"ip" => "%%{[ip]}"},
+        "target" => "server",
+        "tag_on_failure" => ["_jdbcstaticfailure_server"]
+        }
+      end
+      let(:event) { LogStash::Event.new()}
+      let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
+
+      subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
+
+      before(:each) do
+        allow(local_db).to receive(:fetch).once.and_return(records)
+      end
+
+      it "should not enhance an event and it should tag" do
+        subject.enhance(local_db, event)
+        expect(event.get("tags")).to eq(["_jdbcstaticfailure_server"])
+        expect(event.get("server")).to be_nil
+      end
+    end
+
+    describe "normal operations" do
+      let(:local_db) { double("local_db") }
+      let(:lookup_hash) do
+        {
+          "query" => "select * from servers WHERE ip LIKE :ip",
+          "parameters" => {"ip" => "%%{[ip]}"},
+          "target" => "server",
+          "tag_on_failure" => ["_jdbcstaticfailure_server"]
+        }
+      end
+      let(:event) { LogStash::Event.new()}
+      let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
+
+      subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
+
+      before(:each) do
+        allow(local_db).to receive(:fetch).once.and_return(records)
+      end
+
+      it "should be valid" do
+        expect(subject.valid?).to be_truthy
+      end
+
+      it "should have no formatted_errors" do
+        expect(subject.formatted_errors).to eq("")
+      end
+
+      it "should enhance an event" do
+        event.set("ip", "20.20")
+        subject.enhance(local_db, event)
+        expect(event.get("tags")).to be_nil
+        expect(event.get("server")).to eq(records)
+      end
+    end
+  end
+end end end
+

data/spec/filters/jdbc/read_only_database_spec.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/util/password"
+require "logstash/filters/jdbc/read_only_database"
+
+module LogStash module Filters module Jdbc
+  describe ReadOnlyDatabase do
+    let(:db) { Sequel.connect('mock://mydb') }
+    let(:connection_string) { "mock://mydb" }
+    let(:driver_class) { "org.apache.derby.jdbc.EmbeddedDriver" }
+    subject(:read_only_db) { described_class.create(connection_string, driver_class) }
+
+    describe "basic operations" do
+      describe "initializing" do
+        it "tests the connection with defaults" do
+          expect(Sequel::JDBC).to receive(:load_driver).once.with(driver_class)
+          expect(Sequel).to receive(:connect).once.with(connection_string, {:test => true}) #.and_return(db)
+          expect(read_only_db.empty_record_set).to eq([])
+        end
+
+        it "tests the connection with fully specified arguments" do
+          connection_str = "a connection string"
+          user = "a user"
+          password = Util::Password.new("secret")
+          expect(Sequel::JDBC).to receive(:load_driver).once.with("a driver class")
+          expect(Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password =>  password.value, :test => true}).and_return(db)
+          described_class.create(connection_str, "a driver class", nil, user, password)
+        end
+
+        it "connects with defaults" do
+          expect(Sequel::JDBC).to receive(:load_driver).once.with(driver_class)
+          expect(Sequel).to receive(:connect).once.with(connection_string, {:test => true}).and_return(db)
+          expect(Sequel).to receive(:connect).once.with(connection_string, {}).and_return(db)
+          expect(read_only_db.connected?).to be_falsey
+          read_only_db.connect("a caller specific error message")
+          expect(read_only_db.connected?).to be_truthy
+        end
+      end
+
+      describe "methods" do
+        let(:dataset) { double("Sequel::Dataset") }
+
+        before(:each) do
+          allow(Sequel::JDBC).to receive(:load_driver)
+          allow(Sequel).to receive(:connect).thrice.and_return(db)
+          allow(db).to receive(:[]).and_return(dataset)
+          read_only_db.connect("a caller specific error message")
+        end
+
+        after(:each) do
+          read_only_db.disconnect("a caller specific error message")
+        end
+
+        it "the count method gets a count from the dataset" do
+          expect(dataset).to receive(:count).and_return(0)
+          read_only_db.count("select * from table")
+        end
+
+        it "the query method gets all records from the dataset" do
+          expect(dataset).to receive(:all).and_return(read_only_db.empty_record_set)
+          read_only_db.query("select * from table")
+        end
+      end
+    end
+  end
+end end end

data/spec/filters/jdbc/read_write_database_spec.rb

@@ -0,0 +1,89 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash-filter-jdbc_static_jars"
+require "logstash/util/password"
+require "logstash/filters/jdbc/db_object"
+require "logstash/filters/jdbc/read_write_database"
+
+module LogStash module Filters module Jdbc
+  describe ReadWriteDatabase do
+    let(:db) { Sequel.connect('mock://mydb') }
+    let(:connection_string_regex) { /jdbc:derby:memory:\w+;create=true/ }
+    subject(:read_write_db) { described_class.create }
+
+    describe "basic operations" do
+      context "connecting  to a db" do
+        it "connects with defaults" do
+          expect(::Sequel::JDBC).to receive(:load_driver).once.with("org.apache.derby.jdbc.EmbeddedDriver")
+          # two calls to connect because ReadWriteDatabase does verify_connection and connect
+          expect(::Sequel).to receive(:connect).once.with(connection_string_regex, {:test => true}).and_return(db)
+          expect(::Sequel).to receive(:connect).once.with(connection_string_regex, {}).and_return(db)
+          expect(read_write_db.empty_record_set).to eq([])
+        end
+
+        it "connects with fully specified arguments" do
+          connection_str = "a connection string"
+          user = "a user"
+          password = Util::Password.new("secret")
+          expect(::Sequel::JDBC).to receive(:load_driver).once.with("a driver class")
+          expect(::Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password =>  password.value, :test => true}).and_return(db)
+          expect(::Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password =>  password.value}).and_return(db)
+          described_class.create(connection_str, "a driver class", nil, user, password)
+        end
+      end
+
+      describe "methods" do
+        let(:dataset) { double("Sequel::Dataset") }
+        let(:loaders) { [] }
+        let(:loader)  { double("Loader") }
+        let(:table_name) { "users" }
+        let(:temp_name)  { "users_temp" }
+        let(:random_table_name)  { "foobarbaz" }
+
+        before(:each) do
+          allow(::Sequel::JDBC).to receive(:load_driver)
+          allow(::Sequel).to receive(:connect).twice.and_return(db)
+          allow(loader).to receive(:fetch).and_return([1,2,3])
+          allow(loader).to receive(:table).and_return(table_name)
+          allow(loader).to receive(:temp_table).and_return(temp_name)
+          allow(described_class).to receive(:random_name).and_return(random_table_name)
+          loaders.push(loader)
+        end
+
+        it "the populate_all method fills a local_db from the dataset" do
+          expect(db).to receive(:[]).with(loader.temp_table).exactly(2).and_return(dataset)
+          expect(dataset).to receive(:multi_insert).once.with([1,2,3])
+          expect(db).to receive(:rename_table).once.with(temp_name, random_table_name)
+          expect(db).to receive(:rename_table).once.with(table_name, temp_name)
+          expect(db).to receive(:rename_table).once.with(random_table_name, table_name)
+          expect(dataset).to receive(:truncate).once
+          read_write_db.populate_all(loaders)
+        end
+
+        it "the repopulate_all method fills a local_db from the dataset" do
+          expect(db).to receive(:[]).with(loader.temp_table).exactly(2).and_return(dataset)
+          expect(dataset).to receive(:multi_insert).once.with([1,2,3])
+          expect(db).to receive(:rename_table).once.with(temp_name, random_table_name)
+          expect(db).to receive(:rename_table).once.with(table_name, temp_name)
+          expect(db).to receive(:rename_table).once.with(random_table_name, table_name)
+          expect(dataset).to receive(:truncate).once
+          read_write_db.repopulate_all(loaders)
+        end
+
+        it "the fetch method executes a parameterised SQL statement on the local db" do
+          statement = "select 1 from dual"
+          parameters = 42
+          expect(db).to receive(:[]).with(statement, parameters).once.and_return(dataset)
+          expect(dataset).to receive(:all).once.and_return([1,2,3])
+          read_write_db.fetch(statement, parameters)
+        end
+
+        it "lends the local db to a DbObject build instance method" do
+          db_object = DbObject.new("type" => "index", "name" => "servers_idx", "table" => "servers", "columns" => ["ip"])
+          expect(db_object).to receive(:build).once.with(db)
+          read_write_db.build_db_object(db_object)
+        end
+      end
+    end
+  end
+end end end