logstash-filter-jdbc_static 1.0.1 → 1.0.2

data/logstash-filter-jdbc_static.gemspec

@@ -1,29 +1,23 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-filter-jdbc_static'
-  s.version         = '1.0.1'
-  s.licenses        = ['Apache License (2.0)']
+  s.version         = '1.0.2'
+  s.licenses        = ['Apache-2.0']
   s.summary         = "This filter executes a SQL query to fetch a SQL query result, store it locally then use a second SQL query to update an event."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
   s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
-  # to fool jar_dependencies to mimic our other plugin gradle vendor script behaviour
-  # the rake vendor task removes jars dir and jars downloaded to it
-  s.require_paths   = ["lib", "jars"]
+  s.require_paths   = ["lib", "vendor/jar-dependencies"]
 
   # Files
-  s.files = Dir['lib/**/*','vendor/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
-   # Tests
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+
+  # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
-  derby_version = "10.14.1.0"
-  s.requirements << "jar 'org.apache.derby:derby', '#{derby_version}'"
-  s.requirements << "jar 'org.apache.derby:derbyclient', '#{derby_version}'"
-  # we may need 'org.apache.derby:derbynet' in the future, marking this here
-
   s.add_development_dependency 'jar-dependencies', '~> 0.3'
 
   # Gem dependencies
@@ -34,5 +28,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'rufus-scheduler'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'timecop'
   s.add_development_dependency "childprocess"
 end

data/spec/filters/jdbc/column_spec.rb

@@ -2,7 +2,7 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/filters/jdbc/column"
 
-describe LogStash::Filters::Jdbc::Column  do
+describe LogStash::Filters::Jdbc::Column do
   let(:invalid_messages) do
     [
       "The column options must be an array",
@@ -67,4 +67,4 @@ describe LogStash::Filters::Jdbc::Column  do
       expect(instance.formatted_errors).to eq("")
     end
   end
-end
+end

data/spec/filters/jdbc/loader_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/filters/jdbc/loader"
 

data/spec/filters/jdbc/read_only_database_spec.rb

@@ -8,13 +8,14 @@ module LogStash module Filters module Jdbc
     let(:db) { Sequel.connect('mock://mydb') }
     let(:connection_string) { "mock://mydb" }
     let(:driver_class) { "org.apache.derby.jdbc.EmbeddedDriver" }
-    subject(:read_only_db) { described_class.create(connection_string, driver_class) }
+    let(:driver_library) { nil }
+    subject(:read_only_db) { described_class.create(connection_string, driver_class, driver_library) }
 
     describe "basic operations" do
       describe "initializing" do
         it "tests the connection with defaults" do
           expect(Sequel::JDBC).to receive(:load_driver).once.with(driver_class)
-          expect(Sequel).to receive(:connect).once.with(connection_string, {:test => true}) #.and_return(db)
+          expect(Sequel).to receive(:connect).once.with(connection_string, {:test => true})
           expect(read_only_db.empty_record_set).to eq([])
         end
 
@@ -37,6 +38,29 @@ module LogStash module Filters module Jdbc
         end
       end
 
+      describe "multiple driver loading" do
+        before :each do
+          allow(Sequel::JDBC).to receive(:load_driver)
+          allow_any_instance_of(described_class).to receive(:make_driver_path_loadable)
+        end
+
+        context "using three drivers" do
+          it "makes all loadable" do
+            expect_any_instance_of(described_class).to receive(:make_driver_path_loadable).with(anything, "/foo/bar.jar")
+            expect_any_instance_of(described_class).to receive(:make_driver_path_loadable).with(anything, "/foo/baz.jar")
+            expect_any_instance_of(described_class).to receive(:make_driver_path_loadable).with(anything, "/foo/biz.jar")
+            described_class.create("mock://mydb", "a driver class", "/foo/bar.jar, /foo/baz.jar, /foo/biz.jar")
+          end
+        end
+
+        context "just one driver" do
+          it "makes one loadable" do
+            expect_any_instance_of(described_class).to receive(:make_driver_path_loadable).once.with(anything, "/foo/bar.jar")
+            described_class.create("mock://mydb", "a driver class", "/foo/bar.jar")
+          end
+        end
+      end
+
       describe "methods" do
         let(:dataset) { double("Sequel::Dataset") }
 

data/spec/filters/jdbc/read_write_database_spec.rb

@@ -4,11 +4,13 @@ require "logstash-filter-jdbc_static_jars"
 require "logstash/util/password"
 require "logstash/filters/jdbc/db_object"
 require "logstash/filters/jdbc/read_write_database"
+require "stud/temporary"
 
 module LogStash module Filters module Jdbc
   describe ReadWriteDatabase do
     let(:db) { Sequel.connect('mock://mydb') }
     let(:connection_string_regex) { /jdbc:derby:memory:\w+;create=true/ }
+    let(:temp_import_path_plugin) { Stud::Temporary.pathname }
     subject(:read_write_db) { described_class.create }
 
     describe "basic operations" do
@@ -26,8 +28,8 @@ module LogStash module Filters module Jdbc
           user = "a user"
           password = Util::Password.new("secret")
           expect(::Sequel::JDBC).to receive(:load_driver).once.with("a driver class")
-          expect(::Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password =>  password.value, :test => true}).and_return(db)
-          expect(::Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password =>  password.value}).and_return(db)
+          expect(::Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password => password.value, :test => true}).and_return(db)
+          expect(::Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password => password.value}).and_return(db)
           described_class.create(connection_str, "a driver class", nil, user, password)
         end
       end
@@ -37,36 +39,35 @@ module LogStash module Filters module Jdbc
         let(:loaders) { [] }
         let(:loader)  { double("Loader") }
         let(:table_name) { "users" }
-        let(:temp_name)  { "users_temp" }
         let(:random_table_name)  { "foobarbaz" }
+        let(:multi_insert_sql) { "INSERT VALUES (1, 2, 3)" }
 
         before(:each) do
+          FileUtils.mkdir_p(temp_import_path_plugin)
           allow(::Sequel::JDBC).to receive(:load_driver)
+          # allow(::Sequel).to receive(:connect).twice.and_return(db)
           allow(::Sequel).to receive(:connect).twice.and_return(db)
-          allow(loader).to receive(:fetch).and_return([1,2,3])
+          allow(loader).to receive(:fetch).and_return([{:a => 1, :b => 2, :c => 3}])
           allow(loader).to receive(:table).and_return(table_name)
-          allow(loader).to receive(:temp_table).and_return(temp_name)
+          allow(loader).to receive(:id).and_return("some_id")
+          allow(loader).to receive(:staging_directory).and_return(temp_import_path_plugin)
           allow(described_class).to receive(:random_name).and_return(random_table_name)
           loaders.push(loader)
         end
 
         it "the populate_all method fills a local_db from the dataset" do
-          expect(db).to receive(:[]).with(loader.temp_table).exactly(2).and_return(dataset)
-          expect(dataset).to receive(:multi_insert).once.with([1,2,3])
-          expect(db).to receive(:rename_table).once.with(temp_name, random_table_name)
-          expect(db).to receive(:rename_table).once.with(table_name, temp_name)
-          expect(db).to receive(:rename_table).once.with(random_table_name, table_name)
-          expect(dataset).to receive(:truncate).once
+          expect(db).to receive(:[]).with(loader.table).once.and_return(dataset)
+          expect(dataset).to receive(:literal).and_return(1.to_s, 2.to_s, 3.to_s)
+          import_file_path = ::File.join(temp_import_path_plugin, loader.table)
+          expect(db).to receive(:execute_ddl).once.with("CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE (null,'#{loader.table.upcase}','#{import_file_path}',null,'''',null,1)")
           read_write_db.populate_all(loaders)
         end
 
         it "the repopulate_all method fills a local_db from the dataset" do
-          expect(db).to receive(:[]).with(loader.temp_table).exactly(2).and_return(dataset)
-          expect(dataset).to receive(:multi_insert).once.with([1,2,3])
-          expect(db).to receive(:rename_table).once.with(temp_name, random_table_name)
-          expect(db).to receive(:rename_table).once.with(table_name, temp_name)
-          expect(db).to receive(:rename_table).once.with(random_table_name, table_name)
-          expect(dataset).to receive(:truncate).once
+          expect(db).to receive(:[]).with(loader.table).once.and_return(dataset)
+          expect(dataset).to receive(:literal).and_return(1.to_s, 2.to_s, 3.to_s)
+          import_file_path = ::File.join(temp_import_path_plugin, loader.table)
+          expect(db).to receive(:execute_ddl).once.with("CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE (null,'#{loader.table.upcase}','#{import_file_path}',null,'''',null,1)")
           read_write_db.repopulate_all(loaders)
         end
 

data/spec/filters/jdbc/repeating_load_runner_spec.rb

@@ -21,4 +21,4 @@ module LogStash module Filters module Jdbc
       end
     end
   end
-end end end
+end end end

data/spec/filters/jdbc_static_spec.rb

@@ -3,12 +3,25 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/filters/jdbc_static"
 require "sequel"
 require "sequel/adapters/jdbc"
+require "stud/temporary"
+require "timecop"
+
+LogStash::Logging::Logger::configure_logging("ERROR")
 
 module LogStash module Filters
   describe JdbcStatic do
+
+    before(:all) do
+      @thread_abort = Thread.abort_on_exception
+      Thread.abort_on_exception = true
+    end
+
     let(:db1) { ::Sequel.connect("jdbc:derby:memory:testdb;create=true", :user=> nil, :password=> nil) }
-    let(:test_loader) { "SELECT * FROM reference_table" }
-    # let(:test_records) { db1[test_loader].all }
+    let(:loader_statement) { "SELECT ip, name, location FROM reference_table" }
+    let(:lookup_statement) { "SELECT * FROM servers WHERE ip LIKE :ip" }
+    let(:parameters_rhs) { "%%{[ip]}" }
+    let(:temp_import_path_plugin) { Stud::Temporary.pathname }
+    let(:temp_import_path_rspec) { Stud::Temporary.pathname }
 
     let(:local_db_objects) do
       [
@@ -21,15 +34,15 @@ module LogStash module Filters
         "loaders" => [
           {
             "id" =>"servers",
-            "query" => "select ip, name, location from reference_table",
+            "query" => loader_statement,
             "local_table" => "servers"
           }
         ],
         "local_db_objects" => local_db_objects,
         "local_lookups" => [
           {
-            "query" => "select * from servers WHERE ip LIKE :ip",
-            "parameters" => {"ip" => "%%{[ip]}"},
+            "query" => lookup_statement,
+            "parameters" => {"ip" => parameters_rhs},
             "target" => "server"
           }
         ]
@@ -38,33 +51,98 @@ module LogStash module Filters
 
     let(:mixin_settings) do
       { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
-        "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true" }
+        "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true",
+        "staging_directory" => temp_import_path_plugin
+      }
     end
+
+    let(:add_records) do
+      lambda do |fd|
+        fd.puts "'10.1.1.1', 'ldn-server-1', 'LDN-2-3-4'"
+        fd.puts "'10.2.1.1', 'nyc-server-1', 'NYC-5-2-8'"
+        fd.puts "'10.3.1.1', 'mv-serv''r-1', 'MV-9-6-4'"
+      end
+    end
+
     let(:plugin) { JdbcStatic.new(mixin_settings.merge(settings)) }
 
     before do
       db1.drop_table(:reference_table) rescue nil
-      db1.create_table :reference_table do
+      db1.create_table(:reference_table) do
         String :ip
         String :name
         String :location
       end
-      db1[:reference_table].insert(:ip => "10.1.1.1", :name => "ldn-server-1", :location => "LDN-2-3-4")
-      db1[:reference_table].insert(:ip => "10.2.1.1", :name => "nyc-server-1", :location => "NYC-5-2-8")
-      db1[:reference_table].insert(:ip => "10.3.1.1", :name => "mv-server-1", :location => "MV-9-6-4")
-
-      plugin.register
+      ::File.open(temp_import_path_rspec, "w") do |fd|
+        add_records.call(fd)
+      end
+      import_cmd = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE (null,'REFERENCE_TABLE','#{temp_import_path_rspec}',null,'''',null,1)"
+      db1.execute_ddl(import_cmd)
     end
 
-    after { plugin.stop }
-
     let(:event)      { ::LogStash::Event.new("message" => "some text", "ip" => ipaddr) }
 
     let(:ipaddr) { ".3.1.1" }
 
-    it "enhances an event" do
-      plugin.filter(event)
-      expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
+    describe "non scheduled operation" do
+      after { plugin.stop }
+
+      context "under normal conditions" do
+        it "enhances an event" do
+          plugin.register
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-serv'r-1", "location"=>"MV-9-6-4"}])
+        end
+      end
+
+      context "when the loader query returns no results" do
+        let(:loader_statement) { "SELECT ip, name, location FROM reference_table WHERE ip LIKE '20%'" }
+        it "add an empty array to the target field" do
+          plugin.register
+          plugin.filter(event)
+          expect(event.get("server")).to eq([])
+        end
+      end
+
+      context "when the loader query returns a large recordset, local db is filled in chunks" do
+        let(:add_records) do
+          lambda do |fd|
+            256.times do |octet3|
+              256.times do |octet4|
+                fd.puts "'10.4.#{octet3}.#{octet4}', 'server-#{octet3}-#{octet4}', 'MV-10-#{octet3}-#{octet4}'"
+              end
+            end
+          end
+        end
+        let(:ipaddr) { "10.4.254.255" }
+        let(:lookup_statement) { "SELECT * FROM servers WHERE ip = :ip" }
+        let(:parameters_rhs) { "ip" }
+        it "enhances an event" do
+          plugin.register
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"ip"=>ipaddr, "name"=>"server-254-255", "location"=>"MV-10-254-255"}])
+        end
+      end
+    end
+
+    describe "scheduled operation" do
+      context "given a loader_schedule" do
+        it "should properly schedule" do
+          settings["loader_schedule"] = "*/10 * * * * * UTC"
+          Timecop.travel(Time.now.utc - 3600)
+          Timecop.scale(60)
+          static_filter = JdbcStatic.new(mixin_settings.merge(settings))
+          runner = Thread.new(static_filter) do |filter|
+            filter.register
+          end
+          sleep 3
+          static_filter.filter(event)
+          expect(static_filter.loader_runner.reload_count).to be > 1
+          static_filter.stop
+          Timecop.return
+          expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-serv'r-1", "location"=>"MV-9-6-4"}])
+        end
+      end
     end
   end
 end end

data/spec/filters/shared_helpers.rb

@@ -12,12 +12,11 @@ RSpec.shared_examples "a single load runner" do
 
     it "builds local db objects and populates the local db" do
       expect(local_db).to receive(:populate_all).once.with(loaders)
-      expect(local_db).to receive(:build_db_object).twice.with(instance_of(LogStash::Filters::Jdbc::DbObject))
+      expect(local_db).to receive(:build_db_object).once.with(instance_of(LogStash::Filters::Jdbc::DbObject))
       runner.initial_load
       expect(runner.preloaders).to be_a(Array)
-      expect(runner.preloaders.size).to eq(2)
+      expect(runner.preloaders.size).to eq(1)
       expect(runner.preloaders[0].name).to eq(:servers)
-      expect(runner.preloaders[1].name).to eq(:temp_servers)
       expect(runner.local).to eq(local_db)
       expect(runner.loaders).to eq(loaders)
     end
@@ -32,4 +31,4 @@ RSpec.shared_examples "a single load runner" do
       expect(runner.loaders).to eq(loaders)
     end
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-jdbc_static
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-03-17 00:00:00.000000000 Z
+date: 2018-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -114,6 +114,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: timecop
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -141,6 +155,7 @@ files:
 - Gemfile
 - LICENSE
 - README.md
+- docs/index.asciidoc
 - lib/logstash-filter-jdbc_static_jars.rb
 - lib/logstash/filters/jdbc/basic_database.rb
 - lib/logstash/filters/jdbc/column.rb
@@ -173,11 +188,11 @@ files:
 - spec/filters/shared_helpers.rb
 - spec/helpers/WHY-THIS-JAR.txt
 - spec/helpers/derbyrun.jar
-- vendor/jar-dependencies/runtime-jars/derby-10.14.1.0.jar
-- vendor/jar-dependencies/runtime-jars/derbyclient-10.14.1.0.jar
+- vendor/jar-dependencies/org/apache/derby/derby/10.14.1.0/derby-10.14.1.0.jar
+- vendor/jar-dependencies/org/apache/derby/derbyclient/10.14.1.0/derbyclient-10.14.1.0.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
-- Apache License (2.0)
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: filter
@@ -185,7 +200,7 @@ post_install_message:
 rdoc_options: []
 require_paths:
 - lib
-- jars
+- vendor/jar-dependencies
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -196,9 +211,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-requirements:
-- jar 'org.apache.derby:derby', '10.14.1.0'
-- jar 'org.apache.derby:derbyclient', '10.14.1.0'
+requirements: []
 rubyforge_project:
 rubygems_version: 2.6.13
 signing_key: