logstash-filter-jdbc_static 1.0.0

data/spec/filters/jdbc/repeating_load_runner_spec.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require_relative "../shared_helpers"
+require "logstash/filters/jdbc/repeating_load_runner"
+
+module LogStash module Filters module Jdbc
+  describe RepeatingLoadRunner  do
+    let(:local_db) { double("local_db") }
+    let(:loaders) { Object.new }
+    let(:local_db_objects) { [] }
+    subject(:runner) { described_class.new(local_db, loaders, local_db_objects) }
+
+    it_behaves_like "a single load runner"
+
+    context "when repeating" do
+      it "repopulates the local db" do
+        expect(local_db).to receive(:populate_all).once.with(loaders)
+        expect(local_db).to receive(:repopulate_all).once.with(loaders)
+        runner.initial_load
+        subject.call
+      end
+    end
+  end
+end end end

data/spec/filters/jdbc/single_load_runner_spec.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require_relative "../shared_helpers"
+
+require "logstash/filters/jdbc/single_load_runner"
+
+module LogStash module Filters module Jdbc
+  describe SingleLoadRunner  do
+    let(:local_db) { double("local_db") }
+    let(:loaders) { Object.new }
+    let(:local_db_objects) { [] }
+    subject(:runner) { described_class.new(local_db, loaders, local_db_objects) }
+
+    it_behaves_like "a single load runner"
+  end
+end end end

data/spec/filters/jdbc_static_file_local_spec.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+require_relative "env_helper"
+require_relative "remote_server_helper"
+
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc_static"
+require "sequel"
+require "sequel/adapters/jdbc"
+
+module LogStash module Filters
+  describe JdbcStatic, :skip => "temporary disable lookup_jdbc_* settings" do
+    let(:db1) { ::Sequel.connect("jdbc:derby:memory:testdb;create=true", :user=> nil, :password=> nil) }
+    let(:test_loader) { "SELECT * FROM reference_table" }
+    let(:test_records) { db1[test_loader].all }
+    let(:lookup_db) { "lookupdb" }
+
+    let(:local_db_objects) do
+      [
+        {"name" => "servers", "preserve_existing" => true, "index_columns" => ["ip"], "columns" => [["ip", "varchar(64)"], ["name", "varchar(64)"], ["location", "varchar(64)"]]},
+      ]
+    end
+
+    let(:settings) do
+      {
+        "loaders" => [
+          {
+            "id" =>"servers",
+            "query" => "select ip, name, location from reference_table",
+            "local_table" => "servers"
+          }
+        ],
+        "local_db_objects" => local_db_objects,
+        "local_lookups" => [
+          {
+            "query" => "select * from servers WHERE ip LIKE :ip",
+            "parameters" => {"ip" => "%%{[ip]}"},
+            "target" => "server"
+          }
+        ]
+      }
+    end
+
+    let(:client_jar_path) { ::File.join(BASE_DERBY_DIR, "derbyclient.jar") }
+
+    let(:mixin_settings) do
+      { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+        "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true",
+        "lookup_jdbc_driver_class" => "org.apache.derby.jdbc.ClientDriver",
+        "lookup_jdbc_driver_library" => nil,
+        "lookup_jdbc_connection_string" => "jdbc:derby://localhost:1527/#{lookup_db};create=true" }
+    end
+    let(:plugin) { JdbcStatic.new(mixin_settings.merge(settings)) }
+
+    after do
+      plugin.stop
+      ServerProcessHelpers.jdbc_static_stop_derby_server(lookup_db)
+    end
+
+    before do
+      ServerProcessHelpers.jdbc_static_start_derby_server
+      db1.drop_table(:reference_table) rescue nil
+      db1.create_table :reference_table do
+        String :ip
+        String :name
+        String :location
+      end
+      db1[:reference_table].insert(:ip => "10.1.1.1", :name => "ldn-server-1", :location => "LDN-2-3-4")
+      db1[:reference_table].insert(:ip => "10.2.1.1", :name => "nyc-server-1", :location => "NYC-5-2-8")
+      db1[:reference_table].insert(:ip => "10.3.1.1", :name => "mv-server-1", :location => "MV-9-6-4")
+
+      plugin.register
+    end
+
+    let(:event)      { ::LogStash::Event.new("message" => "some text", "ip" => ipaddr) }
+
+    let(:ipaddr) { ".3.1.1" }
+
+    it "enhances an event" do
+      plugin.filter(event)
+      expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
+    end
+  end
+end end

data/spec/filters/jdbc_static_spec.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc_static"
+require "sequel"
+require "sequel/adapters/jdbc"
+
+module LogStash module Filters
+  describe JdbcStatic do
+    let(:db1) { ::Sequel.connect("jdbc:derby:memory:testdb;create=true", :user=> nil, :password=> nil) }
+    let(:test_loader) { "SELECT * FROM reference_table" }
+    # let(:test_records) { db1[test_loader].all }
+
+    let(:local_db_objects) do
+      [
+        {"name" => "servers", "index_columns" => ["ip"], "columns" => [["ip", "varchar(64)"], ["name", "varchar(64)"], ["location", "varchar(64)"]]},
+      ]
+    end
+
+    let(:settings) do
+      {
+        "loaders" => [
+          {
+            "id" =>"servers",
+            "query" => "select ip, name, location from reference_table",
+            "local_table" => "servers"
+          }
+        ],
+        "local_db_objects" => local_db_objects,
+        "local_lookups" => [
+          {
+            "query" => "select * from servers WHERE ip LIKE :ip",
+            "parameters" => {"ip" => "%%{[ip]}"},
+            "target" => "server"
+          }
+        ]
+      }
+    end
+
+    let(:mixin_settings) do
+      { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+        "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true" }
+    end
+    let(:plugin) { JdbcStatic.new(mixin_settings.merge(settings)) }
+
+    before do
+      db1.drop_table(:reference_table) rescue nil
+      db1.create_table :reference_table do
+        String :ip
+        String :name
+        String :location
+      end
+      db1[:reference_table].insert(:ip => "10.1.1.1", :name => "ldn-server-1", :location => "LDN-2-3-4")
+      db1[:reference_table].insert(:ip => "10.2.1.1", :name => "nyc-server-1", :location => "NYC-5-2-8")
+      db1[:reference_table].insert(:ip => "10.3.1.1", :name => "mv-server-1", :location => "MV-9-6-4")
+
+      plugin.register
+    end
+
+    after { plugin.stop }
+
+    let(:event)      { ::LogStash::Event.new("message" => "some text", "ip" => ipaddr) }
+
+    let(:ipaddr) { ".3.1.1" }
+
+    it "enhances an event" do
+      plugin.filter(event)
+      expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
+    end
+  end
+end end

data/spec/filters/remote_server_helper.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "childprocess"
+
+module ServerProcessHelpers
+  def self.jdbc_static_start_derby_server()
+    # client_out = Stud::Temporary.file
+    # client_out.sync
+    ChildProcess.posix_spawn = true
+    cmd = ["java",  "-jar", "#{BASE_DERBY_DIR}/derbyrun.jar", "server",  "start"]
+    process = ChildProcess.build(*cmd)
+    process.start
+
+    sleep(0.1)
+  end
+
+  def self.jdbc_static_stop_derby_server(test_db)
+    cmd = ["java",  "-jar", "#{BASE_DERBY_DIR}/derbyrun.jar", "server",  "shutdown"]
+    process = ChildProcess.build(*cmd)
+    ChildProcess.posix_spawn = true
+    process.start
+    process.wait
+    `rm -rf #{::File.join(GEM_BASE_DIR, test_db)}`
+  end
+end

data/spec/filters/shared_helpers.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+require "logstash/filters/jdbc/db_object"
+
+RSpec.shared_examples "a single load runner" do
+
+  context "with local db objects" do
+    let(:local_db_objects) do
+      [
+       {"name" => "servers", "index_columns" => ["ip"], "columns" => [%w(ip text), %w(name text), %w(location text)]},
+      ]
+    end
+
+    it "builds local db objects and populates the local db" do
+      expect(local_db).to receive(:populate_all).once.with(loaders)
+      expect(local_db).to receive(:build_db_object).twice.with(instance_of(LogStash::Filters::Jdbc::DbObject))
+      runner.initial_load
+      expect(runner.preloaders).to be_a(Array)
+      expect(runner.preloaders.size).to eq(2)
+      expect(runner.preloaders[0].name).to eq(:servers)
+      expect(runner.preloaders[1].name).to eq(:temp_servers)
+      expect(runner.local).to eq(local_db)
+      expect(runner.loaders).to eq(loaders)
+    end
+  end
+
+  context "without local db objects" do
+    it "populates the local db" do
+      expect(local_db).to receive(:populate_all).once.with(loaders)
+      runner.initial_load
+      expect(runner.preloaders).to eq([])
+      expect(runner.local).to eq(local_db)
+      expect(runner.loaders).to eq(loaders)
+    end
+  end
+end

data/spec/helpers/WHY-THIS-JAR.txt

@@ -0,0 +1,4 @@
+The derbyrun.jar is used by spec/filters/jdbc_static_file_local_spec.rb via
+spec/filters/spec_helper
+The spec helper runs and shuts down a separate Derby server
+Have pity, its only little.

metadata

@@ -0,0 +1,224 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-jdbc_static
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Elastic
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-01-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  name: jar-dependencies
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: sequel
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: tzinfo
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: tzinfo-data
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rufus-scheduler
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: childprocess
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: info@elastic.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash-filter-jdbc_static_jars.rb
+- lib/logstash/filters/jdbc/basic_database.rb
+- lib/logstash/filters/jdbc/column.rb
+- lib/logstash/filters/jdbc/db_object.rb
+- lib/logstash/filters/jdbc/loader.rb
+- lib/logstash/filters/jdbc/loader_schedule.rb
+- lib/logstash/filters/jdbc/lookup.rb
+- lib/logstash/filters/jdbc/lookup_processor.rb
+- lib/logstash/filters/jdbc/lookup_result.rb
+- lib/logstash/filters/jdbc/read_only_database.rb
+- lib/logstash/filters/jdbc/read_write_database.rb
+- lib/logstash/filters/jdbc/repeating_load_runner.rb
+- lib/logstash/filters/jdbc/single_load_runner.rb
+- lib/logstash/filters/jdbc/validatable.rb
+- lib/logstash/filters/jdbc_static.rb
+- logstash-filter-jdbc_static.gemspec
+- spec/filters/env_helper.rb
+- spec/filters/jdbc/column_spec.rb
+- spec/filters/jdbc/db_object_spec.rb
+- spec/filters/jdbc/loader_spec.rb
+- spec/filters/jdbc/lookup_processor_spec.rb
+- spec/filters/jdbc/lookup_spec.rb
+- spec/filters/jdbc/read_only_database_spec.rb
+- spec/filters/jdbc/read_write_database_spec.rb
+- spec/filters/jdbc/repeating_load_runner_spec.rb
+- spec/filters/jdbc/single_load_runner_spec.rb
+- spec/filters/jdbc_static_file_local_spec.rb
+- spec/filters/jdbc_static_spec.rb
+- spec/filters/remote_server_helper.rb
+- spec/filters/shared_helpers.rb
+- spec/helpers/WHY-THIS-JAR.txt
+- spec/helpers/derbyrun.jar
+- vendor/jar-dependencies/runtime-jars/derby-10.14.1.0.jar
+- vendor/jar-dependencies/runtime-jars/derbyclient-10.14.1.0.jar
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+- jars
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar 'org.apache.derby:derby', '10.14.1.0'
+- jar 'org.apache.derby:derbyclient', '10.14.1.0'
+rubyforge_project:
+rubygems_version: 2.6.13
+signing_key:
+specification_version: 4
+summary: This filter executes a SQL query to fetch a SQL query result, store it locally
+  then use a second SQL query to update an event.
+test_files:
+- spec/filters/env_helper.rb
+- spec/filters/jdbc/column_spec.rb
+- spec/filters/jdbc/db_object_spec.rb
+- spec/filters/jdbc/loader_spec.rb
+- spec/filters/jdbc/lookup_processor_spec.rb
+- spec/filters/jdbc/lookup_spec.rb
+- spec/filters/jdbc/read_only_database_spec.rb
+- spec/filters/jdbc/read_write_database_spec.rb
+- spec/filters/jdbc/repeating_load_runner_spec.rb
+- spec/filters/jdbc/single_load_runner_spec.rb
+- spec/filters/jdbc_static_file_local_spec.rb
+- spec/filters/jdbc_static_spec.rb
+- spec/filters/remote_server_helper.rb
+- spec/filters/shared_helpers.rb
+- spec/helpers/WHY-THIS-JAR.txt
+- spec/helpers/derbyrun.jar