logstash-filter-jdbc_static 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +2 -0
  3. data/CONTRIBUTORS +22 -0
  4. data/Gemfile +2 -0
  5. data/LICENSE +13 -0
  6. data/README.md +94 -0
  7. data/lib/logstash-filter-jdbc_static_jars.rb +5 -0
  8. data/lib/logstash/filters/jdbc/basic_database.rb +117 -0
  9. data/lib/logstash/filters/jdbc/column.rb +38 -0
  10. data/lib/logstash/filters/jdbc/db_object.rb +103 -0
  11. data/lib/logstash/filters/jdbc/loader.rb +114 -0
  12. data/lib/logstash/filters/jdbc/loader_schedule.rb +38 -0
  13. data/lib/logstash/filters/jdbc/lookup.rb +192 -0
  14. data/lib/logstash/filters/jdbc/lookup_processor.rb +91 -0
  15. data/lib/logstash/filters/jdbc/lookup_result.rb +39 -0
  16. data/lib/logstash/filters/jdbc/read_only_database.rb +57 -0
  17. data/lib/logstash/filters/jdbc/read_write_database.rb +86 -0
  18. data/lib/logstash/filters/jdbc/repeating_load_runner.rb +11 -0
  19. data/lib/logstash/filters/jdbc/single_load_runner.rb +43 -0
  20. data/lib/logstash/filters/jdbc/validatable.rb +49 -0
  21. data/lib/logstash/filters/jdbc_static.rb +216 -0
  22. data/logstash-filter-jdbc_static.gemspec +38 -0
  23. data/spec/filters/env_helper.rb +10 -0
  24. data/spec/filters/jdbc/column_spec.rb +70 -0
  25. data/spec/filters/jdbc/db_object_spec.rb +81 -0
  26. data/spec/filters/jdbc/loader_spec.rb +76 -0
  27. data/spec/filters/jdbc/lookup_processor_spec.rb +132 -0
  28. data/spec/filters/jdbc/lookup_spec.rb +129 -0
  29. data/spec/filters/jdbc/read_only_database_spec.rb +66 -0
  30. data/spec/filters/jdbc/read_write_database_spec.rb +89 -0
  31. data/spec/filters/jdbc/repeating_load_runner_spec.rb +24 -0
  32. data/spec/filters/jdbc/single_load_runner_spec.rb +16 -0
  33. data/spec/filters/jdbc_static_file_local_spec.rb +83 -0
  34. data/spec/filters/jdbc_static_spec.rb +70 -0
  35. data/spec/filters/remote_server_helper.rb +24 -0
  36. data/spec/filters/shared_helpers.rb +35 -0
  37. data/spec/helpers/WHY-THIS-JAR.txt +4 -0
  38. data/spec/helpers/derbyrun.jar +0 -0
  39. data/vendor/jar-dependencies/runtime-jars/derby-10.14.1.0.jar +0 -0
  40. data/vendor/jar-dependencies/runtime-jars/derbyclient-10.14.1.0.jar +0 -0
  41. metadata +224 -0
@@ -0,0 +1,76 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/filters/jdbc/loader"
3
+
4
+ describe LogStash::Filters::Jdbc::Loader do
5
+ let(:local_table) { "servers" }
6
+ let(:options) do
7
+ {
8
+ "jdbc_driver_class" => "org.postgresql.Driver",
9
+ "jdbc_connection_string" => "jdbc:postgres://user:password@remotedb/infra",
10
+ "query" => "select ip, name, location from INTERNAL.SERVERS",
11
+ "jdbc_user" => "bob",
12
+ "jdbc_password" => "letmein",
13
+ "max_rows" => 2000,
14
+ "local_table" => local_table
15
+ }
16
+ end
17
+ subject { described_class.new(options) }
18
+
19
+ context "when correct options are given" do
20
+ it "validation succeeds" do
21
+ expect(subject.valid?).to be_truthy
22
+ expect(subject.formatted_errors).to eq("")
23
+ end
24
+ end
25
+
26
+ context "when incorrect options are given" do
27
+ let(:options) do
28
+ {
29
+ "jdbc_driver_class" => 42,
30
+ "jdbc_connection_string" => 42,
31
+ "max_rows" => ["2000"]
32
+ }
33
+ end
34
+
35
+ it "validation fails" do
36
+ expect(subject.valid?).to be_falsey
37
+ expect(subject.formatted_errors).to eq("The options must include a 'local_table' string, The options for '' must include a 'query' string, The 'max_rows' option for '' must be an integer, The 'jdbc_driver_class' option for '' must be a string, The 'jdbc_connection_string' option for '' must be a string")
38
+ end
39
+ end
40
+
41
+ context "attr_reader methods" do
42
+ it "#table" do
43
+ expect(subject.table).to eq(:servers)
44
+ end
45
+
46
+ it "#query" do
47
+ expect(subject.query).to eq("select ip, name, location from INTERNAL.SERVERS")
48
+ end
49
+
50
+ it "#max_rows" do
51
+ expect(subject.max_rows).to eq(2000)
52
+ end
53
+
54
+ it "#connection_string" do
55
+ expect(subject.connection_string).to eq("jdbc:postgres://user:password@remotedb/infra")
56
+ end
57
+
58
+ it "#driver_class" do
59
+ expect(subject.driver_class).to eq("org.postgresql.Driver")
60
+ end
61
+
62
+ it "#driver_library" do
63
+ expect(subject.driver_library).to be_nil
64
+ end
65
+
66
+ it "#user" do
67
+ expect(subject.user).to eq("bob")
68
+ end
69
+
70
+ it "#password" do
71
+ expect(subject.password.value).to eq("letmein")
72
+ end
73
+ end
74
+
75
+
76
+ end
@@ -0,0 +1,132 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/filters/jdbc/lookup_processor"
4
+
5
+ module LogStash module Filters module Jdbc
6
+ describe LookupProcessor do
7
+ describe "class method find_validation_errors" do
8
+ context "when supplied with an invalid arg" do
9
+ it "nil as arg, fails validation" do
10
+ result = described_class.find_validation_errors(nil)
11
+ expect(result).to eq("The options must be an Array")
12
+ end
13
+
14
+ it "hash as arg, fails validation" do
15
+ result = described_class.find_validation_errors({})
16
+ expect(result).to eq("The options must be an Array")
17
+ end
18
+
19
+ it "array of lookup hash without query key as arg, fails validation" do
20
+ lookup_hash = {
21
+ "parameters" => {"ip" => "%%{[ip]}"},
22
+ "target" => "server"
23
+ }
24
+ result = described_class.find_validation_errors([lookup_hash])
25
+ expect(result).to eq("The options for 'lookup-1' must include a 'query' string")
26
+ end
27
+
28
+ it "array of lookup hash with bad parameters value as arg, fails validation" do
29
+ lookup_hash = {
30
+ "query" => "select * from servers WHERE ip LIKE :ip",
31
+ "parameters" => %w(ip %%{[ip]}),
32
+ "target" => "server"
33
+ }
34
+ result = described_class.find_validation_errors([lookup_hash])
35
+ expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
36
+ end
37
+
38
+ it "array of lookup hash with bad parameters value as arg and no target, fails validation" do
39
+ lookup_hash = {
40
+ "query" => "select * from servers WHERE ip LIKE :ip",
41
+ "parameters" => %w(ip %%{[ip]})
42
+ }
43
+ result = described_class.find_validation_errors([lookup_hash])
44
+ expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
45
+ end
46
+
47
+ it "array of lookup hashes with invalid settings, fails validation with messages from each lookup" do
48
+ lookup_hash1 = {
49
+ "query" => "select * from servers WHERE ip LIKE :ip",
50
+ "parameters" => ["ip", "%%{[ip]}"],
51
+ "target" => "server"
52
+ }
53
+ lookup_hash2 = {
54
+ "parameters" => {"id" => "%%{[id]}"},
55
+ "default_hash" => {},
56
+ "target" => "server"
57
+ }
58
+ result = described_class.find_validation_errors([lookup_hash1, lookup_hash2])
59
+ expected = ["The 'parameters' option for 'lookup-1' must be a Hash"]
60
+ expected << "The options for 'lookup-2' must include a 'query' string"
61
+ expected << "Target setting must be different across all lookups, 'lookup-1', 'lookup-2' have the same target field setting"
62
+ expect(result).to eq(expected.join("; "))
63
+ end
64
+
65
+ it "array of lookup hashes with same id, fails validation with messages from each lookup" do
66
+ lookup_hash1 = {
67
+ "id" => "L1",
68
+ "query" => "select * from servers WHERE ip LIKE :ip",
69
+ "parameters" => ["ip", "%%{[ip]}"],
70
+ }
71
+ lookup_hash2 = {
72
+ "id" => "L1",
73
+ "parameters" => {"id" => "%%{[id]}"},
74
+ "default_hash" => {},
75
+ }
76
+ result = described_class.find_validation_errors([lookup_hash1, lookup_hash2])
77
+ expected = ["The 'parameters' option for 'L1' must be a Hash"]
78
+ expected << "The options for 'L1' must include a 'query' string"
79
+ expected << "Id setting must be different across all lookups, 'L1' is specified multiple times"
80
+ expect(result).to eq(expected.join("; "))
81
+ end
82
+
83
+ it "array of valid lookup hashes as arg with the same target, fails validation" do
84
+ lookup_hash1 = {
85
+ "id" => "L1",
86
+ "query" => "select * from servers WHERE ip LIKE :ip",
87
+ "parameters" => {"ip" => "%%{[ip]}"},
88
+ "target" => "server"
89
+ }
90
+ lookup_hash2 = {
91
+ "id" => "L2",
92
+ "query" => "select * from users WHERE id LIKE :id",
93
+ "parameters" => {"id" => "%%{[id]}"},
94
+ "target" => "server"
95
+ }
96
+ lookup_hash3 = {
97
+ "id" => "L3",
98
+ "query" => "select * from table1 WHERE ip LIKE :ip",
99
+ "parameters" => {"ip" => "%%{[ip]}"},
100
+ "target" => "somefield"
101
+ }
102
+ lookup_hash4 = {
103
+ "id" => "L4",
104
+ "query" => "select * from table2 WHERE id LIKE :id",
105
+ "parameters" => {"id" => "%%{[id]}"},
106
+ "target" => "somefield"
107
+ }
108
+ result = described_class.find_validation_errors([lookup_hash1, lookup_hash2, lookup_hash3, lookup_hash4])
109
+ expect(result).to eq("Target setting must be different across all lookups, 'L1', 'L2' have the same target field setting, 'L3', 'L4' have the same target field setting")
110
+ end
111
+ end
112
+
113
+ context "when supplied with a valid arg" do
114
+ it "empty array as arg, passes validation" do
115
+ result = described_class.find_validation_errors([])
116
+ expect(result).to eq(nil)
117
+ end
118
+
119
+ it "array of valid lookup hash as arg, passes validation" do
120
+ lookup_hash = {
121
+ "query" => "select * from servers WHERE ip LIKE :ip",
122
+ "parameters" => {"ip" => "%%{[ip]}"},
123
+ "target" => "server"
124
+ }
125
+ result = described_class.find_validation_errors([lookup_hash])
126
+ expect(result).to eq(nil)
127
+ end
128
+ end
129
+ end
130
+ end
131
+ end end end
132
+
@@ -0,0 +1,129 @@
1
+ # encoding: utf-8
2
+ require_relative "../env_helper"
3
+ require "logstash/devutils/rspec/spec_helper"
4
+ require "logstash/filters/jdbc/lookup"
5
+
6
+ module LogStash module Filters module Jdbc
7
+ describe Lookup do
8
+ describe "class method find_validation_errors" do
9
+ context "when supplied with an invalid arg" do
10
+ it "nil as arg, fails validation" do
11
+ result = described_class.find_validation_errors(nil)
12
+ expect(result).to eq("The options must be an Array")
13
+ end
14
+
15
+ it "hash as arg, fails validation" do
16
+ result = described_class.find_validation_errors({})
17
+ expect(result).to eq("The options must be an Array")
18
+ end
19
+
20
+ it "array of lookup hash without query key as arg, fails validation" do
21
+ lookup_hash = {
22
+ "parameters" => {"ip" => "%%{[ip]}"},
23
+ "target" => "server"
24
+ }
25
+ result = described_class.find_validation_errors([lookup_hash])
26
+ expect(result).to eq("The options for 'lookup-1' must include a 'query' string")
27
+ end
28
+
29
+ it "array of lookup hash with bad parameters value as arg, fails validation" do
30
+ lookup_hash = {
31
+ "query" => "select * from servers WHERE ip LIKE :ip",
32
+ "parameters" => %w(ip %%{[ip]}),
33
+ "target" => "server"
34
+ }
35
+ result = described_class.find_validation_errors([lookup_hash])
36
+ expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
37
+ end
38
+
39
+ it "array of lookup hash with bad parameters value as arg and no target, fails validation" do
40
+ lookup_hash = {
41
+ "query" => "select * from servers WHERE ip LIKE :ip",
42
+ "parameters" => %w(ip %%{[ip]})
43
+ }
44
+ result = described_class.find_validation_errors([lookup_hash])
45
+ expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
46
+ end
47
+ end
48
+
49
+ context "when supplied with a valid arg" do
50
+ it "empty array as arg, passes validation" do
51
+ result = described_class.find_validation_errors([])
52
+ expect(result).to eq(nil)
53
+ end
54
+
55
+ it "array of valid lookup hash as arg, passes validation" do
56
+ lookup_hash = {
57
+ "query" => "select * from servers WHERE ip LIKE :ip",
58
+ "parameters" => {"ip" => "%%{[ip]}"},
59
+ "target" => "server"
60
+ }
61
+ result = described_class.find_validation_errors([lookup_hash])
62
+ expect(result).to eq(nil)
63
+ end
64
+ end
65
+ end
66
+
67
+ describe "abnormal operations" do
68
+ let(:local_db) { double("local_db") }
69
+ let(:lookup_hash) do
70
+ {
71
+ "query" => "select * from servers WHERE ip LIKE :ip",
72
+ "parameters" => {"ip" => "%%{[ip]}"},
73
+ "target" => "server",
74
+ "tag_on_failure" => ["_jdbcstaticfailure_server"]
75
+ }
76
+ end
77
+ let(:event) { LogStash::Event.new()}
78
+ let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
79
+
80
+ subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
81
+
82
+ before(:each) do
83
+ allow(local_db).to receive(:fetch).once.and_return(records)
84
+ end
85
+
86
+ it "should not enhance an event and it should tag" do
87
+ subject.enhance(local_db, event)
88
+ expect(event.get("tags")).to eq(["_jdbcstaticfailure_server"])
89
+ expect(event.get("server")).to be_nil
90
+ end
91
+ end
92
+
93
+ describe "normal operations" do
94
+ let(:local_db) { double("local_db") }
95
+ let(:lookup_hash) do
96
+ {
97
+ "query" => "select * from servers WHERE ip LIKE :ip",
98
+ "parameters" => {"ip" => "%%{[ip]}"},
99
+ "target" => "server",
100
+ "tag_on_failure" => ["_jdbcstaticfailure_server"]
101
+ }
102
+ end
103
+ let(:event) { LogStash::Event.new()}
104
+ let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
105
+
106
+ subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
107
+
108
+ before(:each) do
109
+ allow(local_db).to receive(:fetch).once.and_return(records)
110
+ end
111
+
112
+ it "should be valid" do
113
+ expect(subject.valid?).to be_truthy
114
+ end
115
+
116
+ it "should have no formatted_errors" do
117
+ expect(subject.formatted_errors).to eq("")
118
+ end
119
+
120
+ it "should enhance an event" do
121
+ event.set("ip", "20.20")
122
+ subject.enhance(local_db, event)
123
+ expect(event.get("tags")).to be_nil
124
+ expect(event.get("server")).to eq(records)
125
+ end
126
+ end
127
+ end
128
+ end end end
129
+
@@ -0,0 +1,66 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/util/password"
4
+ require "logstash/filters/jdbc/read_only_database"
5
+
6
+ module LogStash module Filters module Jdbc
7
+ describe ReadOnlyDatabase do
8
+ let(:db) { Sequel.connect('mock://mydb') }
9
+ let(:connection_string) { "mock://mydb" }
10
+ let(:driver_class) { "org.apache.derby.jdbc.EmbeddedDriver" }
11
+ subject(:read_only_db) { described_class.create(connection_string, driver_class) }
12
+
13
+ describe "basic operations" do
14
+ describe "initializing" do
15
+ it "tests the connection with defaults" do
16
+ expect(Sequel::JDBC).to receive(:load_driver).once.with(driver_class)
17
+ expect(Sequel).to receive(:connect).once.with(connection_string, {:test => true}) #.and_return(db)
18
+ expect(read_only_db.empty_record_set).to eq([])
19
+ end
20
+
21
+ it "tests the connection with fully specified arguments" do
22
+ connection_str = "a connection string"
23
+ user = "a user"
24
+ password = Util::Password.new("secret")
25
+ expect(Sequel::JDBC).to receive(:load_driver).once.with("a driver class")
26
+ expect(Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password => password.value, :test => true}).and_return(db)
27
+ described_class.create(connection_str, "a driver class", nil, user, password)
28
+ end
29
+
30
+ it "connects with defaults" do
31
+ expect(Sequel::JDBC).to receive(:load_driver).once.with(driver_class)
32
+ expect(Sequel).to receive(:connect).once.with(connection_string, {:test => true}).and_return(db)
33
+ expect(Sequel).to receive(:connect).once.with(connection_string, {}).and_return(db)
34
+ expect(read_only_db.connected?).to be_falsey
35
+ read_only_db.connect("a caller specific error message")
36
+ expect(read_only_db.connected?).to be_truthy
37
+ end
38
+ end
39
+
40
+ describe "methods" do
41
+ let(:dataset) { double("Sequel::Dataset") }
42
+
43
+ before(:each) do
44
+ allow(Sequel::JDBC).to receive(:load_driver)
45
+ allow(Sequel).to receive(:connect).thrice.and_return(db)
46
+ allow(db).to receive(:[]).and_return(dataset)
47
+ read_only_db.connect("a caller specific error message")
48
+ end
49
+
50
+ after(:each) do
51
+ read_only_db.disconnect("a caller specific error message")
52
+ end
53
+
54
+ it "the count method gets a count from the dataset" do
55
+ expect(dataset).to receive(:count).and_return(0)
56
+ read_only_db.count("select * from table")
57
+ end
58
+
59
+ it "the query method gets all records from the dataset" do
60
+ expect(dataset).to receive(:all).and_return(read_only_db.empty_record_set)
61
+ read_only_db.query("select * from table")
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end end end
@@ -0,0 +1,89 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash-filter-jdbc_static_jars"
4
+ require "logstash/util/password"
5
+ require "logstash/filters/jdbc/db_object"
6
+ require "logstash/filters/jdbc/read_write_database"
7
+
8
+ module LogStash module Filters module Jdbc
9
+ describe ReadWriteDatabase do
10
+ let(:db) { Sequel.connect('mock://mydb') }
11
+ let(:connection_string_regex) { /jdbc:derby:memory:\w+;create=true/ }
12
+ subject(:read_write_db) { described_class.create }
13
+
14
+ describe "basic operations" do
15
+ context "connecting to a db" do
16
+ it "connects with defaults" do
17
+ expect(::Sequel::JDBC).to receive(:load_driver).once.with("org.apache.derby.jdbc.EmbeddedDriver")
18
+ # two calls to connect because ReadWriteDatabase does verify_connection and connect
19
+ expect(::Sequel).to receive(:connect).once.with(connection_string_regex, {:test => true}).and_return(db)
20
+ expect(::Sequel).to receive(:connect).once.with(connection_string_regex, {}).and_return(db)
21
+ expect(read_write_db.empty_record_set).to eq([])
22
+ end
23
+
24
+ it "connects with fully specified arguments" do
25
+ connection_str = "a connection string"
26
+ user = "a user"
27
+ password = Util::Password.new("secret")
28
+ expect(::Sequel::JDBC).to receive(:load_driver).once.with("a driver class")
29
+ expect(::Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password => password.value, :test => true}).and_return(db)
30
+ expect(::Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password => password.value}).and_return(db)
31
+ described_class.create(connection_str, "a driver class", nil, user, password)
32
+ end
33
+ end
34
+
35
+ describe "methods" do
36
+ let(:dataset) { double("Sequel::Dataset") }
37
+ let(:loaders) { [] }
38
+ let(:loader) { double("Loader") }
39
+ let(:table_name) { "users" }
40
+ let(:temp_name) { "users_temp" }
41
+ let(:random_table_name) { "foobarbaz" }
42
+
43
+ before(:each) do
44
+ allow(::Sequel::JDBC).to receive(:load_driver)
45
+ allow(::Sequel).to receive(:connect).twice.and_return(db)
46
+ allow(loader).to receive(:fetch).and_return([1,2,3])
47
+ allow(loader).to receive(:table).and_return(table_name)
48
+ allow(loader).to receive(:temp_table).and_return(temp_name)
49
+ allow(described_class).to receive(:random_name).and_return(random_table_name)
50
+ loaders.push(loader)
51
+ end
52
+
53
+ it "the populate_all method fills a local_db from the dataset" do
54
+ expect(db).to receive(:[]).with(loader.temp_table).exactly(2).and_return(dataset)
55
+ expect(dataset).to receive(:multi_insert).once.with([1,2,3])
56
+ expect(db).to receive(:rename_table).once.with(temp_name, random_table_name)
57
+ expect(db).to receive(:rename_table).once.with(table_name, temp_name)
58
+ expect(db).to receive(:rename_table).once.with(random_table_name, table_name)
59
+ expect(dataset).to receive(:truncate).once
60
+ read_write_db.populate_all(loaders)
61
+ end
62
+
63
+ it "the repopulate_all method fills a local_db from the dataset" do
64
+ expect(db).to receive(:[]).with(loader.temp_table).exactly(2).and_return(dataset)
65
+ expect(dataset).to receive(:multi_insert).once.with([1,2,3])
66
+ expect(db).to receive(:rename_table).once.with(temp_name, random_table_name)
67
+ expect(db).to receive(:rename_table).once.with(table_name, temp_name)
68
+ expect(db).to receive(:rename_table).once.with(random_table_name, table_name)
69
+ expect(dataset).to receive(:truncate).once
70
+ read_write_db.repopulate_all(loaders)
71
+ end
72
+
73
+ it "the fetch method executes a parameterised SQL statement on the local db" do
74
+ statement = "select 1 from dual"
75
+ parameters = 42
76
+ expect(db).to receive(:[]).with(statement, parameters).once.and_return(dataset)
77
+ expect(dataset).to receive(:all).once.and_return([1,2,3])
78
+ read_write_db.fetch(statement, parameters)
79
+ end
80
+
81
+ it "lends the local db to a DbObject build instance method" do
82
+ db_object = DbObject.new("type" => "index", "name" => "servers_idx", "table" => "servers", "columns" => ["ip"])
83
+ expect(db_object).to receive(:build).once.with(db)
84
+ read_write_db.build_db_object(db_object)
85
+ end
86
+ end
87
+ end
88
+ end
89
+ end end end