logstash-integration-jdbc 5.0.0.alpha1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +8 -0
- data/CONTRIBUTORS +22 -0
- data/Gemfile +11 -0
- data/LICENSE +13 -0
- data/NOTICE.TXT +5 -0
- data/README.md +105 -0
- data/docs/filter-jdbc_static.asciidoc +606 -0
- data/docs/filter-jdbc_streaming.asciidoc +317 -0
- data/docs/index.asciidoc +32 -0
- data/docs/input-jdbc.asciidoc +573 -0
- data/lib/logstash/filters/jdbc/basic_database.rb +125 -0
- data/lib/logstash/filters/jdbc/column.rb +39 -0
- data/lib/logstash/filters/jdbc/db_object.rb +101 -0
- data/lib/logstash/filters/jdbc/loader.rb +119 -0
- data/lib/logstash/filters/jdbc/loader_schedule.rb +64 -0
- data/lib/logstash/filters/jdbc/lookup.rb +253 -0
- data/lib/logstash/filters/jdbc/lookup_processor.rb +100 -0
- data/lib/logstash/filters/jdbc/lookup_result.rb +40 -0
- data/lib/logstash/filters/jdbc/read_only_database.rb +57 -0
- data/lib/logstash/filters/jdbc/read_write_database.rb +108 -0
- data/lib/logstash/filters/jdbc/repeating_load_runner.rb +13 -0
- data/lib/logstash/filters/jdbc/single_load_runner.rb +46 -0
- data/lib/logstash/filters/jdbc/validatable.rb +46 -0
- data/lib/logstash/filters/jdbc_static.rb +240 -0
- data/lib/logstash/filters/jdbc_streaming.rb +196 -0
- data/lib/logstash/inputs/jdbc.rb +341 -0
- data/lib/logstash/inputs/tzinfo_jruby_patch.rb +57 -0
- data/lib/logstash/plugin_mixins/jdbc/checked_count_logger.rb +43 -0
- data/lib/logstash/plugin_mixins/jdbc/jdbc.rb +298 -0
- data/lib/logstash/plugin_mixins/jdbc/statement_handler.rb +129 -0
- data/lib/logstash/plugin_mixins/jdbc/value_tracking.rb +140 -0
- data/lib/logstash/plugin_mixins/jdbc_streaming/cache_payload.rb +28 -0
- data/lib/logstash/plugin_mixins/jdbc_streaming/parameter_handler.rb +64 -0
- data/lib/logstash/plugin_mixins/jdbc_streaming/statement_handler.rb +143 -0
- data/lib/logstash/plugin_mixins/jdbc_streaming.rb +100 -0
- data/lib/logstash/plugin_mixins/statement_handler.rb +0 -0
- data/lib/logstash-integration-jdbc_jars.rb +5 -0
- data/logstash-integration-jdbc.gemspec +44 -0
- data/spec/filters/env_helper.rb +10 -0
- data/spec/filters/integration/jdbc_static_spec.rb +154 -0
- data/spec/filters/integration/jdbcstreaming_spec.rb +173 -0
- data/spec/filters/jdbc/column_spec.rb +70 -0
- data/spec/filters/jdbc/db_object_spec.rb +81 -0
- data/spec/filters/jdbc/loader_spec.rb +77 -0
- data/spec/filters/jdbc/lookup_processor_spec.rb +132 -0
- data/spec/filters/jdbc/lookup_spec.rb +253 -0
- data/spec/filters/jdbc/read_only_database_spec.rb +67 -0
- data/spec/filters/jdbc/read_write_database_spec.rb +90 -0
- data/spec/filters/jdbc/repeating_load_runner_spec.rb +24 -0
- data/spec/filters/jdbc/single_load_runner_spec.rb +16 -0
- data/spec/filters/jdbc_static_file_local_spec.rb +83 -0
- data/spec/filters/jdbc_static_spec.rb +162 -0
- data/spec/filters/jdbc_streaming_spec.rb +350 -0
- data/spec/filters/remote_server_helper.rb +24 -0
- data/spec/filters/shared_helpers.rb +34 -0
- data/spec/helpers/WHY-THIS-JAR.txt +4 -0
- data/spec/helpers/derbyrun.jar +0 -0
- data/spec/inputs/integration/integ_spec.rb +78 -0
- data/spec/inputs/jdbc_spec.rb +1431 -0
- data/vendor/jar-dependencies/org/apache/derby/derby/10.14.1.0/derby-10.14.1.0.jar +0 -0
- data/vendor/jar-dependencies/org/apache/derby/derbyclient/10.14.1.0/derbyclient-10.14.1.0.jar +0 -0
- metadata +319 -0
@@ -0,0 +1,70 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/filters/jdbc/column"
|
4
|
+
|
5
|
+
describe LogStash::Filters::Jdbc::Column do
|
6
|
+
let(:invalid_messages) do
|
7
|
+
[
|
8
|
+
"The column options must be an array",
|
9
|
+
"The first column option is the name and must be a string",
|
10
|
+
"The second column option is the datatype and must be a string"
|
11
|
+
]
|
12
|
+
end
|
13
|
+
|
14
|
+
context "various invalid non-array arguments" do
|
15
|
+
it "a nil does not validate" do
|
16
|
+
instance = described_class.new(nil)
|
17
|
+
expect(instance.valid?).to be_falsey
|
18
|
+
expect(instance.formatted_errors).to eq(invalid_messages.join(", "))
|
19
|
+
end
|
20
|
+
|
21
|
+
it "a string does not validate" do
|
22
|
+
instance = described_class.new("foo")
|
23
|
+
expect(instance.valid?).to be_falsey
|
24
|
+
expect(instance.formatted_errors).to eq(invalid_messages.values_at(0,2).join(", "))
|
25
|
+
end
|
26
|
+
|
27
|
+
it "a number does not validate" do
|
28
|
+
instance = described_class.new(42)
|
29
|
+
expect(instance.valid?).to be_falsey
|
30
|
+
expect(instance.formatted_errors).to eq(invalid_messages.join(", "))
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
context "various invalid array arguments" do
|
35
|
+
it "a single string element does not validate" do
|
36
|
+
instance = described_class.new(["foo"])
|
37
|
+
expect(instance.valid?).to be_falsey
|
38
|
+
expect(instance.formatted_errors).to eq(invalid_messages.last)
|
39
|
+
end
|
40
|
+
[ [], [1, 2] ].each do |arg|
|
41
|
+
it "do not validate" do
|
42
|
+
instance = described_class.new(arg)
|
43
|
+
expect(instance.valid?).to be_falsey
|
44
|
+
expect(instance.formatted_errors).to eq(invalid_messages.values_at(1,2).join(", "))
|
45
|
+
end
|
46
|
+
end
|
47
|
+
[ ["foo", 3], ["foo", nil] ].each do |arg|
|
48
|
+
it "do not validate" do
|
49
|
+
instance = described_class.new(arg)
|
50
|
+
expect(instance.valid?).to be_falsey
|
51
|
+
expect(instance.formatted_errors).to eq(invalid_messages.last)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
[ [3, "foo"], [nil, "foo"] ].each do |arg|
|
55
|
+
it "do not validate" do
|
56
|
+
instance = described_class.new(arg)
|
57
|
+
expect(instance.valid?).to be_falsey
|
58
|
+
expect(instance.formatted_errors).to eq(invalid_messages[1])
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
context "a valid array argument" do
|
64
|
+
it "does validate" do
|
65
|
+
instance = described_class.new(["foo", "varchar2"])
|
66
|
+
expect(instance.valid?).to be_truthy
|
67
|
+
expect(instance.formatted_errors).to eq("")
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/filters/jdbc/db_object"
|
4
|
+
|
5
|
+
describe LogStash::Filters::Jdbc::DbObject do
|
6
|
+
context "various invalid non-hash arguments" do
|
7
|
+
it "a nil does not validate" do
|
8
|
+
instance = described_class.new(nil)
|
9
|
+
expect(instance.valid?).to be_falsey
|
10
|
+
expect(instance.formatted_errors).to eq("DbObject options must be a Hash")
|
11
|
+
end
|
12
|
+
|
13
|
+
it "a string does not validate" do
|
14
|
+
instance = described_class.new("foo")
|
15
|
+
expect(instance.valid?).to be_falsey
|
16
|
+
expect(instance.formatted_errors).to eq("DbObject options must be a Hash")
|
17
|
+
end
|
18
|
+
|
19
|
+
it "a number does not validate" do
|
20
|
+
instance = described_class.new(42)
|
21
|
+
expect(instance.valid?).to be_falsey
|
22
|
+
expect(instance.formatted_errors).to eq("DbObject options must be a Hash")
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
context "various invalid hash arguments" do
|
27
|
+
let(:error_messages) do
|
28
|
+
[
|
29
|
+
"DbObject options must include a 'name' string",
|
30
|
+
"DbObject options for 'foo' must include a 'columns' array"
|
31
|
+
]
|
32
|
+
end
|
33
|
+
"DbObject options must include a 'name' string, DbObject options for 'unnamed' must include a 'columns' array"
|
34
|
+
it "an empty hash does not validate" do
|
35
|
+
instance = described_class.new({})
|
36
|
+
expect(instance.valid?).to be_falsey
|
37
|
+
expect(instance.formatted_errors).to eq(error_messages.values_at(0,1).join(", ").gsub('foo', 'unnamed'))
|
38
|
+
end
|
39
|
+
|
40
|
+
it "a name key value only" do
|
41
|
+
instance = described_class.new({"name" => "foo"})
|
42
|
+
expect(instance.valid?).to be_falsey
|
43
|
+
expect(instance.formatted_errors).to eq(error_messages[1])
|
44
|
+
end
|
45
|
+
|
46
|
+
it "a name and bad columns" do
|
47
|
+
instance = described_class.new({"name" => "foo", "columns" => 42})
|
48
|
+
expect(instance.valid?).to be_falsey
|
49
|
+
expect(instance.formatted_errors).to eq(error_messages[1])
|
50
|
+
end
|
51
|
+
|
52
|
+
it "a name and bad columns - empty array" do
|
53
|
+
instance = described_class.new({"name" => "foo", "columns" => []})
|
54
|
+
expect(instance.valid?).to be_falsey
|
55
|
+
msg = "The columns array for 'foo' is not uniform, it should contain arrays of two strings only"
|
56
|
+
expect(instance.formatted_errors).to eq(msg)
|
57
|
+
end
|
58
|
+
|
59
|
+
it "a name and bad columns - irregular arrays" do
|
60
|
+
instance = described_class.new({"name" => "foo", "columns" => [["ip", "text"], ["name"], ["a", "b", "c"]]})
|
61
|
+
expect(instance.valid?).to be_falsey
|
62
|
+
msg = "The columns array for 'foo' is not uniform, it should contain arrays of two strings only"
|
63
|
+
expect(instance.formatted_errors).to eq(msg)
|
64
|
+
end
|
65
|
+
|
66
|
+
it "a name, good columns and bad index_column" do
|
67
|
+
instance = described_class.new({"name" => "foo_index", "index_columns" => ["bar"], "columns" => [["ip", "text"], ["name", "text"]]})
|
68
|
+
expect(instance.valid?).to be_falsey
|
69
|
+
msg = "The index_columns element: 'bar' must be a column defined in the columns array"
|
70
|
+
expect(instance.formatted_errors).to eq(msg)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
context "a valid hash argument" do
|
75
|
+
it "does validate" do
|
76
|
+
instance = described_class.new({"name" => "foo", "index_columns" => ["ip"], "columns" => [["ip", "text"], ["name", "text"]]})
|
77
|
+
expect(instance.formatted_errors).to eq("")
|
78
|
+
expect(instance.valid?).to be_truthy
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/filters/jdbc/loader"
|
4
|
+
|
5
|
+
describe LogStash::Filters::Jdbc::Loader do
|
6
|
+
let(:local_table) { "servers" }
|
7
|
+
let(:options) do
|
8
|
+
{
|
9
|
+
"jdbc_driver_class" => "org.postgresql.Driver",
|
10
|
+
"jdbc_connection_string" => "jdbc:postgres://user:password@remotedb/infra",
|
11
|
+
"query" => "select ip, name, location from INTERNAL.SERVERS",
|
12
|
+
"jdbc_user" => "bob",
|
13
|
+
"jdbc_password" => "letmein",
|
14
|
+
"max_rows" => 2000,
|
15
|
+
"local_table" => local_table
|
16
|
+
}
|
17
|
+
end
|
18
|
+
subject { described_class.new(options) }
|
19
|
+
|
20
|
+
context "when correct options are given" do
|
21
|
+
it "validation succeeds" do
|
22
|
+
expect(subject.valid?).to be_truthy
|
23
|
+
expect(subject.formatted_errors).to eq("")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
context "when incorrect options are given" do
|
28
|
+
let(:options) do
|
29
|
+
{
|
30
|
+
"jdbc_driver_class" => 42,
|
31
|
+
"jdbc_connection_string" => 42,
|
32
|
+
"max_rows" => ["2000"]
|
33
|
+
}
|
34
|
+
end
|
35
|
+
|
36
|
+
it "validation fails" do
|
37
|
+
expect(subject.valid?).to be_falsey
|
38
|
+
expect(subject.formatted_errors).to eq("The options must include a 'local_table' string, The options for '' must include a 'query' string, The 'max_rows' option for '' must be an integer, The 'jdbc_driver_class' option for '' must be a string, The 'jdbc_connection_string' option for '' must be a string")
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
context "attr_reader methods" do
|
43
|
+
it "#table" do
|
44
|
+
expect(subject.table).to eq(:servers)
|
45
|
+
end
|
46
|
+
|
47
|
+
it "#query" do
|
48
|
+
expect(subject.query).to eq("select ip, name, location from INTERNAL.SERVERS")
|
49
|
+
end
|
50
|
+
|
51
|
+
it "#max_rows" do
|
52
|
+
expect(subject.max_rows).to eq(2000)
|
53
|
+
end
|
54
|
+
|
55
|
+
it "#connection_string" do
|
56
|
+
expect(subject.connection_string).to eq("jdbc:postgres://user:password@remotedb/infra")
|
57
|
+
end
|
58
|
+
|
59
|
+
it "#driver_class" do
|
60
|
+
expect(subject.driver_class).to eq("org.postgresql.Driver")
|
61
|
+
end
|
62
|
+
|
63
|
+
it "#driver_library" do
|
64
|
+
expect(subject.driver_library).to be_nil
|
65
|
+
end
|
66
|
+
|
67
|
+
it "#user" do
|
68
|
+
expect(subject.user).to eq("bob")
|
69
|
+
end
|
70
|
+
|
71
|
+
it "#password" do
|
72
|
+
expect(subject.password.value).to eq("letmein")
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
|
77
|
+
end
|
@@ -0,0 +1,132 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/filters/jdbc/lookup_processor"
|
4
|
+
|
5
|
+
module LogStash module Filters module Jdbc
|
6
|
+
describe LookupProcessor do
|
7
|
+
describe "class method find_validation_errors" do
|
8
|
+
context "when supplied with an invalid arg" do
|
9
|
+
it "nil as arg, fails validation" do
|
10
|
+
result = described_class.find_validation_errors(nil)
|
11
|
+
expect(result).to eq("The options must be an Array")
|
12
|
+
end
|
13
|
+
|
14
|
+
it "hash as arg, fails validation" do
|
15
|
+
result = described_class.find_validation_errors({})
|
16
|
+
expect(result).to eq("The options must be an Array")
|
17
|
+
end
|
18
|
+
|
19
|
+
it "array of lookup hash without query key as arg, fails validation" do
|
20
|
+
lookup_hash = {
|
21
|
+
"parameters" => {"ip" => "%%{[ip]}"},
|
22
|
+
"target" => "server"
|
23
|
+
}
|
24
|
+
result = described_class.find_validation_errors([lookup_hash])
|
25
|
+
expect(result).to eq("The options for 'lookup-1' must include a 'query' string")
|
26
|
+
end
|
27
|
+
|
28
|
+
it "array of lookup hash with bad parameters value as arg, fails validation" do
|
29
|
+
lookup_hash = {
|
30
|
+
"query" => "select * from servers WHERE ip LIKE :ip",
|
31
|
+
"parameters" => %w(ip %%{[ip]}),
|
32
|
+
"target" => "server"
|
33
|
+
}
|
34
|
+
result = described_class.find_validation_errors([lookup_hash])
|
35
|
+
expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
|
36
|
+
end
|
37
|
+
|
38
|
+
it "array of lookup hash with bad parameters value as arg and no target, fails validation" do
|
39
|
+
lookup_hash = {
|
40
|
+
"query" => "select * from servers WHERE ip LIKE :ip",
|
41
|
+
"parameters" => %w(ip %%{[ip]})
|
42
|
+
}
|
43
|
+
result = described_class.find_validation_errors([lookup_hash])
|
44
|
+
expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
|
45
|
+
end
|
46
|
+
|
47
|
+
it "array of lookup hashes with invalid settings, fails validation with messages from each lookup" do
|
48
|
+
lookup_hash1 = {
|
49
|
+
"query" => "select * from servers WHERE ip LIKE :ip",
|
50
|
+
"parameters" => ["ip", "%%{[ip]}"],
|
51
|
+
"target" => "server"
|
52
|
+
}
|
53
|
+
lookup_hash2 = {
|
54
|
+
"parameters" => {"id" => "%%{[id]}"},
|
55
|
+
"default_hash" => {},
|
56
|
+
"target" => "server"
|
57
|
+
}
|
58
|
+
result = described_class.find_validation_errors([lookup_hash1, lookup_hash2])
|
59
|
+
expected = ["The 'parameters' option for 'lookup-1' must be a Hash"]
|
60
|
+
expected << "The options for 'lookup-2' must include a 'query' string"
|
61
|
+
expected << "Target setting must be different across all lookups, 'lookup-1', 'lookup-2' have the same target field setting"
|
62
|
+
expect(result).to eq(expected.join("; "))
|
63
|
+
end
|
64
|
+
|
65
|
+
it "array of lookup hashes with same id, fails validation with messages from each lookup" do
|
66
|
+
lookup_hash1 = {
|
67
|
+
"id" => "L1",
|
68
|
+
"query" => "select * from servers WHERE ip LIKE :ip",
|
69
|
+
"parameters" => ["ip", "%%{[ip]}"],
|
70
|
+
}
|
71
|
+
lookup_hash2 = {
|
72
|
+
"id" => "L1",
|
73
|
+
"parameters" => {"id" => "%%{[id]}"},
|
74
|
+
"default_hash" => {},
|
75
|
+
}
|
76
|
+
result = described_class.find_validation_errors([lookup_hash1, lookup_hash2])
|
77
|
+
expected = ["The 'parameters' option for 'L1' must be a Hash"]
|
78
|
+
expected << "The options for 'L1' must include a 'query' string"
|
79
|
+
expected << "Id setting must be different across all lookups, 'L1' is specified multiple times"
|
80
|
+
expect(result).to eq(expected.join("; "))
|
81
|
+
end
|
82
|
+
|
83
|
+
it "array of valid lookup hashes as arg with the same target, fails validation" do
|
84
|
+
lookup_hash1 = {
|
85
|
+
"id" => "L1",
|
86
|
+
"query" => "select * from servers WHERE ip LIKE :ip",
|
87
|
+
"parameters" => {"ip" => "%%{[ip]}"},
|
88
|
+
"target" => "server"
|
89
|
+
}
|
90
|
+
lookup_hash2 = {
|
91
|
+
"id" => "L2",
|
92
|
+
"query" => "select * from users WHERE id LIKE :id",
|
93
|
+
"parameters" => {"id" => "%%{[id]}"},
|
94
|
+
"target" => "server"
|
95
|
+
}
|
96
|
+
lookup_hash3 = {
|
97
|
+
"id" => "L3",
|
98
|
+
"query" => "select * from table1 WHERE ip LIKE :ip",
|
99
|
+
"parameters" => {"ip" => "%%{[ip]}"},
|
100
|
+
"target" => "somefield"
|
101
|
+
}
|
102
|
+
lookup_hash4 = {
|
103
|
+
"id" => "L4",
|
104
|
+
"query" => "select * from table2 WHERE id LIKE :id",
|
105
|
+
"parameters" => {"id" => "%%{[id]}"},
|
106
|
+
"target" => "somefield"
|
107
|
+
}
|
108
|
+
result = described_class.find_validation_errors([lookup_hash1, lookup_hash2, lookup_hash3, lookup_hash4])
|
109
|
+
expect(result).to eq("Target setting must be different across all lookups, 'L1', 'L2' have the same target field setting, 'L3', 'L4' have the same target field setting")
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
context "when supplied with a valid arg" do
|
114
|
+
it "empty array as arg, passes validation" do
|
115
|
+
result = described_class.find_validation_errors([])
|
116
|
+
expect(result).to eq(nil)
|
117
|
+
end
|
118
|
+
|
119
|
+
it "array of valid lookup hash as arg, passes validation" do
|
120
|
+
lookup_hash = {
|
121
|
+
"query" => "select * from servers WHERE ip LIKE :ip",
|
122
|
+
"parameters" => {"ip" => "%%{[ip]}"},
|
123
|
+
"target" => "server"
|
124
|
+
}
|
125
|
+
result = described_class.find_validation_errors([lookup_hash])
|
126
|
+
expect(result).to eq(nil)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end end end
|
132
|
+
|
@@ -0,0 +1,253 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative "../env_helper"
|
3
|
+
require "logstash/devutils/rspec/spec_helper"
|
4
|
+
require "logstash/filters/jdbc/lookup"
|
5
|
+
|
6
|
+
module LogStash module Filters module Jdbc
|
7
|
+
describe Lookup do
|
8
|
+
describe "class method find_validation_errors" do
|
9
|
+
context "when supplied with an invalid arg" do
|
10
|
+
it "nil as arg, fails validation" do
|
11
|
+
result = described_class.find_validation_errors(nil)
|
12
|
+
expect(result).to eq("The options must be an Array")
|
13
|
+
end
|
14
|
+
|
15
|
+
it "hash as arg, fails validation" do
|
16
|
+
result = described_class.find_validation_errors({})
|
17
|
+
expect(result).to eq("The options must be an Array")
|
18
|
+
end
|
19
|
+
|
20
|
+
it "array of lookup hash without query key as arg, fails validation" do
|
21
|
+
lookup_hash = {
|
22
|
+
"parameters" => {"ip" => "%%{[ip]}"},
|
23
|
+
"target" => "server"
|
24
|
+
}
|
25
|
+
result = described_class.find_validation_errors([lookup_hash])
|
26
|
+
expect(result).to eq("The options for 'lookup-1' must include a 'query' string")
|
27
|
+
end
|
28
|
+
|
29
|
+
it "array of lookup hash with bad parameters value as arg, fails validation" do
|
30
|
+
lookup_hash = {
|
31
|
+
"query" => "select * from servers WHERE ip LIKE :ip",
|
32
|
+
"parameters" => %w(ip %%{[ip]}),
|
33
|
+
"target" => "server"
|
34
|
+
}
|
35
|
+
result = described_class.find_validation_errors([lookup_hash])
|
36
|
+
expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
|
37
|
+
end
|
38
|
+
|
39
|
+
it "array of lookup hash with bad parameters value as arg and no target, fails validation" do
|
40
|
+
lookup_hash = {
|
41
|
+
"query" => "select * from servers WHERE ip LIKE :ip",
|
42
|
+
"parameters" => %w(ip %%{[ip]})
|
43
|
+
}
|
44
|
+
result = described_class.find_validation_errors([lookup_hash])
|
45
|
+
expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
|
46
|
+
end
|
47
|
+
|
48
|
+
it "parameters and prepared_parameters are defined at same time" do
|
49
|
+
lookup_hash = {
|
50
|
+
"query" => "SELECT * FROM table WHERE ip=?",
|
51
|
+
"parameters" => {"ip" => "%%{[ip]}"},
|
52
|
+
"prepared_parameters" => ["%%{[ip]}"],
|
53
|
+
"target" => "server"
|
54
|
+
}
|
55
|
+
result = described_class.find_validation_errors([lookup_hash])
|
56
|
+
expect(result).to eq("Can't specify 'parameters' and 'prepared_parameters' in the same lookup")
|
57
|
+
end
|
58
|
+
|
59
|
+
it "prepared_parameters count doesn't match the number of '?' in the query" do
|
60
|
+
lookup_hash = {
|
61
|
+
"query" => "SELECT * FROM table WHERE ip=? AND host=?",
|
62
|
+
"prepared_parameters" => ["%%{[ip]}"],
|
63
|
+
"target" => "server"
|
64
|
+
}
|
65
|
+
result = described_class.find_validation_errors([lookup_hash])
|
66
|
+
expect(result).to eq("The 'prepared_parameters' option for 'lookup-1' doesn't match count with query's placeholder")
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
context "when supplied with a valid arg" do
|
71
|
+
it "empty array as arg, passes validation" do
|
72
|
+
result = described_class.find_validation_errors([])
|
73
|
+
expect(result).to eq(nil)
|
74
|
+
end
|
75
|
+
|
76
|
+
it "array of valid lookup hash as arg, passes validation" do
|
77
|
+
lookup_hash = {
|
78
|
+
"query" => "select * from servers WHERE ip LIKE :ip",
|
79
|
+
"parameters" => {"ip" => "%%{[ip]}"},
|
80
|
+
"target" => "server"
|
81
|
+
}
|
82
|
+
result = described_class.find_validation_errors([lookup_hash])
|
83
|
+
expect(result).to eq(nil)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
describe "abnormal operations" do
|
89
|
+
let(:local_db) { double("local_db") }
|
90
|
+
let(:lookup_hash) do
|
91
|
+
{
|
92
|
+
"query" => "select * from servers WHERE ip LIKE :ip",
|
93
|
+
"parameters" => {"ip" => "%%{[ip]}"},
|
94
|
+
"target" => "server",
|
95
|
+
"tag_on_failure" => ["_jdbcstaticfailure_server"]
|
96
|
+
}
|
97
|
+
end
|
98
|
+
let(:event) { LogStash::Event.new()}
|
99
|
+
let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
|
100
|
+
|
101
|
+
subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
|
102
|
+
|
103
|
+
before(:each) do
|
104
|
+
allow(local_db).to receive(:fetch).once.and_return(records)
|
105
|
+
end
|
106
|
+
|
107
|
+
it "should not enhance an event and it should tag" do
|
108
|
+
subject.enhance(local_db, event)
|
109
|
+
expect(event.get("tags")).to eq(["_jdbcstaticfailure_server"])
|
110
|
+
expect(event.get("server")).to be_nil
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
describe "normal operations" do
|
115
|
+
let(:local_db) { double("local_db") }
|
116
|
+
let(:lookup_hash) do
|
117
|
+
{
|
118
|
+
"query" => "select * from servers WHERE ip LIKE :ip",
|
119
|
+
"parameters" => {"ip" => "%%{[ip]}"},
|
120
|
+
"target" => "server",
|
121
|
+
"tag_on_failure" => ["_jdbcstaticfailure_server"]
|
122
|
+
}
|
123
|
+
end
|
124
|
+
let(:event) { LogStash::Event.new()}
|
125
|
+
let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
|
126
|
+
|
127
|
+
subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
|
128
|
+
|
129
|
+
before(:each) do
|
130
|
+
allow(local_db).to receive(:fetch).once.and_return(records)
|
131
|
+
end
|
132
|
+
|
133
|
+
it "should be valid" do
|
134
|
+
expect(subject.valid?).to be_truthy
|
135
|
+
end
|
136
|
+
|
137
|
+
it "should have no formatted_errors" do
|
138
|
+
expect(subject.formatted_errors).to eq("")
|
139
|
+
end
|
140
|
+
|
141
|
+
it "should enhance an event" do
|
142
|
+
event.set("ip", "20.20")
|
143
|
+
subject.enhance(local_db, event)
|
144
|
+
expect(event.get("tags")).to be_nil
|
145
|
+
expect(event.get("server")).to eq(records)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
describe "lookup operations with prepared statement" do
|
150
|
+
let(:local_db) { double("local_db") }
|
151
|
+
let(:lookup_hash) do
|
152
|
+
{
|
153
|
+
"query" => "select * from servers WHERE ip LIKE ?",
|
154
|
+
"prepared_parameters" => ["%%{[ip]}"],
|
155
|
+
"target" => "server",
|
156
|
+
"tag_on_failure" => ["_jdbcstaticfailure_server"]
|
157
|
+
}
|
158
|
+
end
|
159
|
+
let(:event) { LogStash::Event.new()}
|
160
|
+
let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
|
161
|
+
let(:prepared_statement) { double("prepared_statement")}
|
162
|
+
|
163
|
+
subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
|
164
|
+
|
165
|
+
before(:each) do
|
166
|
+
allow(local_db).to receive(:prepare).once.and_return(prepared_statement)
|
167
|
+
allow(prepared_statement).to receive(:call).once.and_return(records)
|
168
|
+
end
|
169
|
+
|
170
|
+
it "should be valid" do
|
171
|
+
expect(subject.valid?).to be_truthy
|
172
|
+
end
|
173
|
+
|
174
|
+
it "should have no formatted_errors" do
|
175
|
+
expect(subject.formatted_errors).to eq("")
|
176
|
+
end
|
177
|
+
|
178
|
+
it "should enhance an event" do
|
179
|
+
event.set("ip", "20.20")
|
180
|
+
subject.prepare(local_db)
|
181
|
+
subject.enhance(local_db, event)
|
182
|
+
expect(event.get("tags")).to be_nil
|
183
|
+
expect(event.get("server")).to eq(records)
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
describe "lookup operations with prepared statement multiple parameters" do
|
188
|
+
let(:local_db) { double("local_db") }
|
189
|
+
let(:lookup_hash) do
|
190
|
+
{
|
191
|
+
"query" => "select * from servers WHERE ip LIKE ? AND os LIKE ?",
|
192
|
+
"prepared_parameters" => ["%%{[ip]}", "os"],
|
193
|
+
"target" => "server",
|
194
|
+
"tag_on_failure" => ["_jdbcstaticfailure_server"]
|
195
|
+
}
|
196
|
+
end
|
197
|
+
let(:event) { LogStash::Event.new()}
|
198
|
+
let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
|
199
|
+
let(:prepared_statement) { double("prepared_statement")}
|
200
|
+
|
201
|
+
subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
|
202
|
+
|
203
|
+
before(:each) do
|
204
|
+
allow(local_db).to receive(:prepare).once.and_return(prepared_statement)
|
205
|
+
allow(prepared_statement).to receive(:call).once.and_return(records)
|
206
|
+
end
|
207
|
+
|
208
|
+
it "should be valid" do
|
209
|
+
expect(subject.valid?).to be_truthy
|
210
|
+
end
|
211
|
+
|
212
|
+
it "should have no formatted_errors" do
|
213
|
+
expect(subject.formatted_errors).to eq("")
|
214
|
+
end
|
215
|
+
|
216
|
+
it "should enhance an event" do
|
217
|
+
event.set("ip", "20.20")
|
218
|
+
event.set("os", "MacOS")
|
219
|
+
subject.prepare(local_db)
|
220
|
+
subject.enhance(local_db, event)
|
221
|
+
expect(event.get("tags")).to be_nil
|
222
|
+
expect(event.get("server")).to eq(records)
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
226
|
+
describe "lookup operations with badly configured prepared statement" do
|
227
|
+
let(:local_db) { double("local_db") }
|
228
|
+
let(:lookup_hash) do
|
229
|
+
{
|
230
|
+
"query" => "select * from servers WHERE ip LIKE ? AND os LIKE ?",
|
231
|
+
"prepared_parameters" => ["%%{[ip]}"],
|
232
|
+
"target" => "server",
|
233
|
+
"tag_on_failure" => ["_jdbcstaticfailure_server"]
|
234
|
+
}
|
235
|
+
end
|
236
|
+
let(:event) { LogStash::Event.new()}
|
237
|
+
let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
|
238
|
+
let(:prepared_statement) { double("prepared_statement")}
|
239
|
+
|
240
|
+
subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
|
241
|
+
|
242
|
+
before(:each) do
|
243
|
+
allow(local_db).to receive(:prepare).once.and_return(prepared_statement)
|
244
|
+
allow(prepared_statement).to receive(:call).once.and_return(records)
|
245
|
+
end
|
246
|
+
|
247
|
+
it "must not be valid" do
|
248
|
+
expect(subject.valid?).to be_falsey
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end end end
|
253
|
+
|
@@ -0,0 +1,67 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/util/password"
|
4
|
+
require "logstash/filters/jdbc/read_only_database"
|
5
|
+
|
6
|
+
module LogStash module Filters module Jdbc
|
7
|
+
describe ReadOnlyDatabase do
|
8
|
+
let(:db) { Sequel.connect('mock://mydb') }
|
9
|
+
let(:connection_string) { "mock://mydb" }
|
10
|
+
let(:driver_class) { "org.apache.derby.jdbc.EmbeddedDriver" }
|
11
|
+
let(:driver_library) { nil }
|
12
|
+
subject(:read_only_db) { described_class.create(connection_string, driver_class, driver_library) }
|
13
|
+
|
14
|
+
describe "basic operations" do
|
15
|
+
describe "initializing" do
|
16
|
+
it "tests the connection with defaults" do
|
17
|
+
expect(Sequel::JDBC).to receive(:load_driver).once.with(driver_class)
|
18
|
+
expect(Sequel).to receive(:connect).once.with(connection_string, {:test => true})
|
19
|
+
expect(read_only_db.empty_record_set).to eq([])
|
20
|
+
end
|
21
|
+
|
22
|
+
it "tests the connection with fully specified arguments" do
|
23
|
+
connection_str = "a connection string"
|
24
|
+
user = "a user"
|
25
|
+
password = Util::Password.new("secret")
|
26
|
+
expect(Sequel::JDBC).to receive(:load_driver).once.with("a driver class")
|
27
|
+
expect(Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password => password.value, :test => true}).and_return(db)
|
28
|
+
described_class.create(connection_str, "a driver class", nil, user, password)
|
29
|
+
end
|
30
|
+
|
31
|
+
it "connects with defaults" do
|
32
|
+
expect(Sequel::JDBC).to receive(:load_driver).once.with(driver_class)
|
33
|
+
expect(Sequel).to receive(:connect).once.with(connection_string, {:test => true}).and_return(db)
|
34
|
+
expect(Sequel).to receive(:connect).once.with(connection_string, {}).and_return(db)
|
35
|
+
expect(read_only_db.connected?).to be_falsey
|
36
|
+
read_only_db.connect("a caller specific error message")
|
37
|
+
expect(read_only_db.connected?).to be_truthy
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
describe "methods" do
|
42
|
+
let(:dataset) { double("Sequel::Dataset") }
|
43
|
+
|
44
|
+
before(:each) do
|
45
|
+
allow(Sequel::JDBC).to receive(:load_driver)
|
46
|
+
allow(Sequel).to receive(:connect).thrice.and_return(db)
|
47
|
+
allow(db).to receive(:[]).and_return(dataset)
|
48
|
+
read_only_db.connect("a caller specific error message")
|
49
|
+
end
|
50
|
+
|
51
|
+
after(:each) do
|
52
|
+
read_only_db.disconnect("a caller specific error message")
|
53
|
+
end
|
54
|
+
|
55
|
+
it "the count method gets a count from the dataset" do
|
56
|
+
expect(dataset).to receive(:count).and_return(0)
|
57
|
+
read_only_db.count("select * from table")
|
58
|
+
end
|
59
|
+
|
60
|
+
it "the query method gets all records from the dataset" do
|
61
|
+
expect(dataset).to receive(:all).and_return(read_only_db.empty_record_set)
|
62
|
+
read_only_db.query("select * from table")
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end end end
|