logstash-filter-jdbc_static 1.0.7 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 44a9dc021efa08eb1c02d143f9c3b16e89b1dab5be971a1c1bd18c047f1b9dcb
4
- data.tar.gz: ff306839f602b625350173005b9780504b9ec6f3bfba2abe5cad4f92a654ab4a
3
+ metadata.gz: 0d482ab05b8b7e9904183077976c33624b4cdd396d5701cd4db5d501772e7f33
4
+ data.tar.gz: f0168f06b31172b98c843e788ffbba5d97f9904726244c293de280f63effa56b
5
5
  SHA512:
6
- metadata.gz: 2eb10e33bcbd0db98bdbad8ad8575cf2ed631a606285ea2ac489588d033b7c5179e89b2b5133003ac0dad979fa374e303fe55bf6ca5f70f1906b3487f182094e
7
- data.tar.gz: 8fb1524a50778571c1fc7bfd8811de682eecdf842cacc0e1c467f42d7296297c6a92b2fe34785fe9d23470d18a0d54e447c212a36f2ff19a0dade3122ab1bb7e
6
+ metadata.gz: 723f1e3cd20a3cbe703bd9894f8036237ce22f39702cd6e4a750c9d5c289fad9c8249243ae51f13b4cc1400ae0171dea534c791e60f2694bd917b2e69f48220a
7
+ data.tar.gz: 4c7a55addf3ed97394ec533e0c86707b7151aee333ced400d5e546ac5482ef6e8e68bcfc658a04a6faf021e0a64249db801d5b351677505cf118b52822888ced
@@ -1,3 +1,6 @@
1
+ ## 1.1.0
2
+ - Added prepared statement support in local lookups [#53](https://github.com/logstash-plugins/logstash-filter-jdbc_static/pull/53)
3
+
1
4
  ## 1.0.7
2
5
  - Fixed issue with driver verification using Java 11 [#51](https://github.com/logstash-plugins/logstash-filter-jdbc_static/pull/51)
3
6
 
@@ -101,21 +101,21 @@ filter {
101
101
  local_lookups => [ <3>
102
102
  {
103
103
  id => "local-servers"
104
- query => "select descr as description from servers WHERE ip = :ip"
104
+ query => "SELECT descr as description FROM servers WHERE ip = :ip"
105
105
  parameters => {ip => "[from_ip]"}
106
106
  target => "server"
107
107
  },
108
108
  {
109
109
  id => "local-users"
110
- query => "select firstname, lastname from users WHERE userid = :id"
111
- parameters => {id => "[loggedin_userid]"}
112
- target => "user" <4>
110
+ query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?"
111
+ prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4>
112
+ target => "user" <5>
113
113
  }
114
114
  ]
115
115
  # using add_field here to add & rename values to the event root
116
- add_field => { server_name => "%{[server][0][description]}" }
117
- add_field => { user_firstname => "%{[user][0][firstname]}" } <5>
118
- add_field => { user_lastname => "%{[user][0][lastname]}" } <5>
116
+ add_field => { server_name => "%{[server][0][description]}" } <6>
117
+ add_field => { user_firstname => "%{[user][0][firstname]}" }
118
+ add_field => { user_lastname => "%{[user][0][lastname]}" }
119
119
  remove_field => ["server", "user"]
120
120
  staging_directory => "/tmp/logstash/jdbc_static/import_data"
121
121
  loader_schedule => "* */2 * * *" # run loaders every 2 hours
@@ -134,9 +134,11 @@ structure. The column names and types should match the external database.
134
134
  The order of table definitions is significant and should match that of the loader queries.
135
135
  See <<plugins-{type}s-{plugin}-object_order>>.
136
136
  <3> Performs lookup queries on the local database to enrich the events.
137
- <4> Specifies the event field that will store the looked-up data. If the lookup
137
+ <4> Local lookup queries can also use prepared statements where the parameters
138
+ follow the positional ordering.
139
+ <5> Specifies the event field that will store the looked-up data. If the lookup
138
140
  returns multiple columns, the data is stored as a JSON object within the field.
139
- <5> Takes data from the JSON object and stores it in top-level event fields for
141
+ <6> Takes data from the JSON object and stores it in top-level event fields for
140
142
  easier analysis in Kibana.
141
143
 
142
144
  Here's a full example:
@@ -546,7 +548,9 @@ default id is used instead.
546
548
  query::
547
549
  A SQL SELECT statement that is executed to achieve the lookup. To use
548
550
  parameters, use named parameter syntax, for example
549
- `"SELECT * FROM MYTABLE WHERE ID = :id"`.
551
+ `"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign
552
+ can be used as a prepared statement parameter, in which case
553
+ the `prepared_parameters` array is used to populate the values
550
554
 
551
555
  parameters::
552
556
  A key/value Hash or dictionary. The key (LHS) is the text that is
@@ -563,6 +567,16 @@ an id and a location, and you have a table of sensors that have a
563
567
  column of `id-loc_id`. In this case your parameter hash would look
564
568
  like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`.
565
569
 
570
+ prepared_parameters::
571
+ An Array, where the position is related to the position of the `?` in
572
+ the query syntax. The values of array follow the same semantic of `parameters`.
573
+ If `prepared_parameters` is valorized the filter is forced to use JDBC's
574
+ prepared statement to query the local database.
575
+ Prepared statements provides two benefits: one on the performance side, because
576
+ avoid the DBMS to parse and compile the SQL expression for every call;
577
+ the other benefit is on the security side, using prepared statements
578
+ avoid SQL-injection attacks based on query string concatenation.
579
+
566
580
  target::
567
581
  An optional name for the field that will receive the looked-up data.
568
582
  If you omit this setting then the `id` setting (or the default id) is
@@ -63,6 +63,8 @@ module LogStash module Filters module Jdbc
63
63
  @valid = false
64
64
  @option_errors = []
65
65
  @default_result = nil
66
+ @prepared_statement = nil
67
+ @symbol_parameters = nil
66
68
  parse_options
67
69
  end
68
70
 
@@ -79,8 +81,11 @@ module LogStash module Filters module Jdbc
79
81
  end
80
82
 
81
83
  def enhance(local, event)
82
- result = fetch(local, event) # should return a LookupResult
83
-
84
+ if @prepared_statement
85
+ result = call_prepared(local, event)
86
+ else
87
+ result = fetch(local, event) # should return a LookupResult
88
+ end
84
89
  if result.failed? || result.parameters_invalid?
85
90
  tag_failure(event)
86
91
  end
@@ -98,6 +103,17 @@ module LogStash module Filters module Jdbc
98
103
  end
99
104
  end
100
105
 
106
+ def use_prepared_statement?
107
+ @prepared_parameters && !@prepared_parameters.empty?
108
+ end
109
+
110
+ def prepare(local)
111
+ hash = {}
112
+ @prepared_parameters.each_with_index { |v, i| hash[:"$p#{i}"] = v }
113
+ @prepared_param_placeholder_map = hash
114
+ @prepared_statement = local.prepare(query, hash.keys)
115
+ end
116
+
101
117
  private
102
118
 
103
119
  def tag_failure(event)
@@ -139,6 +155,33 @@ module LogStash module Filters module Jdbc
139
155
  result
140
156
  end
141
157
 
158
+ def call_prepared(local, event)
159
+ result = LookupResult.new()
160
+ if @parameters_specified
161
+ params = prepare_parameters_from_event(event, result)
162
+ if result.parameters_invalid?
163
+ logger.warn? && logger.warn("Parameter field not found in event", :lookup_id => @id, :invalid_parameters => result.invalid_parameters)
164
+ return result
165
+ end
166
+ else
167
+ params = {}
168
+ end
169
+ begin
170
+ logger.debug? && logger.debug("Executing Jdbc query", :lookup_id => @id, :statement => query, :parameters => params)
171
+ @prepared_statement.call(params).each do |row|
172
+ stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
173
+ result.push(stringified)
174
+ end
175
+ rescue ::Sequel::Error => e
176
+ # all sequel errors are a subclass of this, let all other standard or runtime errors bubble up
177
+ result.failed!
178
+ logger.warn? && logger.warn("Exception when executing Jdbc query", :lookup_id => @id, :exception => e.message, :backtrace => e.backtrace.take(8))
179
+ end
180
+ # if either of: no records or a Sequel exception occurs the payload is
181
+ # empty and the default can be substituted later.
182
+ result
183
+ end
184
+
142
185
  def process_event(event, result)
143
186
  # use deep clone here so other filter function don't taint the payload by reference
144
187
  event.set(@target, ::LogStash::Util.deep_clone(result.payload))
@@ -162,17 +205,34 @@ module LogStash module Filters module Jdbc
162
205
  @option_errors << "The options for '#{@id}' must include a 'query' string"
163
206
  end
164
207
 
165
- @parameters = @options["parameters"]
166
- @parameters_specified = false
167
- if @parameters
168
- if !@parameters.is_a?(Hash)
169
- @option_errors << "The 'parameters' option for '#{@id}' must be a Hash"
170
- else
171
- # this is done once per lookup at start, i.e. Sprintfier.new et.al is done once.
172
- @symbol_parameters = @parameters.inject({}) {|hash,(k,v)| hash[k.to_sym] = sprintf_or_get(v) ; hash }
173
- # the user might specify an empty hash parameters => {}
174
- # maybe due to an unparameterised query
175
- @parameters_specified = !@symbol_parameters.empty?
208
+ if @options["parameters"] && @options["prepared_parameters"]
209
+ @option_errors << "Can't specify 'parameters' and 'prepared_parameters' in the same lookup"
210
+ else
211
+ @parameters = @options["parameters"]
212
+ @prepared_parameters = @options["prepared_parameters"]
213
+ @parameters_specified = false
214
+ if @parameters
215
+ if !@parameters.is_a?(Hash)
216
+ @option_errors << "The 'parameters' option for '#{@id}' must be a Hash"
217
+ else
218
+ # this is done once per lookup at start, i.e. Sprintfier.new et.al is done once.
219
+ @symbol_parameters = @parameters.inject({}) {|hash,(k,v)| hash[k.to_sym] = sprintf_or_get(v) ; hash }
220
+ # the user might specify an empty hash parameters => {}
221
+ # maybe due to an unparameterised query
222
+ @parameters_specified = !@symbol_parameters.empty?
223
+ end
224
+ elsif @prepared_parameters
225
+ if !@prepared_parameters.is_a?(Array)
226
+ @option_errors << "The 'prepared_parameters' option for '#{@id}' must be an Array"
227
+ elsif @query.count("?") != @prepared_parameters.size
228
+ @option_errors << "The 'prepared_parameters' option for '#{@id}' doesn't match count with query's placeholder"
229
+ else
230
+ #prepare the map @symbol_parameters :n => sprintf_or_get
231
+ hash = {}
232
+ @prepared_parameters.each_with_index {|v,i| hash[:"p#{i}"] = sprintf_or_get(v)}
233
+ @symbol_parameters = hash
234
+ @parameters_specified = !@prepared_parameters.empty?
235
+ end
176
236
  end
177
237
  end
178
238
 
@@ -38,6 +38,8 @@ module LogStash module Filters module Jdbc
38
38
  "lookup_jdbc_driver_class",
39
39
  "lookup_jdbc_driver_library").compact)
40
40
  @local.connect(CONNECTION_ERROR_MSG)
41
+
42
+ create_prepared_statements_for_lookups
41
43
  end
42
44
  end
43
45
 
@@ -60,6 +62,14 @@ module LogStash module Filters module Jdbc
60
62
 
61
63
  private
62
64
 
65
+ def create_prepared_statements_for_lookups()
66
+ @lookups.each do |lookup|
67
+ if lookup.use_prepared_statement?
68
+ lookup.prepare(@local)
69
+ end
70
+ end
71
+ end
72
+
63
73
  def validate_lookups(lookups_errors = [])
64
74
  ids = Hash.new(0)
65
75
  errors = []
@@ -27,6 +27,13 @@ module LogStash module Filters module Jdbc
27
27
  @rwlock.readLock().unlock()
28
28
  end
29
29
 
30
+ def prepare(statement, parameters)
31
+ @rwlock.readLock().lock()
32
+ @db[statement, parameters].prepare(:select, @id)
33
+ ensure
34
+ @rwlock.readLock().unlock()
35
+ end
36
+
30
37
  def build_db_object(db_object)
31
38
  begin
32
39
  @rwlock.writeLock().lock()
@@ -60,14 +60,19 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
60
60
  # For example:
61
61
  # local_lookups => [
62
62
  # {
63
- # "query" => "select * from country WHERE code = :code",
63
+ # "query" => "SELECT * FROM country WHERE code = :code",
64
64
  # "parameters" => {"code" => "country_code"}
65
65
  # "target" => "country_details"
66
66
  # },
67
67
  # {
68
- # "query" => "select ip, name from servers WHERE ip LIKE :ip",
68
+ # "query" => "SELECT ip, name FROM servers WHERE ip LIKE :ip",
69
69
  # "parameters" => {"ip" => "%{[response][ip]}%"}
70
70
  # "target" => "servers"
71
+ # },
72
+ # {
73
+ # "query" => "SELECT ip, name FROM servers WHERE ip = ?",
74
+ # "prepared_parameters" => ["from_ip"]
75
+ # "target" => "servers"
71
76
  # }
72
77
  # ]
73
78
  config :local_lookups, :required => true, :validate => [LogStash::Filters::Jdbc::LookupProcessor]
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-filter-jdbc_static'
3
- s.version = '1.0.7'
3
+ s.version = '1.1.0'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "This filter executes a SQL query to fetch a SQL query result, store it locally then use a second SQL query to update an event."
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -83,6 +83,40 @@ module LogStash module Filters
83
83
  end
84
84
  end
85
85
 
86
+ context "under normal conditions with prepared statement" do
87
+ let(:lookup_statement) { "SELECT * FROM servers WHERE ip LIKE ?" }
88
+ let(:settings) do
89
+ {
90
+ "jdbc_user" => ENV['USER'],
91
+ "jdbc_driver_class" => "org.postgresql.Driver",
92
+ "jdbc_driver_library" => "/usr/share/logstash/postgresql.jar",
93
+ "staging_directory" => temp_import_path_plugin,
94
+ "jdbc_connection_string" => jdbc_connection_string,
95
+ "loaders" => [
96
+ {
97
+ "id" =>"servers",
98
+ "query" => loader_statement,
99
+ "local_table" => "servers"
100
+ }
101
+ ],
102
+ "local_db_objects" => local_db_objects,
103
+ "local_lookups" => [
104
+ {
105
+ "query" => lookup_statement,
106
+ "prepared_parameters" => [parameters_rhs],
107
+ "target" => "server"
108
+ }
109
+ ]
110
+ }
111
+ end
112
+
113
+ it "enhances an event" do
114
+ plugin.register
115
+ plugin.filter(event)
116
+ expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
117
+ end
118
+ end
119
+
86
120
  context "under normal conditions when index_columns is not specified" do
87
121
  let(:local_db_objects) do
88
122
  [
@@ -44,6 +44,27 @@ module LogStash module Filters module Jdbc
44
44
  result = described_class.find_validation_errors([lookup_hash])
45
45
  expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
46
46
  end
47
+
48
+ it "parameters and prepared_parameters are defined at same time" do
49
+ lookup_hash = {
50
+ "query" => "SELECT * FROM table WHERE ip=?",
51
+ "parameters" => {"ip" => "%%{[ip]}"},
52
+ "prepared_parameters" => ["%%{[ip]}"],
53
+ "target" => "server"
54
+ }
55
+ result = described_class.find_validation_errors([lookup_hash])
56
+ expect(result).to eq("Can't specify 'parameters' and 'prepared_parameters' in the same lookup")
57
+ end
58
+
59
+ it "prepared_parameters count doesn't match the number of '?' in the query" do
60
+ lookup_hash = {
61
+ "query" => "SELECT * FROM table WHERE ip=? AND host=?",
62
+ "prepared_parameters" => ["%%{[ip]}"],
63
+ "target" => "server"
64
+ }
65
+ result = described_class.find_validation_errors([lookup_hash])
66
+ expect(result).to eq("The 'prepared_parameters' option for 'lookup-1' doesn't match count with query's placeholder")
67
+ end
47
68
  end
48
69
 
49
70
  context "when supplied with a valid arg" do
@@ -124,6 +145,109 @@ module LogStash module Filters module Jdbc
124
145
  expect(event.get("server")).to eq(records)
125
146
  end
126
147
  end
148
+
149
+ describe "lookup operations with prepared statement" do
150
+ let(:local_db) { double("local_db") }
151
+ let(:lookup_hash) do
152
+ {
153
+ "query" => "select * from servers WHERE ip LIKE ?",
154
+ "prepared_parameters" => ["%%{[ip]}"],
155
+ "target" => "server",
156
+ "tag_on_failure" => ["_jdbcstaticfailure_server"]
157
+ }
158
+ end
159
+ let(:event) { LogStash::Event.new()}
160
+ let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
161
+ let(:prepared_statement) { double("prepared_statement")}
162
+
163
+ subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
164
+
165
+ before(:each) do
166
+ allow(local_db).to receive(:prepare).once.and_return(prepared_statement)
167
+ allow(prepared_statement).to receive(:call).once.and_return(records)
168
+ end
169
+
170
+ it "should be valid" do
171
+ expect(subject.valid?).to be_truthy
172
+ end
173
+
174
+ it "should have no formatted_errors" do
175
+ expect(subject.formatted_errors).to eq("")
176
+ end
177
+
178
+ it "should enhance an event" do
179
+ event.set("ip", "20.20")
180
+ subject.prepare(local_db)
181
+ subject.enhance(local_db, event)
182
+ expect(event.get("tags")).to be_nil
183
+ expect(event.get("server")).to eq(records)
184
+ end
185
+ end
186
+
187
+ describe "lookup operations with prepared statement multiple parameters" do
188
+ let(:local_db) { double("local_db") }
189
+ let(:lookup_hash) do
190
+ {
191
+ "query" => "select * from servers WHERE ip LIKE ? AND os LIKE ?",
192
+ "prepared_parameters" => ["%%{[ip]}", "os"],
193
+ "target" => "server",
194
+ "tag_on_failure" => ["_jdbcstaticfailure_server"]
195
+ }
196
+ end
197
+ let(:event) { LogStash::Event.new()}
198
+ let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
199
+ let(:prepared_statement) { double("prepared_statement")}
200
+
201
+ subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
202
+
203
+ before(:each) do
204
+ allow(local_db).to receive(:prepare).once.and_return(prepared_statement)
205
+ allow(prepared_statement).to receive(:call).once.and_return(records)
206
+ end
207
+
208
+ it "should be valid" do
209
+ expect(subject.valid?).to be_truthy
210
+ end
211
+
212
+ it "should have no formatted_errors" do
213
+ expect(subject.formatted_errors).to eq("")
214
+ end
215
+
216
+ it "should enhance an event" do
217
+ event.set("ip", "20.20")
218
+ event.set("os", "MacOS")
219
+ subject.prepare(local_db)
220
+ subject.enhance(local_db, event)
221
+ expect(event.get("tags")).to be_nil
222
+ expect(event.get("server")).to eq(records)
223
+ end
224
+ end
225
+
226
+ describe "lookup operations with badly configured prepared statement" do
227
+ let(:local_db) { double("local_db") }
228
+ let(:lookup_hash) do
229
+ {
230
+ "query" => "select * from servers WHERE ip LIKE ? AND os LIKE ?",
231
+ "prepared_parameters" => ["%%{[ip]}"],
232
+ "target" => "server",
233
+ "tag_on_failure" => ["_jdbcstaticfailure_server"]
234
+ }
235
+ end
236
+ let(:event) { LogStash::Event.new()}
237
+ let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
238
+ let(:prepared_statement) { double("prepared_statement")}
239
+
240
+ subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
241
+
242
+ before(:each) do
243
+ allow(local_db).to receive(:prepare).once.and_return(prepared_statement)
244
+ allow(prepared_statement).to receive(:call).once.and_return(records)
245
+ end
246
+
247
+ it "must not be valid" do
248
+ expect(subject.valid?).to be_falsey
249
+ end
250
+ end
127
251
  end
128
252
  end end end
129
253
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-jdbc_static
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.7
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-10-29 00:00:00.000000000 Z
11
+ date: 2019-11-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement