logstash-filter-jdbc_static 1.0.7 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 44a9dc021efa08eb1c02d143f9c3b16e89b1dab5be971a1c1bd18c047f1b9dcb
4
- data.tar.gz: ff306839f602b625350173005b9780504b9ec6f3bfba2abe5cad4f92a654ab4a
3
+ metadata.gz: 0d482ab05b8b7e9904183077976c33624b4cdd396d5701cd4db5d501772e7f33
4
+ data.tar.gz: f0168f06b31172b98c843e788ffbba5d97f9904726244c293de280f63effa56b
5
5
  SHA512:
6
- metadata.gz: 2eb10e33bcbd0db98bdbad8ad8575cf2ed631a606285ea2ac489588d033b7c5179e89b2b5133003ac0dad979fa374e303fe55bf6ca5f70f1906b3487f182094e
7
- data.tar.gz: 8fb1524a50778571c1fc7bfd8811de682eecdf842cacc0e1c467f42d7296297c6a92b2fe34785fe9d23470d18a0d54e447c212a36f2ff19a0dade3122ab1bb7e
6
+ metadata.gz: 723f1e3cd20a3cbe703bd9894f8036237ce22f39702cd6e4a750c9d5c289fad9c8249243ae51f13b4cc1400ae0171dea534c791e60f2694bd917b2e69f48220a
7
+ data.tar.gz: 4c7a55addf3ed97394ec533e0c86707b7151aee333ced400d5e546ac5482ef6e8e68bcfc658a04a6faf021e0a64249db801d5b351677505cf118b52822888ced
@@ -1,3 +1,6 @@
1
+ ## 1.1.0
2
+ - Added prepared statement support in local lookups [#53](https://github.com/logstash-plugins/logstash-filter-jdbc_static/pull/53)
3
+
1
4
  ## 1.0.7
2
5
  - Fixed issue with driver verification using Java 11 [#51](https://github.com/logstash-plugins/logstash-filter-jdbc_static/pull/51)
3
6
 
@@ -101,21 +101,21 @@ filter {
101
101
  local_lookups => [ <3>
102
102
  {
103
103
  id => "local-servers"
104
- query => "select descr as description from servers WHERE ip = :ip"
104
+ query => "SELECT descr as description FROM servers WHERE ip = :ip"
105
105
  parameters => {ip => "[from_ip]"}
106
106
  target => "server"
107
107
  },
108
108
  {
109
109
  id => "local-users"
110
- query => "select firstname, lastname from users WHERE userid = :id"
111
- parameters => {id => "[loggedin_userid]"}
112
- target => "user" <4>
110
+ query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?"
111
+ prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4>
112
+ target => "user" <5>
113
113
  }
114
114
  ]
115
115
  # using add_field here to add & rename values to the event root
116
- add_field => { server_name => "%{[server][0][description]}" }
117
- add_field => { user_firstname => "%{[user][0][firstname]}" } <5>
118
- add_field => { user_lastname => "%{[user][0][lastname]}" } <5>
116
+ add_field => { server_name => "%{[server][0][description]}" } <6>
117
+ add_field => { user_firstname => "%{[user][0][firstname]}" }
118
+ add_field => { user_lastname => "%{[user][0][lastname]}" }
119
119
  remove_field => ["server", "user"]
120
120
  staging_directory => "/tmp/logstash/jdbc_static/import_data"
121
121
  loader_schedule => "* */2 * * *" # run loaders every 2 hours
@@ -134,9 +134,11 @@ structure. The column names and types should match the external database.
134
134
  The order of table definitions is significant and should match that of the loader queries.
135
135
  See <<plugins-{type}s-{plugin}-object_order>>.
136
136
  <3> Performs lookup queries on the local database to enrich the events.
137
- <4> Specifies the event field that will store the looked-up data. If the lookup
137
+ <4> Local lookup queries can also use prepared statements where the parameters
138
+ follow the positional ordering.
139
+ <5> Specifies the event field that will store the looked-up data. If the lookup
138
140
  returns multiple columns, the data is stored as a JSON object within the field.
139
- <5> Takes data from the JSON object and stores it in top-level event fields for
141
+ <6> Takes data from the JSON object and stores it in top-level event fields for
140
142
  easier analysis in Kibana.
141
143
 
142
144
  Here's a full example:
@@ -546,7 +548,9 @@ default id is used instead.
546
548
  query::
547
549
  A SQL SELECT statement that is executed to achieve the lookup. To use
548
550
  parameters, use named parameter syntax, for example
549
- `"SELECT * FROM MYTABLE WHERE ID = :id"`.
551
+ `"SELECT * FROM MYTABLE WHERE ID = :id"`. Alternatively, the `?` sign
552
+ can be used as a prepared statement parameter, in which case
553
+ the `prepared_parameters` array is used to populate the values
550
554
 
551
555
  parameters::
552
556
  A key/value Hash or dictionary. The key (LHS) is the text that is
@@ -563,6 +567,16 @@ an id and a location, and you have a table of sensors that have a
563
567
  column of `id-loc_id`. In this case your parameter hash would look
564
568
  like this: `parameters => { "p1" => "%{[id]}-%{[loc_id]}" }`.
565
569
 
570
+ prepared_parameters::
571
+ An Array, where the position is related to the position of the `?` in
572
+ the query syntax. The values of array follow the same semantic of `parameters`.
573
+ If `prepared_parameters` is valorized the filter is forced to use JDBC's
574
+ prepared statement to query the local database.
575
+ Prepared statements provides two benefits: one on the performance side, because
576
+ avoid the DBMS to parse and compile the SQL expression for every call;
577
+ the other benefit is on the security side, using prepared statements
578
+ avoid SQL-injection attacks based on query string concatenation.
579
+
566
580
  target::
567
581
  An optional name for the field that will receive the looked-up data.
568
582
  If you omit this setting then the `id` setting (or the default id) is
@@ -63,6 +63,8 @@ module LogStash module Filters module Jdbc
63
63
  @valid = false
64
64
  @option_errors = []
65
65
  @default_result = nil
66
+ @prepared_statement = nil
67
+ @symbol_parameters = nil
66
68
  parse_options
67
69
  end
68
70
 
@@ -79,8 +81,11 @@ module LogStash module Filters module Jdbc
79
81
  end
80
82
 
81
83
  def enhance(local, event)
82
- result = fetch(local, event) # should return a LookupResult
83
-
84
+ if @prepared_statement
85
+ result = call_prepared(local, event)
86
+ else
87
+ result = fetch(local, event) # should return a LookupResult
88
+ end
84
89
  if result.failed? || result.parameters_invalid?
85
90
  tag_failure(event)
86
91
  end
@@ -98,6 +103,17 @@ module LogStash module Filters module Jdbc
98
103
  end
99
104
  end
100
105
 
106
+ def use_prepared_statement?
107
+ @prepared_parameters && !@prepared_parameters.empty?
108
+ end
109
+
110
+ def prepare(local)
111
+ hash = {}
112
+ @prepared_parameters.each_with_index { |v, i| hash[:"$p#{i}"] = v }
113
+ @prepared_param_placeholder_map = hash
114
+ @prepared_statement = local.prepare(query, hash.keys)
115
+ end
116
+
101
117
  private
102
118
 
103
119
  def tag_failure(event)
@@ -139,6 +155,33 @@ module LogStash module Filters module Jdbc
139
155
  result
140
156
  end
141
157
 
158
+ def call_prepared(local, event)
159
+ result = LookupResult.new()
160
+ if @parameters_specified
161
+ params = prepare_parameters_from_event(event, result)
162
+ if result.parameters_invalid?
163
+ logger.warn? && logger.warn("Parameter field not found in event", :lookup_id => @id, :invalid_parameters => result.invalid_parameters)
164
+ return result
165
+ end
166
+ else
167
+ params = {}
168
+ end
169
+ begin
170
+ logger.debug? && logger.debug("Executing Jdbc query", :lookup_id => @id, :statement => query, :parameters => params)
171
+ @prepared_statement.call(params).each do |row|
172
+ stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
173
+ result.push(stringified)
174
+ end
175
+ rescue ::Sequel::Error => e
176
+ # all sequel errors are a subclass of this, let all other standard or runtime errors bubble up
177
+ result.failed!
178
+ logger.warn? && logger.warn("Exception when executing Jdbc query", :lookup_id => @id, :exception => e.message, :backtrace => e.backtrace.take(8))
179
+ end
180
+ # if either of: no records or a Sequel exception occurs the payload is
181
+ # empty and the default can be substituted later.
182
+ result
183
+ end
184
+
142
185
  def process_event(event, result)
143
186
  # use deep clone here so other filter function don't taint the payload by reference
144
187
  event.set(@target, ::LogStash::Util.deep_clone(result.payload))
@@ -162,17 +205,34 @@ module LogStash module Filters module Jdbc
162
205
  @option_errors << "The options for '#{@id}' must include a 'query' string"
163
206
  end
164
207
 
165
- @parameters = @options["parameters"]
166
- @parameters_specified = false
167
- if @parameters
168
- if !@parameters.is_a?(Hash)
169
- @option_errors << "The 'parameters' option for '#{@id}' must be a Hash"
170
- else
171
- # this is done once per lookup at start, i.e. Sprintfier.new et.al is done once.
172
- @symbol_parameters = @parameters.inject({}) {|hash,(k,v)| hash[k.to_sym] = sprintf_or_get(v) ; hash }
173
- # the user might specify an empty hash parameters => {}
174
- # maybe due to an unparameterised query
175
- @parameters_specified = !@symbol_parameters.empty?
208
+ if @options["parameters"] && @options["prepared_parameters"]
209
+ @option_errors << "Can't specify 'parameters' and 'prepared_parameters' in the same lookup"
210
+ else
211
+ @parameters = @options["parameters"]
212
+ @prepared_parameters = @options["prepared_parameters"]
213
+ @parameters_specified = false
214
+ if @parameters
215
+ if !@parameters.is_a?(Hash)
216
+ @option_errors << "The 'parameters' option for '#{@id}' must be a Hash"
217
+ else
218
+ # this is done once per lookup at start, i.e. Sprintfier.new et.al is done once.
219
+ @symbol_parameters = @parameters.inject({}) {|hash,(k,v)| hash[k.to_sym] = sprintf_or_get(v) ; hash }
220
+ # the user might specify an empty hash parameters => {}
221
+ # maybe due to an unparameterised query
222
+ @parameters_specified = !@symbol_parameters.empty?
223
+ end
224
+ elsif @prepared_parameters
225
+ if !@prepared_parameters.is_a?(Array)
226
+ @option_errors << "The 'prepared_parameters' option for '#{@id}' must be an Array"
227
+ elsif @query.count("?") != @prepared_parameters.size
228
+ @option_errors << "The 'prepared_parameters' option for '#{@id}' doesn't match count with query's placeholder"
229
+ else
230
+ #prepare the map @symbol_parameters :n => sprintf_or_get
231
+ hash = {}
232
+ @prepared_parameters.each_with_index {|v,i| hash[:"p#{i}"] = sprintf_or_get(v)}
233
+ @symbol_parameters = hash
234
+ @parameters_specified = !@prepared_parameters.empty?
235
+ end
176
236
  end
177
237
  end
178
238
 
@@ -38,6 +38,8 @@ module LogStash module Filters module Jdbc
38
38
  "lookup_jdbc_driver_class",
39
39
  "lookup_jdbc_driver_library").compact)
40
40
  @local.connect(CONNECTION_ERROR_MSG)
41
+
42
+ create_prepared_statements_for_lookups
41
43
  end
42
44
  end
43
45
 
@@ -60,6 +62,14 @@ module LogStash module Filters module Jdbc
60
62
 
61
63
  private
62
64
 
65
+ def create_prepared_statements_for_lookups()
66
+ @lookups.each do |lookup|
67
+ if lookup.use_prepared_statement?
68
+ lookup.prepare(@local)
69
+ end
70
+ end
71
+ end
72
+
63
73
  def validate_lookups(lookups_errors = [])
64
74
  ids = Hash.new(0)
65
75
  errors = []
@@ -27,6 +27,13 @@ module LogStash module Filters module Jdbc
27
27
  @rwlock.readLock().unlock()
28
28
  end
29
29
 
30
+ def prepare(statement, parameters)
31
+ @rwlock.readLock().lock()
32
+ @db[statement, parameters].prepare(:select, @id)
33
+ ensure
34
+ @rwlock.readLock().unlock()
35
+ end
36
+
30
37
  def build_db_object(db_object)
31
38
  begin
32
39
  @rwlock.writeLock().lock()
@@ -60,14 +60,19 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
60
60
  # For example:
61
61
  # local_lookups => [
62
62
  # {
63
- # "query" => "select * from country WHERE code = :code",
63
+ # "query" => "SELECT * FROM country WHERE code = :code",
64
64
  # "parameters" => {"code" => "country_code"}
65
65
  # "target" => "country_details"
66
66
  # },
67
67
  # {
68
- # "query" => "select ip, name from servers WHERE ip LIKE :ip",
68
+ # "query" => "SELECT ip, name FROM servers WHERE ip LIKE :ip",
69
69
  # "parameters" => {"ip" => "%{[response][ip]}%"}
70
70
  # "target" => "servers"
71
+ # },
72
+ # {
73
+ # "query" => "SELECT ip, name FROM servers WHERE ip = ?",
74
+ # "prepared_parameters" => ["from_ip"]
75
+ # "target" => "servers"
71
76
  # }
72
77
  # ]
73
78
  config :local_lookups, :required => true, :validate => [LogStash::Filters::Jdbc::LookupProcessor]
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-filter-jdbc_static'
3
- s.version = '1.0.7'
3
+ s.version = '1.1.0'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "This filter executes a SQL query to fetch a SQL query result, store it locally then use a second SQL query to update an event."
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -83,6 +83,40 @@ module LogStash module Filters
83
83
  end
84
84
  end
85
85
 
86
+ context "under normal conditions with prepared statement" do
87
+ let(:lookup_statement) { "SELECT * FROM servers WHERE ip LIKE ?" }
88
+ let(:settings) do
89
+ {
90
+ "jdbc_user" => ENV['USER'],
91
+ "jdbc_driver_class" => "org.postgresql.Driver",
92
+ "jdbc_driver_library" => "/usr/share/logstash/postgresql.jar",
93
+ "staging_directory" => temp_import_path_plugin,
94
+ "jdbc_connection_string" => jdbc_connection_string,
95
+ "loaders" => [
96
+ {
97
+ "id" =>"servers",
98
+ "query" => loader_statement,
99
+ "local_table" => "servers"
100
+ }
101
+ ],
102
+ "local_db_objects" => local_db_objects,
103
+ "local_lookups" => [
104
+ {
105
+ "query" => lookup_statement,
106
+ "prepared_parameters" => [parameters_rhs],
107
+ "target" => "server"
108
+ }
109
+ ]
110
+ }
111
+ end
112
+
113
+ it "enhances an event" do
114
+ plugin.register
115
+ plugin.filter(event)
116
+ expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
117
+ end
118
+ end
119
+
86
120
  context "under normal conditions when index_columns is not specified" do
87
121
  let(:local_db_objects) do
88
122
  [
@@ -44,6 +44,27 @@ module LogStash module Filters module Jdbc
44
44
  result = described_class.find_validation_errors([lookup_hash])
45
45
  expect(result).to eq("The 'parameters' option for 'lookup-1' must be a Hash")
46
46
  end
47
+
48
+ it "parameters and prepared_parameters are defined at same time" do
49
+ lookup_hash = {
50
+ "query" => "SELECT * FROM table WHERE ip=?",
51
+ "parameters" => {"ip" => "%%{[ip]}"},
52
+ "prepared_parameters" => ["%%{[ip]}"],
53
+ "target" => "server"
54
+ }
55
+ result = described_class.find_validation_errors([lookup_hash])
56
+ expect(result).to eq("Can't specify 'parameters' and 'prepared_parameters' in the same lookup")
57
+ end
58
+
59
+ it "prepared_parameters count doesn't match the number of '?' in the query" do
60
+ lookup_hash = {
61
+ "query" => "SELECT * FROM table WHERE ip=? AND host=?",
62
+ "prepared_parameters" => ["%%{[ip]}"],
63
+ "target" => "server"
64
+ }
65
+ result = described_class.find_validation_errors([lookup_hash])
66
+ expect(result).to eq("The 'prepared_parameters' option for 'lookup-1' doesn't match count with query's placeholder")
67
+ end
47
68
  end
48
69
 
49
70
  context "when supplied with a valid arg" do
@@ -124,6 +145,109 @@ module LogStash module Filters module Jdbc
124
145
  expect(event.get("server")).to eq(records)
125
146
  end
126
147
  end
148
+
149
+ describe "lookup operations with prepared statement" do
150
+ let(:local_db) { double("local_db") }
151
+ let(:lookup_hash) do
152
+ {
153
+ "query" => "select * from servers WHERE ip LIKE ?",
154
+ "prepared_parameters" => ["%%{[ip]}"],
155
+ "target" => "server",
156
+ "tag_on_failure" => ["_jdbcstaticfailure_server"]
157
+ }
158
+ end
159
+ let(:event) { LogStash::Event.new()}
160
+ let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
161
+ let(:prepared_statement) { double("prepared_statement")}
162
+
163
+ subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
164
+
165
+ before(:each) do
166
+ allow(local_db).to receive(:prepare).once.and_return(prepared_statement)
167
+ allow(prepared_statement).to receive(:call).once.and_return(records)
168
+ end
169
+
170
+ it "should be valid" do
171
+ expect(subject.valid?).to be_truthy
172
+ end
173
+
174
+ it "should have no formatted_errors" do
175
+ expect(subject.formatted_errors).to eq("")
176
+ end
177
+
178
+ it "should enhance an event" do
179
+ event.set("ip", "20.20")
180
+ subject.prepare(local_db)
181
+ subject.enhance(local_db, event)
182
+ expect(event.get("tags")).to be_nil
183
+ expect(event.get("server")).to eq(records)
184
+ end
185
+ end
186
+
187
+ describe "lookup operations with prepared statement multiple parameters" do
188
+ let(:local_db) { double("local_db") }
189
+ let(:lookup_hash) do
190
+ {
191
+ "query" => "select * from servers WHERE ip LIKE ? AND os LIKE ?",
192
+ "prepared_parameters" => ["%%{[ip]}", "os"],
193
+ "target" => "server",
194
+ "tag_on_failure" => ["_jdbcstaticfailure_server"]
195
+ }
196
+ end
197
+ let(:event) { LogStash::Event.new()}
198
+ let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
199
+ let(:prepared_statement) { double("prepared_statement")}
200
+
201
+ subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
202
+
203
+ before(:each) do
204
+ allow(local_db).to receive(:prepare).once.and_return(prepared_statement)
205
+ allow(prepared_statement).to receive(:call).once.and_return(records)
206
+ end
207
+
208
+ it "should be valid" do
209
+ expect(subject.valid?).to be_truthy
210
+ end
211
+
212
+ it "should have no formatted_errors" do
213
+ expect(subject.formatted_errors).to eq("")
214
+ end
215
+
216
+ it "should enhance an event" do
217
+ event.set("ip", "20.20")
218
+ event.set("os", "MacOS")
219
+ subject.prepare(local_db)
220
+ subject.enhance(local_db, event)
221
+ expect(event.get("tags")).to be_nil
222
+ expect(event.get("server")).to eq(records)
223
+ end
224
+ end
225
+
226
+ describe "lookup operations with badly configured prepared statement" do
227
+ let(:local_db) { double("local_db") }
228
+ let(:lookup_hash) do
229
+ {
230
+ "query" => "select * from servers WHERE ip LIKE ? AND os LIKE ?",
231
+ "prepared_parameters" => ["%%{[ip]}"],
232
+ "target" => "server",
233
+ "tag_on_failure" => ["_jdbcstaticfailure_server"]
234
+ }
235
+ end
236
+ let(:event) { LogStash::Event.new()}
237
+ let(:records) { [{"name" => "ldn-1-23", "rack" => "2:1:6"}] }
238
+ let(:prepared_statement) { double("prepared_statement")}
239
+
240
+ subject(:lookup) { described_class.new(lookup_hash, {}, "lookup-1") }
241
+
242
+ before(:each) do
243
+ allow(local_db).to receive(:prepare).once.and_return(prepared_statement)
244
+ allow(prepared_statement).to receive(:call).once.and_return(records)
245
+ end
246
+
247
+ it "must not be valid" do
248
+ expect(subject.valid?).to be_falsey
249
+ end
250
+ end
127
251
  end
128
252
  end end end
129
253
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-jdbc_static
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.7
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-10-29 00:00:00.000000000 Z
11
+ date: 2019-11-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement