logstash-integration-jdbc 5.2.4 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ad77fc92ab07e1da7e4c6285075b6583f9f293c073199b614e3852cd22bbc6a9
4
- data.tar.gz: 9e7b2c0f0c16930903bdee1f22b2651db7cf9f80d01a12f212b7a40155d4ebea
3
+ metadata.gz: d9937a8c1c5ab26acb4cd887f4bdb9dfff51e9348c7187ef2d44ff6055d137ee
4
+ data.tar.gz: 0042c3e29b5b08abe8baa4f84e4c240960bbc615778df9c77415f8db7985c831
5
5
  SHA512:
6
- metadata.gz: 2928a01bbe1ef418f36d6ddc7847416fc6d43278b20242926037bef02e8a429ba1d4587f1bd88f8d52de2ec1fef26e749aeb358de82a8bf69c36e5c3c9d42f8e
7
- data.tar.gz: 8aae596826ec1d9155e0f8bdfad9a882e92fa2be610f89d7c67a7b3f44f0e6937bcbafd78900457d92b20a5a28188163f7b4f25040d8deb229f505ab6faf2c22
6
+ metadata.gz: dbc38871c106b045a500645cf6c4517b30ce15e52bfc68520845e22276efdb4a21df3e4a7709ae682cd6b55ff76e063a615b9fe397795272535aa7c5298a9a92
7
+ data.tar.gz: 70b8e1dc84ba60d478affb22c93400cb153d4d0adc4904667970220ac03cb9d22b94e28f1b1a5504a30b8592c543d3234ee441d0ed57a31aefef70dfb6c2b9a3
data/CHANGELOG.md CHANGED
@@ -1,3 +1,12 @@
1
+ ## 5.3.0
2
+ - Refactor: start using scheduler mixin [#110](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/110)
3
+
4
+ ## 5.2.6
5
+ - Fix: change default path of 'last_run_metadata_path' to be rooted in the LS data.path folder and not in $HOME [#106](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/106)
6
+
7
+ ## 5.2.5
8
+ - Fix: do not execute more queries with debug logging [#109](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/109)
9
+
1
10
  ## 5.2.4
2
11
  - Fix: compatibility with all (>= 3.0) rufus-scheduler versions [#97](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/97)
3
12
 
@@ -3,7 +3,7 @@ require "logstash-integration-jdbc_jars"
3
3
  require "logstash/filters/base"
4
4
  require "logstash/namespace"
5
5
  require "logstash/plugin_mixins/ecs_compatibility_support"
6
- require "logstash/plugin_mixins/jdbc/scheduler"
6
+ require "logstash/plugin_mixins/scheduler"
7
7
  require_relative "jdbc/loader"
8
8
  require_relative "jdbc/loader_schedule"
9
9
  require_relative "jdbc/repeating_load_runner"
@@ -19,6 +19,8 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
19
19
  # adds ecs_compatibility config which could be :disabled or :v1
20
20
  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
21
21
 
22
+ include LogStash::PluginMixins::Scheduler
23
+
22
24
  config_name "jdbc_static"
23
25
 
24
26
  # Define the loaders, an Array of Hashes, to fetch remote data and create local tables.
@@ -162,7 +164,6 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
162
164
  end
163
165
 
164
166
  def close
165
- @scheduler.stop if @scheduler
166
167
  @parsed_loaders.each(&:close)
167
168
  @processor.close
168
169
  end
@@ -175,9 +176,23 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
175
176
  private
176
177
 
177
178
  def prepare_data_dir
179
+ # cleanup existing Derby file left behind in $HOME
180
+ derby_log = "#{ENV['HOME']}/derby.log"
181
+ if ::File.exist?(derby_log)
182
+ begin
183
+ ::File.delete(derby_log)
184
+ rescue Errno::EPERM => e
185
+ @logger.warn("Can't delete temporary file #{derby_log} due to access permissions")
186
+ rescue e
187
+ @logger.warn("Can't delete temporary file #{derby_log}", {message => e.message})
188
+ end
189
+ end
190
+
178
191
  # later, when local persistent databases are allowed set this property to LS_HOME/data/jdbc-static/
179
192
  # must take multi-pipelines into account and more than one config using the same jdbc-static settings
180
- java.lang.System.setProperty("derby.system.home", ENV["HOME"])
193
+ path_data = Pathname.new(LogStash::SETTINGS.get_value("path.data")).join("plugins", "shared", "derby_home")
194
+ path_data.mkpath
195
+ java.lang.System.setProperty("derby.system.home", path_data.to_path)
181
196
  logger.info("derby.system.home is: #{java.lang.System.getProperty("derby.system.home")}")
182
197
  end
183
198
 
@@ -194,13 +209,7 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
194
209
  if @loader_schedule
195
210
  @loader_runner = Jdbc::RepeatingLoadRunner.new(*runner_args)
196
211
  @loader_runner.initial_load
197
- @scheduler = LogStash::PluginMixins::Jdbc::Scheduler.
198
- start_cron_scheduler(@loader_schedule, thread_name: "[#{id}]-jdbc_static__scheduler") { @loader_runner.repeated_load }
199
- cron_job = @scheduler.cron_jobs.first
200
- if cron_job
201
- frequency = cron_job.respond_to?(:rough_frequency) ? cron_job.rough_frequency : cron_job.frequency
202
- logger.info("Loaders will execute every #{frequency} seconds", loader_schedule: @loader_schedule)
203
- end
212
+ scheduler.cron(@loader_schedule) { @loader_runner.repeated_load }
204
213
  else
205
214
  @loader_runner = Jdbc::SingleLoadRunner.new(*runner_args)
206
215
  @loader_runner.initial_load
@@ -3,12 +3,12 @@ require "logstash/inputs/base"
3
3
  require "logstash/namespace"
4
4
  require "logstash/plugin_mixins/jdbc/common"
5
5
  require "logstash/plugin_mixins/jdbc/jdbc"
6
- require "logstash/plugin_mixins/jdbc/scheduler"
7
6
  require "logstash/plugin_mixins/ecs_compatibility_support"
8
7
  require "logstash/plugin_mixins/ecs_compatibility_support/target_check"
9
8
  require "logstash/plugin_mixins/validator_support/field_reference_validation_adapter"
10
-
11
9
  require "logstash/plugin_mixins/event_support/event_factory_adapter"
10
+ require "logstash/plugin_mixins/scheduler"
11
+ require "fileutils"
12
12
 
13
13
  # this require_relative returns early unless the JRuby version is between 9.2.0.0 and 9.2.8.0
14
14
  require_relative "tzinfo_jruby_patch"
@@ -146,6 +146,8 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
146
146
  # adds :field_reference validator adapter
147
147
  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
148
148
 
149
+ include LogStash::PluginMixins::Scheduler
150
+
149
151
  config_name "jdbc"
150
152
 
151
153
  # If undefined, Logstash will complain, even if codec is unused.
@@ -178,8 +180,10 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
178
180
  # exactly once.
179
181
  config :schedule, :validate => :string
180
182
 
181
- # Path to file with last run time
182
- config :last_run_metadata_path, :validate => :string, :default => "#{ENV['HOME']}/.logstash_jdbc_last_run"
183
+ # Path to file with last run time.
184
+ # The default will write file to `<path.data>/plugins/inputs/jdbc/logstash_jdbc_last_run`
185
+ # NOTE: it must be a file path and not a directory path
186
+ config :last_run_metadata_path, :validate => :string
183
187
 
184
188
  # Use an incremental column value rather than a timestamp
185
189
  config :use_column_value, :validate => :boolean, :default => false
@@ -230,12 +234,33 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
230
234
  config :target, :validate => :field_reference, :required => false
231
235
 
232
236
  attr_reader :database # for test mocking/stubbing
237
+ attr_reader :last_run_metadata_file_path # path to the file used as last run storage
233
238
 
234
239
  public
235
240
 
236
241
  def register
237
242
  @logger = self.logger
238
- require "rufus/scheduler"
243
+
244
+ if @record_last_run
245
+ if @last_run_metadata_path.nil?
246
+ logstash_data_path = LogStash::SETTINGS.get_value("path.data")
247
+ logstash_data_path = Pathname.new(logstash_data_path).join("plugins", "inputs", "jdbc")
248
+ # Ensure that the filepath exists before writing, since it's deeply nested.
249
+ logstash_data_path.mkpath
250
+ logstash_data_file_path = logstash_data_path.join("logstash_jdbc_last_run")
251
+
252
+ ensure_default_metadatafile_location(logstash_data_file_path)
253
+
254
+ @last_run_metadata_file_path = logstash_data_file_path.to_path
255
+ else
256
+ # validate the path is a file and not a directory
257
+ if Pathname.new(@last_run_metadata_path).directory?
258
+ raise LogStash::ConfigurationError.new("The \"last_run_metadata_path\" argument must point to a file, received a directory: \"#{last_run_metadata_path}\"")
259
+ end
260
+ @last_run_metadata_file_path = @last_run_metadata_path
261
+ end
262
+ end
263
+
239
264
  prepare_jdbc_connection
240
265
 
241
266
  if @use_column_value
@@ -259,8 +284,8 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
259
284
  end
260
285
  end
261
286
 
262
- set_value_tracker(LogStash::PluginMixins::Jdbc::ValueTracking.build_last_value_tracker(self))
263
- set_statement_logger(LogStash::PluginMixins::Jdbc::CheckedCountLogger.new(@logger))
287
+ set_value_tracker LogStash::PluginMixins::Jdbc::ValueTracking.build_last_value_tracker(self)
288
+ set_statement_handler LogStash::PluginMixins::Jdbc::StatementHandler.build_statement_handler(self)
264
289
 
265
290
  @enable_encoding = !@charset.nil? || !@columns_charset.empty?
266
291
 
@@ -283,8 +308,8 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
283
308
  end # def register
284
309
 
285
310
  # test injection points
286
- def set_statement_logger(instance)
287
- @statement_handler = LogStash::PluginMixins::Jdbc::StatementHandler.build_statement_handler(self, instance)
311
+ def set_statement_handler(handler)
312
+ @statement_handler = handler
288
313
  end
289
314
 
290
315
  def set_value_tracker(instance)
@@ -294,22 +319,16 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
294
319
  def run(queue)
295
320
  load_driver
296
321
  if @schedule
297
- # input thread (Java) name example "[my-oracle]<jdbc"
298
- @scheduler = LogStash::PluginMixins::Jdbc::Scheduler.
299
- start_cron_scheduler(@schedule, thread_name: "[#{id}]<jdbc__scheduler") { execute_query(queue) }
300
- @scheduler.join
322
+ # scheduler input thread name example: "[my-oracle]|input|jdbc|scheduler"
323
+ scheduler.cron(@schedule) { execute_query(queue) }
324
+ scheduler.join
301
325
  else
302
326
  execute_query(queue)
303
327
  end
304
328
  end # def run
305
329
 
306
- def close
307
- @scheduler.shutdown if @scheduler
308
- end
309
-
310
330
  def stop
311
331
  close_jdbc_connection
312
- @scheduler.shutdown(:wait) if @scheduler
313
332
  end
314
333
 
315
334
  private
@@ -363,4 +382,23 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
363
382
  value
364
383
  end
365
384
  end
385
+
386
+ def ensure_default_metadatafile_location(metadata_new_path)
387
+ old_default_path = Pathname.new("#{ENV['HOME']}/.logstash_jdbc_last_run")
388
+
389
+ if old_default_path.exist? && !metadata_new_path.exist?
390
+ # Previous versions of the plugin hosted the state file into $HOME/.logstash_jdbc_last_run.
391
+ # Copy in the new location
392
+ FileUtils.cp(old_default_path.to_path, metadata_new_path.to_path)
393
+ begin
394
+ # If there is a permission error in the delete of the old file inform the user to give
395
+ # the correct access rights
396
+ ::File.delete(old_default_path.to_path)
397
+ @logger.info("Successfully moved the #{old_default_path.to_path} into #{metadata_new_path.to_path}")
398
+ rescue e
399
+ @logger.warn("Using new metadata file at #{metadata_new_path.to_path} but #{old_default_path} can't be removed.")
400
+ end
401
+ end
402
+ end
403
+
366
404
  end end end # class LogStash::Inputs::Jdbc
@@ -4,7 +4,6 @@ require "logstash/config/mixin"
4
4
  require "time"
5
5
  require "date"
6
6
  require_relative "value_tracking"
7
- require_relative "checked_count_logger"
8
7
  require_relative "statement_handler"
9
8
 
10
9
  java_import java.util.concurrent.locks.ReentrantLock
@@ -2,7 +2,7 @@
2
2
 
3
3
  module LogStash module PluginMixins module Jdbc
4
4
  class StatementHandler
5
- def self.build_statement_handler(plugin, logger)
5
+ def self.build_statement_handler(plugin)
6
6
  if plugin.use_prepared_statements
7
7
  klass = PreparedStatementHandler
8
8
  else
@@ -16,27 +16,39 @@ module LogStash module PluginMixins module Jdbc
16
16
  klass = NormalStatementHandler
17
17
  end
18
18
  end
19
- klass.new(plugin, logger)
19
+ klass.new(plugin)
20
20
  end
21
21
 
22
- attr_reader :statement, :parameters, :statement_logger
22
+ attr_reader :statement, :parameters
23
23
 
24
- def initialize(plugin, statement_logger)
24
+ def initialize(plugin)
25
25
  @statement = plugin.statement
26
- @statement_logger = statement_logger
27
- post_init(plugin)
28
26
  end
29
27
 
30
28
  def build_query(db, sql_last_value)
31
- # override in subclass
29
+ fail NotImplementedError # override in subclass
32
30
  end
33
31
 
34
- def post_init(plugin)
35
- # override in subclass, if needed
36
- end
37
32
  end
38
33
 
39
34
  class NormalStatementHandler < StatementHandler
35
+
36
+ attr_reader :parameters
37
+
38
+ def initialize(plugin)
39
+ super(plugin)
40
+ @parameter_keys = ["sql_last_value"] + plugin.parameters.keys
41
+ @parameters = plugin.parameters.inject({}) do |hash,(k,v)|
42
+ case v
43
+ when LogStash::Timestamp
44
+ hash[k.to_sym] = v.time
45
+ else
46
+ hash[k.to_sym] = v
47
+ end
48
+ hash
49
+ end
50
+ end
51
+
40
52
  # Performs the query, yielding once per row of data
41
53
  # @param db [Sequel::Database]
42
54
  # @param sql_last_value [Integer|DateTime|Time]
@@ -52,27 +64,18 @@ module LogStash module PluginMixins module Jdbc
52
64
 
53
65
  def build_query(db, sql_last_value)
54
66
  parameters[:sql_last_value] = sql_last_value
55
- query = db[statement, parameters]
56
- statement_logger.log_statement_parameters(statement, parameters, query)
57
- query
67
+ db[statement, parameters]
58
68
  end
59
69
 
60
- def post_init(plugin)
61
- @parameter_keys = ["sql_last_value"] + plugin.parameters.keys
62
- @parameters = plugin.parameters.inject({}) do |hash,(k,v)|
63
- case v
64
- when LogStash::Timestamp
65
- hash[k.to_sym] = v.time
66
- else
67
- hash[k.to_sym] = v
68
- end
69
- hash
70
- end
71
- end
72
70
  end
73
71
 
74
72
  class PagedNormalStatementHandler < NormalStatementHandler
75
- attr_reader :jdbc_page_size
73
+
74
+ def initialize(plugin)
75
+ super(plugin)
76
+ @jdbc_page_size = plugin.jdbc_page_size
77
+ @logger = plugin.logger
78
+ end
76
79
 
77
80
  # Performs the query, respecting our pagination settings, yielding once per row of data
78
81
  # @param db [Sequel::Database]
@@ -81,16 +84,22 @@ module LogStash module PluginMixins module Jdbc
81
84
  def perform_query(db, sql_last_value)
82
85
  query = build_query(db, sql_last_value)
83
86
  query.each_page(@jdbc_page_size) do |paged_dataset|
87
+ log_dataset_page(paged_dataset) if @logger.debug?
84
88
  paged_dataset.each do |row|
85
89
  yield row
86
90
  end
87
91
  end
88
92
  end
89
93
 
90
- def post_init(plugin)
91
- super(plugin)
92
- @jdbc_page_size = plugin.jdbc_page_size
94
+ private
95
+
96
+ # @param paged_dataset [Sequel::Dataset::Pagination] like object
97
+ def log_dataset_page(paged_dataset)
98
+ @logger.debug "fetching paged dataset", current_page: paged_dataset.current_page,
99
+ record_count: paged_dataset.current_page_record_count,
100
+ total_record_count: paged_dataset.pagination_record_count
93
101
  end
102
+
94
103
  end
95
104
 
96
105
  class ExplicitPagingModeStatementHandler < PagedNormalStatementHandler
@@ -101,20 +110,29 @@ module LogStash module PluginMixins module Jdbc
101
110
  def perform_query(db, sql_last_value)
102
111
  query = build_query(db, sql_last_value)
103
112
  offset = 0
113
+ page_size = @jdbc_page_size
104
114
  loop do
105
115
  rows_in_page = 0
106
- query.with_sql(query.sql, offset: offset, size: jdbc_page_size).each do |row|
116
+ query.with_sql(query.sql, offset: offset, size: page_size).each do |row|
107
117
  yield row
108
118
  rows_in_page += 1
109
119
  end
110
- break unless rows_in_page == jdbc_page_size
111
- offset += jdbc_page_size
120
+ break unless rows_in_page == page_size
121
+ offset += page_size
112
122
  end
113
123
  end
114
124
  end
115
125
 
116
126
  class PreparedStatementHandler < StatementHandler
117
- attr_reader :name, :bind_values_array, :statement_prepared, :prepared
127
+ attr_reader :name, :bind_values_array, :statement_prepared, :prepared, :parameters
128
+
129
+ def initialize(plugin)
130
+ super(plugin)
131
+ @name = plugin.prepared_statement_name.to_sym
132
+ @bind_values_array = plugin.prepared_statement_bind_values
133
+ @parameters = plugin.parameters
134
+ @statement_prepared = Concurrent::AtomicBoolean.new(false)
135
+ end
118
136
 
119
137
  # Performs the query, ignoring our pagination settings, yielding once per row of data
120
138
  # @param db [Sequel::Database]
@@ -142,7 +160,6 @@ module LogStash module PluginMixins module Jdbc
142
160
  db.set_prepared_statement(name, prepared)
143
161
  end
144
162
  bind_value_sql_last_value(sql_last_value)
145
- statement_logger.log_statement_parameters(statement, parameters, nil)
146
163
  begin
147
164
  db.call(name, parameters)
148
165
  rescue => e
@@ -153,17 +170,6 @@ module LogStash module PluginMixins module Jdbc
153
170
  end
154
171
  end
155
172
 
156
- def post_init(plugin)
157
- # don't log statement count when using prepared statements for now...
158
- # needs enhancement to allow user to supply a bindable count prepared statement in settings.
159
- @statement_logger.disable_count
160
-
161
- @name = plugin.prepared_statement_name.to_sym
162
- @bind_values_array = plugin.prepared_statement_bind_values
163
- @parameters = plugin.parameters
164
- @statement_prepared = Concurrent::AtomicBoolean.new(false)
165
- end
166
-
167
173
  def create_bind_values_hash
168
174
  hash = {}
169
175
  bind_values_array.each_with_index {|v,i| hash[:"p#{i}"] = v}
@@ -5,7 +5,7 @@ module LogStash module PluginMixins module Jdbc
5
5
  class ValueTracking
6
6
 
7
7
  def self.build_last_value_tracker(plugin)
8
- handler = plugin.record_last_run ? FileHandler.new(plugin.last_run_metadata_path) : NullFileHandler.new(plugin.last_run_metadata_path)
8
+ handler = plugin.record_last_run ? FileHandler.new(plugin.last_run_metadata_file_path) : NullFileHandler.new(plugin.last_run_metadata_file_path)
9
9
  if plugin.clean_run
10
10
  handler.clean
11
11
  end
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-integration-jdbc'
3
- s.version = '5.2.4'
3
+ s.version = '5.3.0'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Integration with JDBC - input and filter plugins"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -34,12 +34,11 @@ Gem::Specification.new do |s|
34
34
 
35
35
  s.add_runtime_dependency 'tzinfo'
36
36
  s.add_runtime_dependency 'tzinfo-data'
37
- # plugin maintains compatibility with < 3.5 (3.0.9)
38
- # but works with newer rufus-scheduler >= 3.5 as well
39
- s.add_runtime_dependency 'rufus-scheduler'
37
+
40
38
  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
41
39
  s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
42
40
  s.add_runtime_dependency "logstash-mixin-event_support", '~> 1.0'
41
+ s.add_runtime_dependency "logstash-mixin-scheduler", '~> 1.0'
43
42
 
44
43
  s.add_development_dependency "childprocess"
45
44
  s.add_development_dependency 'logstash-devutils', '>= 2.3'
@@ -5,6 +5,7 @@ require "sequel"
5
5
  require "sequel/adapters/jdbc"
6
6
  require "stud/temporary"
7
7
  require "timecop"
8
+ require "pathname"
8
9
 
9
10
  # LogStash::Logging::Logger::configure_logging("WARN")
10
11
 
@@ -85,6 +86,15 @@ module LogStash module Filters
85
86
 
86
87
  let(:ipaddr) { ".3.1.1" }
87
88
 
89
+ describe "verify derby path property" do
90
+ it "should be set into Logstash data path" do
91
+ plugin.register
92
+
93
+ expected = Pathname.new(LogStash::SETTINGS.get_value("path.data")).join("plugins", "shared", "derby_home").to_path
94
+ expect(java.lang.System.getProperty("derby.system.home")).to eq(expected)
95
+ end
96
+ end
97
+
88
98
  describe "non scheduled operation" do
89
99
  after { plugin.close }
90
100
 
@@ -41,6 +41,31 @@ describe LogStash::Inputs::Jdbc, :integration => true do
41
41
  expect(event.get('first_name')).to eq("Mark")
42
42
  expect(event.get('last_name')).to eq("Guckenheimer")
43
43
  end
44
+
45
+ context 'with paging' do
46
+ let(:settings) do
47
+ super().merge 'jdbc_paging_enabled' => true, 'jdbc_page_size' => 1,
48
+ "statement" => 'SELECT * FROM "employee" WHERE EMP_NO >= :p1 ORDER BY EMP_NO',
49
+ 'parameters' => { 'p1' => 0 }
50
+ end
51
+
52
+ before do # change plugin logger level to debug - to exercise logging
53
+ logger = plugin.class.name.gsub('::', '.').downcase
54
+ logger = org.apache.logging.log4j.LogManager.getLogger(logger)
55
+ @prev_logger_level = [ logger.getName, logger.getLevel ]
56
+ org.apache.logging.log4j.core.config.Configurator.setLevel logger.getName, org.apache.logging.log4j.Level::DEBUG
57
+ end
58
+
59
+ after do
60
+ org.apache.logging.log4j.core.config.Configurator.setLevel *@prev_logger_level
61
+ end
62
+
63
+ it "should populate the event with database entries" do
64
+ plugin.run(queue)
65
+ event = queue.pop
66
+ expect(event.get('first_name')).to eq('David')
67
+ end
68
+ end
44
69
  end
45
70
 
46
71
  context "when supplying a non-existent library" do
@@ -9,6 +9,7 @@ require "timecop"
9
9
  require "stud/temporary"
10
10
  require "time"
11
11
  require "date"
12
+ require "pathname"
12
13
 
13
14
  # We do not need to set TZ env var anymore because we can have 'Sequel.application_timezone' set to utc by default now.
14
15
 
@@ -51,6 +52,9 @@ describe LogStash::Inputs::Jdbc do
51
52
  db.drop_table(:types_table)
52
53
  db.drop_table(:test1_table)
53
54
  end
55
+
56
+ last_run_default_path = LogStash::SETTINGS.get_value("path.data")
57
+ FileUtils.rm_f("#{last_run_default_path}/plugins/inputs/jdbc/logstash_jdbc_last_run")
54
58
  end
55
59
 
56
60
  context "when registering and tearing down" do
@@ -247,18 +251,6 @@ describe LogStash::Inputs::Jdbc do
247
251
  Timecop.return
248
252
  end
249
253
 
250
- it "cleans up scheduler resources on close" do
251
- runner = Thread.new do
252
- plugin.run(queue)
253
- end
254
- sleep 1
255
- plugin.do_close
256
-
257
- scheduler = plugin.instance_variable_get(:@scheduler)
258
- expect(scheduler).to_not be_nil
259
- expect(scheduler.down?).to be_truthy
260
- end
261
-
262
254
  end
263
255
 
264
256
  context "when scheduling and previous runs are to be preserved" do
@@ -1114,6 +1106,86 @@ describe LogStash::Inputs::Jdbc do
1114
1106
  end
1115
1107
  end
1116
1108
 
1109
+ context "when state is persisted" do
1110
+ context "to file" do
1111
+ let(:settings) do
1112
+ {
1113
+ "statement" => "SELECT * FROM test_table",
1114
+ "record_last_run" => true
1115
+ }
1116
+ end
1117
+
1118
+ before do
1119
+ plugin.register
1120
+ end
1121
+
1122
+ after do
1123
+ plugin.stop
1124
+ end
1125
+
1126
+ context "with default last_run_metadata_path" do
1127
+ it "should save state in data.data subpath" do
1128
+ path = LogStash::SETTINGS.get_value("path.data")
1129
+ expect(plugin.last_run_metadata_file_path).to start_with(path)
1130
+ end
1131
+ end
1132
+
1133
+ context "with customized last_run_metadata_path" do
1134
+ let(:settings) { super().merge({ "last_run_metadata_path" => Stud::Temporary.pathname })}
1135
+
1136
+ it "should save state in data.data subpath" do
1137
+ expect(plugin.last_run_metadata_file_path).to start_with(settings["last_run_metadata_path"])
1138
+ end
1139
+ end
1140
+ end
1141
+
1142
+ context "with customized last_run_metadata_path point to directory" do
1143
+ let(:settings) do
1144
+ path = Stud::Temporary.pathname
1145
+ Pathname.new(path).tap {|path| path.mkpath}
1146
+ super().merge({ "last_run_metadata_path" => path})
1147
+ end
1148
+
1149
+ it "raise configuration error" do
1150
+ expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
1151
+ end
1152
+ end
1153
+ end
1154
+
1155
+ context "update the previous default last_run_metadata_path" do
1156
+ let(:settings) do
1157
+ {
1158
+ "statement" => "SELECT * FROM test_table",
1159
+ "record_last_run" => true
1160
+ }
1161
+ end
1162
+
1163
+ let(:fake_home) do
1164
+ path = Stud::Temporary.pathname
1165
+ Pathname.new(path).tap {|path| path.mkpath}
1166
+ path
1167
+ end
1168
+
1169
+ context "when a file exists" do
1170
+ before do
1171
+ # in a faked HOME folder save a valid previous last_run metadata file
1172
+ allow(ENV).to receive(:[]).with('HOME').and_return(fake_home)
1173
+ File.open("#{ENV['HOME']}/.logstash_jdbc_last_run", 'w') do |file|
1174
+ file.write("--- !ruby/object:DateTime '2022-03-08 08:10:00.486889000 Z'")
1175
+ end
1176
+ end
1177
+
1178
+ it "should be moved" do
1179
+ plugin.register
1180
+
1181
+ expect(::File.exist?("#{ENV['HOME']}/.logstash_jdbc_last_run")).to be false
1182
+ path = LogStash::SETTINGS.get_value("path.data")
1183
+ full_path = "#{path}/plugins/inputs/jdbc/logstash_jdbc_last_run"
1184
+ expect(::File.exist?(full_path)).to be true
1185
+ end
1186
+ end
1187
+ end
1188
+
1117
1189
  context "when setting fetch size" do
1118
1190
 
1119
1191
  let(:settings) do
@@ -1432,54 +1504,6 @@ describe LogStash::Inputs::Jdbc do
1432
1504
  end
1433
1505
  end
1434
1506
 
1435
- context "when debug logging and a count query raises a count related error" do
1436
- let(:settings) do
1437
- { "statement" => "SELECT * from types_table" }
1438
- end
1439
- let(:logger) { double("logger", :debug? => true) }
1440
- let(:statement_logger) { LogStash::PluginMixins::Jdbc::CheckedCountLogger.new(logger) }
1441
- let(:value_tracker) { double("value tracker", :set_value => nil, :write => nil) }
1442
- let(:msg) { 'Java::JavaSql::SQLSyntaxErrorException: Dynamic SQL Error; SQL error code = -104; Token unknown - line 1, column 105; LIMIT [SQLState:42000, ISC error code:335544634]' }
1443
- let(:error_args) do
1444
- {"exception" => msg}
1445
- end
1446
-
1447
- before do
1448
- db << "INSERT INTO types_table (num, string, started_at, custom_time, ranking) VALUES (1, 'A test', '1999-12-31', '1999-12-31 23:59:59', 95.67)"
1449
- plugin.register
1450
- plugin.set_statement_logger(statement_logger)
1451
- plugin.set_value_tracker(value_tracker)
1452
- allow(value_tracker).to receive(:value).and_return("bar")
1453
- allow(statement_logger).to receive(:execute_count).once.and_raise(StandardError.new(msg))
1454
- end
1455
-
1456
- after do
1457
- plugin.stop
1458
- end
1459
-
1460
- context "if the count query raises an error" do
1461
- it "should log a debug line without a count key as its unknown whether a count works at this stage" do
1462
- expect(logger).to receive(:warn).once.with("Attempting a count query raised an error, the generated count statement is most likely incorrect but check networking, authentication or your statement syntax", error_args)
1463
- expect(logger).to receive(:warn).once.with("Ongoing count statement generation is being prevented")
1464
- expect(logger).to receive(:debug).once.with("Executing JDBC query", :statement => settings["statement"], :parameters => {:sql_last_value=>"bar"})
1465
- plugin.run(queue)
1466
- queue.pop
1467
- end
1468
-
1469
- it "should create an event normally" do
1470
- allow(logger).to receive(:warn)
1471
- allow(logger).to receive(:debug)
1472
- plugin.run(queue)
1473
- event = queue.pop
1474
- expect(event.get("num")).to eq(1)
1475
- expect(event.get("string")).to eq("A test")
1476
- expect(event.get("started_at")).to be_a_logstash_timestamp_equivalent_to("1999-12-31T00:00:00.000Z")
1477
- expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("1999-12-31T23:59:59.000Z")
1478
- expect(event.get("ranking").to_f).to eq(95.67)
1479
- end
1480
- end
1481
- end
1482
-
1483
1507
  context "when an unreadable jdbc_driver_path entry is present" do
1484
1508
  let(:driver_jar_path) do
1485
1509
  jar_file = $CLASSPATH.find { |name| name.index(Jdbc::Derby.driver_jar) }
@@ -1681,7 +1705,7 @@ describe LogStash::Inputs::Jdbc do
1681
1705
  let(:jdbc_driver_class) { "org.apache.NonExistentDriver" }
1682
1706
  it "raises a loading error" do
1683
1707
  expect { plugin.send(:load_driver) }.to raise_error LogStash::PluginLoadingError,
1684
- /java.lang.ClassNotFoundException: org.apache.NonExistentDriver/
1708
+ /ClassNotFoundException: org.apache.NonExistentDriver/
1685
1709
  end
1686
1710
  end
1687
1711
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-integration-jdbc
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.2.4
4
+ version: 5.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-03-14 00:00:00.000000000 Z
11
+ date: 2022-06-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -131,38 +131,38 @@ dependencies:
131
131
  - !ruby/object:Gem::Dependency
132
132
  requirement: !ruby/object:Gem::Requirement
133
133
  requirements:
134
- - - ">="
134
+ - - "~>"
135
135
  - !ruby/object:Gem::Version
136
- version: '0'
137
- name: rufus-scheduler
136
+ version: '1.3'
137
+ name: logstash-mixin-ecs_compatibility_support
138
138
  prerelease: false
139
139
  type: :runtime
140
140
  version_requirements: !ruby/object:Gem::Requirement
141
141
  requirements:
142
- - - ">="
142
+ - - "~>"
143
143
  - !ruby/object:Gem::Version
144
- version: '0'
144
+ version: '1.3'
145
145
  - !ruby/object:Gem::Dependency
146
146
  requirement: !ruby/object:Gem::Requirement
147
147
  requirements:
148
148
  - - "~>"
149
149
  - !ruby/object:Gem::Version
150
- version: '1.3'
151
- name: logstash-mixin-ecs_compatibility_support
150
+ version: '1.0'
151
+ name: logstash-mixin-validator_support
152
152
  prerelease: false
153
153
  type: :runtime
154
154
  version_requirements: !ruby/object:Gem::Requirement
155
155
  requirements:
156
156
  - - "~>"
157
157
  - !ruby/object:Gem::Version
158
- version: '1.3'
158
+ version: '1.0'
159
159
  - !ruby/object:Gem::Dependency
160
160
  requirement: !ruby/object:Gem::Requirement
161
161
  requirements:
162
162
  - - "~>"
163
163
  - !ruby/object:Gem::Version
164
164
  version: '1.0'
165
- name: logstash-mixin-validator_support
165
+ name: logstash-mixin-event_support
166
166
  prerelease: false
167
167
  type: :runtime
168
168
  version_requirements: !ruby/object:Gem::Requirement
@@ -176,7 +176,7 @@ dependencies:
176
176
  - - "~>"
177
177
  - !ruby/object:Gem::Version
178
178
  version: '1.0'
179
- name: logstash-mixin-event_support
179
+ name: logstash-mixin-scheduler
180
180
  prerelease: false
181
181
  type: :runtime
182
182
  version_requirements: !ruby/object:Gem::Requirement
@@ -275,10 +275,8 @@ files:
275
275
  - lib/logstash/filters/jdbc_streaming.rb
276
276
  - lib/logstash/inputs/jdbc.rb
277
277
  - lib/logstash/inputs/tzinfo_jruby_patch.rb
278
- - lib/logstash/plugin_mixins/jdbc/checked_count_logger.rb
279
278
  - lib/logstash/plugin_mixins/jdbc/common.rb
280
279
  - lib/logstash/plugin_mixins/jdbc/jdbc.rb
281
- - lib/logstash/plugin_mixins/jdbc/scheduler.rb
282
280
  - lib/logstash/plugin_mixins/jdbc/statement_handler.rb
283
281
  - lib/logstash/plugin_mixins/jdbc/value_tracking.rb
284
282
  - lib/logstash/plugin_mixins/jdbc_streaming.rb
@@ -307,7 +305,6 @@ files:
307
305
  - spec/helpers/derbyrun.jar
308
306
  - spec/inputs/integration/integ_spec.rb
309
307
  - spec/inputs/jdbc_spec.rb
310
- - spec/plugin_mixins/jdbc/scheduler_spec.rb
311
308
  - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb
312
309
  - vendor/jar-dependencies/org/apache/derby/derby/10.14.1.0/derby-10.14.1.0.jar
313
310
  - vendor/jar-dependencies/org/apache/derby/derbyclient/10.14.1.0/derbyclient-10.14.1.0.jar
@@ -360,5 +357,4 @@ test_files:
360
357
  - spec/helpers/derbyrun.jar
361
358
  - spec/inputs/integration/integ_spec.rb
362
359
  - spec/inputs/jdbc_spec.rb
363
- - spec/plugin_mixins/jdbc/scheduler_spec.rb
364
360
  - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb
@@ -1,43 +0,0 @@
1
- # encoding: utf-8
2
-
3
- module LogStash module PluginMixins module Jdbc
4
- class CheckedCountLogger
5
- def initialize(logger)
6
- @logger = logger
7
- @needs_check = true
8
- @count_is_supported = false
9
- @in_debug = @logger.debug?
10
- end
11
-
12
- def disable_count
13
- @needs_check = false
14
- @count_is_supported = false
15
- end
16
-
17
- def log_statement_parameters(statement, parameters, query)
18
- return unless @in_debug
19
- check_count_query(query) if @needs_check && query
20
- if @count_is_supported
21
- @logger.debug("Executing JDBC query", :statement => statement, :parameters => parameters, :count => execute_count(query))
22
- else
23
- @logger.debug("Executing JDBC query", :statement => statement, :parameters => parameters)
24
- end
25
- end
26
-
27
- def check_count_query(query)
28
- @needs_check = false
29
- begin
30
- execute_count(query)
31
- @count_is_supported = true
32
- rescue Exception => e
33
- @logger.warn("Attempting a count query raised an error, the generated count statement is most likely incorrect but check networking, authentication or your statement syntax", "exception" => e.message)
34
- @logger.warn("Ongoing count statement generation is being prevented")
35
- @count_is_supported = false
36
- end
37
- end
38
-
39
- def execute_count(query)
40
- query.count
41
- end
42
- end
43
- end end end
@@ -1,175 +0,0 @@
1
- require 'rufus/scheduler'
2
-
3
- require 'logstash/util/loggable'
4
-
5
- module LogStash module PluginMixins module Jdbc
6
- class Scheduler < Rufus::Scheduler
7
-
8
- include LogStash::Util::Loggable
9
-
10
- # Rufus::Scheduler >= 3.4 moved the Time impl into a gem EoTime = ::EtOrbi::EoTime`
11
- # Rufus::Scheduler 3.1 - 3.3 using it's own Time impl `Rufus::Scheduler::ZoTime`
12
- TimeImpl = defined?(Rufus::Scheduler::EoTime) ? Rufus::Scheduler::EoTime :
13
- (defined?(Rufus::Scheduler::ZoTime) ? Rufus::Scheduler::ZoTime : ::Time)
14
-
15
- # @param cron [String] cron-line
16
- # @param opts [Hash] scheduler options
17
- # @return scheduler instance
18
- def self.start_cron_scheduler(cron, opts = {}, &block)
19
- unless block_given?
20
- raise ArgumentError, 'missing (cron scheduler) block - worker task to execute'
21
- end
22
- scheduler = new_scheduler(opts)
23
- scheduler.schedule_cron(cron, &block)
24
- scheduler
25
- end
26
-
27
- # @param opts [Hash] scheduler options
28
- # @return scheduler instance
29
- def self.new_scheduler(opts)
30
- unless opts.key?(:thread_name)
31
- raise ArgumentError, 'thread_name: option is required to be able to distinguish multiple scheduler threads'
32
- end
33
- opts[:max_work_threads] ||= 1
34
- # amount the scheduler thread sleeps between checking whether jobs
35
- # should trigger, default is 0.3 which is a bit too often ...
36
- # in theory the cron expression '* * * * * *' supports running jobs
37
- # every second but this is very rare, we could potentially go higher
38
- opts[:frequency] ||= 1.0
39
-
40
- new(opts)
41
- end
42
-
43
- # @overload
44
- def timeout_jobs
45
- # Rufus relies on `Thread.list` which is a blocking operation and with many schedulers
46
- # (and threads) within LS will have a negative impact on performance as scheduler
47
- # threads will end up waiting to obtain the `Thread.list` lock.
48
- #
49
- # However, this isn't necessary we can easily detect whether there are any jobs
50
- # that might need to timeout: only when `@opts[:timeout]` is set causes worker thread(s)
51
- # to have a `Thread.current[:rufus_scheduler_timeout]` that is not nil
52
- return unless @opts[:timeout]
53
- super
54
- end
55
-
56
- # @overload
57
- def work_threads(query = :all)
58
- if query == :__all_no_cache__ # special case from JobDecorator#start_work_thread
59
- @_work_threads = nil # when a new worker thread is being added reset
60
- return super(:all)
61
- end
62
-
63
- # Gets executed every time a job is triggered, we're going to cache the
64
- # worker threads for this scheduler (to avoid `Thread.list`) - they only
65
- # change when a new thread is being started from #start_work_thread ...
66
- work_threads = @_work_threads
67
- if work_threads.nil?
68
- work_threads = threads.select { |t| t[:rufus_scheduler_work_thread] }
69
- @_work_threads = work_threads
70
- end
71
-
72
- case query
73
- when :active then work_threads.select { |t| t[:rufus_scheduler_job] }
74
- when :vacant then work_threads.reject { |t| t[:rufus_scheduler_job] }
75
- else work_threads
76
- end
77
- end
78
-
79
- # @overload
80
- def on_error(job, err)
81
- details = { exception: err.class, message: err.message, backtrace: err.backtrace }
82
- details[:cause] = err.cause if err.cause
83
-
84
- details[:now] = debug_format_time(TimeImpl.now)
85
- details[:last_time] = (debug_format_time(job.last_time) rescue nil)
86
- details[:next_time] = (debug_format_time(job.next_time) rescue nil)
87
- details[:job] = job
88
-
89
- details[:opts] = @opts
90
- details[:started_at] = started_at
91
- details[:thread] = thread.inspect
92
- details[:jobs_size] = @jobs.size
93
- details[:work_threads_size] = work_threads.size
94
- details[:work_queue_size] = work_queue.size
95
-
96
- logger.error("Scheduler intercepted an error:", details)
97
-
98
- rescue => e
99
- logger.error("Scheduler failed in #on_error #{e.inspect}")
100
- end
101
-
102
- def debug_format_time(time)
103
- # EtOrbi::EoTime used by (newer) Rufus::Scheduler has to_debug_s https://git.io/JyiPj
104
- time.respond_to?(:to_debug_s) ? time.to_debug_s : time.strftime("%Y-%m-%dT%H:%M:%S.%L")
105
- end
106
- private :debug_format_time
107
-
108
- # @private helper used by JobDecorator
109
- def work_thread_name_prefix
110
- ( @opts[:thread_name] || "#{@thread_key}_scheduler" ) + '_worker-'
111
- end
112
-
113
- protected
114
-
115
- # @overload
116
- def start
117
- ret = super() # @thread[:name] = @opts[:thread_name] || "#{@thread_key}_scheduler"
118
-
119
- # at least set thread.name for easier thread dump analysis
120
- if @thread.is_a?(Thread) && @thread.respond_to?(:name=)
121
- @thread.name = @thread[:name] if @thread[:name]
122
- end
123
-
124
- ret
125
- end
126
-
127
- # @overload
128
- def do_schedule(job_type, t, callable, opts, return_job_instance, block)
129
- job_or_id = super
130
-
131
- job_or_id.extend JobDecorator if return_job_instance
132
-
133
- job_or_id
134
- end
135
-
136
- module JobDecorator
137
-
138
- def start_work_thread
139
- prev_thread_count = @scheduler.work_threads.size
140
-
141
- ret = super() # does not return Thread instance in 3.0
142
-
143
- work_threads = @scheduler.work_threads(:__all_no_cache__)
144
- while prev_thread_count == work_threads.size # very unlikely
145
- Thread.pass
146
- work_threads = @scheduler.work_threads(:__all_no_cache__)
147
- end
148
-
149
- work_thread_name_prefix = @scheduler.work_thread_name_prefix
150
-
151
- work_threads.sort! do |t1, t2|
152
- if t1[:name].nil?
153
- t2[:name].nil? ? 0 : +1 # nils at the end
154
- elsif t2[:name].nil?
155
- t1[:name].nil? ? 0 : -1
156
- else
157
- t1[:name] <=> t2[:name]
158
- end
159
- end
160
-
161
- work_threads.each_with_index do |thread, i|
162
- unless thread[:name]
163
- thread[:name] = "#{work_thread_name_prefix}#{sprintf('%02i', i)}"
164
- thread.name = thread[:name] if thread.respond_to?(:name=)
165
- # e.g. "[oracle]<jdbc_scheduler_worker-00"
166
- end
167
- end
168
-
169
- ret
170
- end
171
-
172
- end
173
-
174
- end
175
- end end end
@@ -1,78 +0,0 @@
1
- # encoding: utf-8
2
- require "logstash/devutils/rspec/spec_helper"
3
- require "logstash/plugin_mixins/jdbc/scheduler"
4
-
5
- describe LogStash::PluginMixins::Jdbc::Scheduler do
6
-
7
- let(:thread_name) { '[test]<jdbc_scheduler' }
8
-
9
- let(:opts) do
10
- { :max_work_threads => 2, :thread_name => thread_name }
11
- end
12
-
13
- subject(:scheduler) { LogStash::PluginMixins::Jdbc::Scheduler.new(opts) }
14
-
15
- after { scheduler.stop(:wait) }
16
-
17
- it "sets scheduler thread name" do
18
- expect( scheduler.thread.name ).to include thread_name
19
- end
20
-
21
- context 'cron schedule' do
22
-
23
- before do
24
- scheduler.schedule_cron('* * * * * *') { sleep 1.25 } # every second
25
- end
26
-
27
- it "sets worker thread names" do
28
- sleep 3.0
29
- threads = scheduler.work_threads
30
- threads.sort! { |t1, t2| (t1.name || '') <=> (t2.name || '') }
31
-
32
- expect( threads.size ).to eql 2
33
- expect( threads.first.name ).to eql "#{thread_name}_worker-00"
34
- expect( threads.last.name ).to eql "#{thread_name}_worker-01"
35
- end
36
-
37
- end
38
-
39
- context 'every 1s' do
40
-
41
- before do
42
- scheduler.schedule_in('1s') { raise 'TEST' } # every second
43
- end
44
-
45
- it "logs errors handled" do
46
- expect( scheduler.logger ).to receive(:error).with /Scheduler intercepted an error/, hash_including(:message => 'TEST')
47
- sleep 1.5
48
- end
49
-
50
- end
51
-
52
- context 'work threads' do
53
-
54
- let(:opts) { super().merge :max_work_threads => 3 }
55
-
56
- let(:counter) { java.util.concurrent.atomic.AtomicLong.new(0) }
57
-
58
- before do
59
- scheduler.schedule_cron('* * * * * *') { counter.increment_and_get; sleep 3.25 } # every second
60
- end
61
-
62
- it "are working" do
63
- sleep(0.05) while counter.get == 0
64
- expect( scheduler.work_threads.size ).to eql 1
65
- sleep(0.05) while counter.get == 1
66
- expect( scheduler.work_threads.size ).to eql 2
67
- sleep(0.05) while counter.get == 2
68
- expect( scheduler.work_threads.size ).to eql 3
69
-
70
- sleep 1.25
71
- expect( scheduler.work_threads.size ).to eql 3
72
- sleep 1.25
73
- expect( scheduler.work_threads.size ).to eql 3
74
- end
75
-
76
- end
77
-
78
- end