logstash-integration-jdbc 5.2.4 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/lib/logstash/filters/jdbc_static.rb +19 -10
- data/lib/logstash/inputs/jdbc.rb +56 -18
- data/lib/logstash/plugin_mixins/jdbc/jdbc.rb +0 -1
- data/lib/logstash/plugin_mixins/jdbc/statement_handler.rb +51 -45
- data/lib/logstash/plugin_mixins/jdbc/value_tracking.rb +1 -1
- data/logstash-integration-jdbc.gemspec +3 -4
- data/spec/filters/jdbc_static_spec.rb +10 -0
- data/spec/inputs/integration/integ_spec.rb +25 -0
- data/spec/inputs/jdbc_spec.rb +85 -61
- metadata +12 -16
- data/lib/logstash/plugin_mixins/jdbc/checked_count_logger.rb +0 -43
- data/lib/logstash/plugin_mixins/jdbc/scheduler.rb +0 -175
- data/spec/plugin_mixins/jdbc/scheduler_spec.rb +0 -78
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d9937a8c1c5ab26acb4cd887f4bdb9dfff51e9348c7187ef2d44ff6055d137ee
|
4
|
+
data.tar.gz: 0042c3e29b5b08abe8baa4f84e4c240960bbc615778df9c77415f8db7985c831
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dbc38871c106b045a500645cf6c4517b30ce15e52bfc68520845e22276efdb4a21df3e4a7709ae682cd6b55ff76e063a615b9fe397795272535aa7c5298a9a92
|
7
|
+
data.tar.gz: 70b8e1dc84ba60d478affb22c93400cb153d4d0adc4904667970220ac03cb9d22b94e28f1b1a5504a30b8592c543d3234ee441d0ed57a31aefef70dfb6c2b9a3
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
## 5.3.0
|
2
|
+
- Refactor: start using scheduler mixin [#110](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/110)
|
3
|
+
|
4
|
+
## 5.2.6
|
5
|
+
- Fix: change default path of 'last_run_metadata_path' to be rooted in the LS data.path folder and not in $HOME [#106](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/106)
|
6
|
+
|
7
|
+
## 5.2.5
|
8
|
+
- Fix: do not execute more queries with debug logging [#109](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/109)
|
9
|
+
|
1
10
|
## 5.2.4
|
2
11
|
- Fix: compatibility with all (>= 3.0) rufus-scheduler versions [#97](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/97)
|
3
12
|
|
@@ -3,7 +3,7 @@ require "logstash-integration-jdbc_jars"
|
|
3
3
|
require "logstash/filters/base"
|
4
4
|
require "logstash/namespace"
|
5
5
|
require "logstash/plugin_mixins/ecs_compatibility_support"
|
6
|
-
require "logstash/plugin_mixins/
|
6
|
+
require "logstash/plugin_mixins/scheduler"
|
7
7
|
require_relative "jdbc/loader"
|
8
8
|
require_relative "jdbc/loader_schedule"
|
9
9
|
require_relative "jdbc/repeating_load_runner"
|
@@ -19,6 +19,8 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
|
|
19
19
|
# adds ecs_compatibility config which could be :disabled or :v1
|
20
20
|
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
|
21
21
|
|
22
|
+
include LogStash::PluginMixins::Scheduler
|
23
|
+
|
22
24
|
config_name "jdbc_static"
|
23
25
|
|
24
26
|
# Define the loaders, an Array of Hashes, to fetch remote data and create local tables.
|
@@ -162,7 +164,6 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
|
|
162
164
|
end
|
163
165
|
|
164
166
|
def close
|
165
|
-
@scheduler.stop if @scheduler
|
166
167
|
@parsed_loaders.each(&:close)
|
167
168
|
@processor.close
|
168
169
|
end
|
@@ -175,9 +176,23 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
|
|
175
176
|
private
|
176
177
|
|
177
178
|
def prepare_data_dir
|
179
|
+
# cleanup existing Derby file left behind in $HOME
|
180
|
+
derby_log = "#{ENV['HOME']}/derby.log"
|
181
|
+
if ::File.exist?(derby_log)
|
182
|
+
begin
|
183
|
+
::File.delete(derby_log)
|
184
|
+
rescue Errno::EPERM => e
|
185
|
+
@logger.warn("Can't delete temporary file #{derby_log} due to access permissions")
|
186
|
+
rescue e
|
187
|
+
@logger.warn("Can't delete temporary file #{derby_log}", {message => e.message})
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
178
191
|
# later, when local persistent databases are allowed set this property to LS_HOME/data/jdbc-static/
|
179
192
|
# must take multi-pipelines into account and more than one config using the same jdbc-static settings
|
180
|
-
|
193
|
+
path_data = Pathname.new(LogStash::SETTINGS.get_value("path.data")).join("plugins", "shared", "derby_home")
|
194
|
+
path_data.mkpath
|
195
|
+
java.lang.System.setProperty("derby.system.home", path_data.to_path)
|
181
196
|
logger.info("derby.system.home is: #{java.lang.System.getProperty("derby.system.home")}")
|
182
197
|
end
|
183
198
|
|
@@ -194,13 +209,7 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
|
|
194
209
|
if @loader_schedule
|
195
210
|
@loader_runner = Jdbc::RepeatingLoadRunner.new(*runner_args)
|
196
211
|
@loader_runner.initial_load
|
197
|
-
@
|
198
|
-
start_cron_scheduler(@loader_schedule, thread_name: "[#{id}]-jdbc_static__scheduler") { @loader_runner.repeated_load }
|
199
|
-
cron_job = @scheduler.cron_jobs.first
|
200
|
-
if cron_job
|
201
|
-
frequency = cron_job.respond_to?(:rough_frequency) ? cron_job.rough_frequency : cron_job.frequency
|
202
|
-
logger.info("Loaders will execute every #{frequency} seconds", loader_schedule: @loader_schedule)
|
203
|
-
end
|
212
|
+
scheduler.cron(@loader_schedule) { @loader_runner.repeated_load }
|
204
213
|
else
|
205
214
|
@loader_runner = Jdbc::SingleLoadRunner.new(*runner_args)
|
206
215
|
@loader_runner.initial_load
|
data/lib/logstash/inputs/jdbc.rb
CHANGED
@@ -3,12 +3,12 @@ require "logstash/inputs/base"
|
|
3
3
|
require "logstash/namespace"
|
4
4
|
require "logstash/plugin_mixins/jdbc/common"
|
5
5
|
require "logstash/plugin_mixins/jdbc/jdbc"
|
6
|
-
require "logstash/plugin_mixins/jdbc/scheduler"
|
7
6
|
require "logstash/plugin_mixins/ecs_compatibility_support"
|
8
7
|
require "logstash/plugin_mixins/ecs_compatibility_support/target_check"
|
9
8
|
require "logstash/plugin_mixins/validator_support/field_reference_validation_adapter"
|
10
|
-
|
11
9
|
require "logstash/plugin_mixins/event_support/event_factory_adapter"
|
10
|
+
require "logstash/plugin_mixins/scheduler"
|
11
|
+
require "fileutils"
|
12
12
|
|
13
13
|
# this require_relative returns early unless the JRuby version is between 9.2.0.0 and 9.2.8.0
|
14
14
|
require_relative "tzinfo_jruby_patch"
|
@@ -146,6 +146,8 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
|
|
146
146
|
# adds :field_reference validator adapter
|
147
147
|
extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
|
148
148
|
|
149
|
+
include LogStash::PluginMixins::Scheduler
|
150
|
+
|
149
151
|
config_name "jdbc"
|
150
152
|
|
151
153
|
# If undefined, Logstash will complain, even if codec is unused.
|
@@ -178,8 +180,10 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
|
|
178
180
|
# exactly once.
|
179
181
|
config :schedule, :validate => :string
|
180
182
|
|
181
|
-
# Path to file with last run time
|
182
|
-
|
183
|
+
# Path to file with last run time.
|
184
|
+
# The default will write file to `<path.data>/plugins/inputs/jdbc/logstash_jdbc_last_run`
|
185
|
+
# NOTE: it must be a file path and not a directory path
|
186
|
+
config :last_run_metadata_path, :validate => :string
|
183
187
|
|
184
188
|
# Use an incremental column value rather than a timestamp
|
185
189
|
config :use_column_value, :validate => :boolean, :default => false
|
@@ -230,12 +234,33 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
|
|
230
234
|
config :target, :validate => :field_reference, :required => false
|
231
235
|
|
232
236
|
attr_reader :database # for test mocking/stubbing
|
237
|
+
attr_reader :last_run_metadata_file_path # path to the file used as last run storage
|
233
238
|
|
234
239
|
public
|
235
240
|
|
236
241
|
def register
|
237
242
|
@logger = self.logger
|
238
|
-
|
243
|
+
|
244
|
+
if @record_last_run
|
245
|
+
if @last_run_metadata_path.nil?
|
246
|
+
logstash_data_path = LogStash::SETTINGS.get_value("path.data")
|
247
|
+
logstash_data_path = Pathname.new(logstash_data_path).join("plugins", "inputs", "jdbc")
|
248
|
+
# Ensure that the filepath exists before writing, since it's deeply nested.
|
249
|
+
logstash_data_path.mkpath
|
250
|
+
logstash_data_file_path = logstash_data_path.join("logstash_jdbc_last_run")
|
251
|
+
|
252
|
+
ensure_default_metadatafile_location(logstash_data_file_path)
|
253
|
+
|
254
|
+
@last_run_metadata_file_path = logstash_data_file_path.to_path
|
255
|
+
else
|
256
|
+
# validate the path is a file and not a directory
|
257
|
+
if Pathname.new(@last_run_metadata_path).directory?
|
258
|
+
raise LogStash::ConfigurationError.new("The \"last_run_metadata_path\" argument must point to a file, received a directory: \"#{last_run_metadata_path}\"")
|
259
|
+
end
|
260
|
+
@last_run_metadata_file_path = @last_run_metadata_path
|
261
|
+
end
|
262
|
+
end
|
263
|
+
|
239
264
|
prepare_jdbc_connection
|
240
265
|
|
241
266
|
if @use_column_value
|
@@ -259,8 +284,8 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
|
|
259
284
|
end
|
260
285
|
end
|
261
286
|
|
262
|
-
set_value_tracker
|
263
|
-
|
287
|
+
set_value_tracker LogStash::PluginMixins::Jdbc::ValueTracking.build_last_value_tracker(self)
|
288
|
+
set_statement_handler LogStash::PluginMixins::Jdbc::StatementHandler.build_statement_handler(self)
|
264
289
|
|
265
290
|
@enable_encoding = !@charset.nil? || !@columns_charset.empty?
|
266
291
|
|
@@ -283,8 +308,8 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
|
|
283
308
|
end # def register
|
284
309
|
|
285
310
|
# test injection points
|
286
|
-
def
|
287
|
-
@statement_handler =
|
311
|
+
def set_statement_handler(handler)
|
312
|
+
@statement_handler = handler
|
288
313
|
end
|
289
314
|
|
290
315
|
def set_value_tracker(instance)
|
@@ -294,22 +319,16 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
|
|
294
319
|
def run(queue)
|
295
320
|
load_driver
|
296
321
|
if @schedule
|
297
|
-
# input thread
|
298
|
-
@
|
299
|
-
|
300
|
-
@scheduler.join
|
322
|
+
# scheduler input thread name example: "[my-oracle]|input|jdbc|scheduler"
|
323
|
+
scheduler.cron(@schedule) { execute_query(queue) }
|
324
|
+
scheduler.join
|
301
325
|
else
|
302
326
|
execute_query(queue)
|
303
327
|
end
|
304
328
|
end # def run
|
305
329
|
|
306
|
-
def close
|
307
|
-
@scheduler.shutdown if @scheduler
|
308
|
-
end
|
309
|
-
|
310
330
|
def stop
|
311
331
|
close_jdbc_connection
|
312
|
-
@scheduler.shutdown(:wait) if @scheduler
|
313
332
|
end
|
314
333
|
|
315
334
|
private
|
@@ -363,4 +382,23 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
|
|
363
382
|
value
|
364
383
|
end
|
365
384
|
end
|
385
|
+
|
386
|
+
def ensure_default_metadatafile_location(metadata_new_path)
|
387
|
+
old_default_path = Pathname.new("#{ENV['HOME']}/.logstash_jdbc_last_run")
|
388
|
+
|
389
|
+
if old_default_path.exist? && !metadata_new_path.exist?
|
390
|
+
# Previous versions of the plugin hosted the state file into $HOME/.logstash_jdbc_last_run.
|
391
|
+
# Copy in the new location
|
392
|
+
FileUtils.cp(old_default_path.to_path, metadata_new_path.to_path)
|
393
|
+
begin
|
394
|
+
# If there is a permission error in the delete of the old file inform the user to give
|
395
|
+
# the correct access rights
|
396
|
+
::File.delete(old_default_path.to_path)
|
397
|
+
@logger.info("Successfully moved the #{old_default_path.to_path} into #{metadata_new_path.to_path}")
|
398
|
+
rescue e
|
399
|
+
@logger.warn("Using new metadata file at #{metadata_new_path.to_path} but #{old_default_path} can't be removed.")
|
400
|
+
end
|
401
|
+
end
|
402
|
+
end
|
403
|
+
|
366
404
|
end end end # class LogStash::Inputs::Jdbc
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module LogStash module PluginMixins module Jdbc
|
4
4
|
class StatementHandler
|
5
|
-
def self.build_statement_handler(plugin
|
5
|
+
def self.build_statement_handler(plugin)
|
6
6
|
if plugin.use_prepared_statements
|
7
7
|
klass = PreparedStatementHandler
|
8
8
|
else
|
@@ -16,27 +16,39 @@ module LogStash module PluginMixins module Jdbc
|
|
16
16
|
klass = NormalStatementHandler
|
17
17
|
end
|
18
18
|
end
|
19
|
-
klass.new(plugin
|
19
|
+
klass.new(plugin)
|
20
20
|
end
|
21
21
|
|
22
|
-
attr_reader :statement, :parameters
|
22
|
+
attr_reader :statement, :parameters
|
23
23
|
|
24
|
-
def initialize(plugin
|
24
|
+
def initialize(plugin)
|
25
25
|
@statement = plugin.statement
|
26
|
-
@statement_logger = statement_logger
|
27
|
-
post_init(plugin)
|
28
26
|
end
|
29
27
|
|
30
28
|
def build_query(db, sql_last_value)
|
31
|
-
# override in subclass
|
29
|
+
fail NotImplementedError # override in subclass
|
32
30
|
end
|
33
31
|
|
34
|
-
def post_init(plugin)
|
35
|
-
# override in subclass, if needed
|
36
|
-
end
|
37
32
|
end
|
38
33
|
|
39
34
|
class NormalStatementHandler < StatementHandler
|
35
|
+
|
36
|
+
attr_reader :parameters
|
37
|
+
|
38
|
+
def initialize(plugin)
|
39
|
+
super(plugin)
|
40
|
+
@parameter_keys = ["sql_last_value"] + plugin.parameters.keys
|
41
|
+
@parameters = plugin.parameters.inject({}) do |hash,(k,v)|
|
42
|
+
case v
|
43
|
+
when LogStash::Timestamp
|
44
|
+
hash[k.to_sym] = v.time
|
45
|
+
else
|
46
|
+
hash[k.to_sym] = v
|
47
|
+
end
|
48
|
+
hash
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
40
52
|
# Performs the query, yielding once per row of data
|
41
53
|
# @param db [Sequel::Database]
|
42
54
|
# @param sql_last_value [Integer|DateTime|Time]
|
@@ -52,27 +64,18 @@ module LogStash module PluginMixins module Jdbc
|
|
52
64
|
|
53
65
|
def build_query(db, sql_last_value)
|
54
66
|
parameters[:sql_last_value] = sql_last_value
|
55
|
-
|
56
|
-
statement_logger.log_statement_parameters(statement, parameters, query)
|
57
|
-
query
|
67
|
+
db[statement, parameters]
|
58
68
|
end
|
59
69
|
|
60
|
-
def post_init(plugin)
|
61
|
-
@parameter_keys = ["sql_last_value"] + plugin.parameters.keys
|
62
|
-
@parameters = plugin.parameters.inject({}) do |hash,(k,v)|
|
63
|
-
case v
|
64
|
-
when LogStash::Timestamp
|
65
|
-
hash[k.to_sym] = v.time
|
66
|
-
else
|
67
|
-
hash[k.to_sym] = v
|
68
|
-
end
|
69
|
-
hash
|
70
|
-
end
|
71
|
-
end
|
72
70
|
end
|
73
71
|
|
74
72
|
class PagedNormalStatementHandler < NormalStatementHandler
|
75
|
-
|
73
|
+
|
74
|
+
def initialize(plugin)
|
75
|
+
super(plugin)
|
76
|
+
@jdbc_page_size = plugin.jdbc_page_size
|
77
|
+
@logger = plugin.logger
|
78
|
+
end
|
76
79
|
|
77
80
|
# Performs the query, respecting our pagination settings, yielding once per row of data
|
78
81
|
# @param db [Sequel::Database]
|
@@ -81,16 +84,22 @@ module LogStash module PluginMixins module Jdbc
|
|
81
84
|
def perform_query(db, sql_last_value)
|
82
85
|
query = build_query(db, sql_last_value)
|
83
86
|
query.each_page(@jdbc_page_size) do |paged_dataset|
|
87
|
+
log_dataset_page(paged_dataset) if @logger.debug?
|
84
88
|
paged_dataset.each do |row|
|
85
89
|
yield row
|
86
90
|
end
|
87
91
|
end
|
88
92
|
end
|
89
93
|
|
90
|
-
|
91
|
-
|
92
|
-
|
94
|
+
private
|
95
|
+
|
96
|
+
# @param paged_dataset [Sequel::Dataset::Pagination] like object
|
97
|
+
def log_dataset_page(paged_dataset)
|
98
|
+
@logger.debug "fetching paged dataset", current_page: paged_dataset.current_page,
|
99
|
+
record_count: paged_dataset.current_page_record_count,
|
100
|
+
total_record_count: paged_dataset.pagination_record_count
|
93
101
|
end
|
102
|
+
|
94
103
|
end
|
95
104
|
|
96
105
|
class ExplicitPagingModeStatementHandler < PagedNormalStatementHandler
|
@@ -101,20 +110,29 @@ module LogStash module PluginMixins module Jdbc
|
|
101
110
|
def perform_query(db, sql_last_value)
|
102
111
|
query = build_query(db, sql_last_value)
|
103
112
|
offset = 0
|
113
|
+
page_size = @jdbc_page_size
|
104
114
|
loop do
|
105
115
|
rows_in_page = 0
|
106
|
-
query.with_sql(query.sql, offset: offset, size:
|
116
|
+
query.with_sql(query.sql, offset: offset, size: page_size).each do |row|
|
107
117
|
yield row
|
108
118
|
rows_in_page += 1
|
109
119
|
end
|
110
|
-
break unless rows_in_page ==
|
111
|
-
offset +=
|
120
|
+
break unless rows_in_page == page_size
|
121
|
+
offset += page_size
|
112
122
|
end
|
113
123
|
end
|
114
124
|
end
|
115
125
|
|
116
126
|
class PreparedStatementHandler < StatementHandler
|
117
|
-
attr_reader :name, :bind_values_array, :statement_prepared, :prepared
|
127
|
+
attr_reader :name, :bind_values_array, :statement_prepared, :prepared, :parameters
|
128
|
+
|
129
|
+
def initialize(plugin)
|
130
|
+
super(plugin)
|
131
|
+
@name = plugin.prepared_statement_name.to_sym
|
132
|
+
@bind_values_array = plugin.prepared_statement_bind_values
|
133
|
+
@parameters = plugin.parameters
|
134
|
+
@statement_prepared = Concurrent::AtomicBoolean.new(false)
|
135
|
+
end
|
118
136
|
|
119
137
|
# Performs the query, ignoring our pagination settings, yielding once per row of data
|
120
138
|
# @param db [Sequel::Database]
|
@@ -142,7 +160,6 @@ module LogStash module PluginMixins module Jdbc
|
|
142
160
|
db.set_prepared_statement(name, prepared)
|
143
161
|
end
|
144
162
|
bind_value_sql_last_value(sql_last_value)
|
145
|
-
statement_logger.log_statement_parameters(statement, parameters, nil)
|
146
163
|
begin
|
147
164
|
db.call(name, parameters)
|
148
165
|
rescue => e
|
@@ -153,17 +170,6 @@ module LogStash module PluginMixins module Jdbc
|
|
153
170
|
end
|
154
171
|
end
|
155
172
|
|
156
|
-
def post_init(plugin)
|
157
|
-
# don't log statement count when using prepared statements for now...
|
158
|
-
# needs enhancement to allow user to supply a bindable count prepared statement in settings.
|
159
|
-
@statement_logger.disable_count
|
160
|
-
|
161
|
-
@name = plugin.prepared_statement_name.to_sym
|
162
|
-
@bind_values_array = plugin.prepared_statement_bind_values
|
163
|
-
@parameters = plugin.parameters
|
164
|
-
@statement_prepared = Concurrent::AtomicBoolean.new(false)
|
165
|
-
end
|
166
|
-
|
167
173
|
def create_bind_values_hash
|
168
174
|
hash = {}
|
169
175
|
bind_values_array.each_with_index {|v,i| hash[:"p#{i}"] = v}
|
@@ -5,7 +5,7 @@ module LogStash module PluginMixins module Jdbc
|
|
5
5
|
class ValueTracking
|
6
6
|
|
7
7
|
def self.build_last_value_tracker(plugin)
|
8
|
-
handler = plugin.record_last_run ? FileHandler.new(plugin.
|
8
|
+
handler = plugin.record_last_run ? FileHandler.new(plugin.last_run_metadata_file_path) : NullFileHandler.new(plugin.last_run_metadata_file_path)
|
9
9
|
if plugin.clean_run
|
10
10
|
handler.clean
|
11
11
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-integration-jdbc'
|
3
|
-
s.version = '5.
|
3
|
+
s.version = '5.3.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Integration with JDBC - input and filter plugins"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -34,12 +34,11 @@ Gem::Specification.new do |s|
|
|
34
34
|
|
35
35
|
s.add_runtime_dependency 'tzinfo'
|
36
36
|
s.add_runtime_dependency 'tzinfo-data'
|
37
|
-
|
38
|
-
# but works with newer rufus-scheduler >= 3.5 as well
|
39
|
-
s.add_runtime_dependency 'rufus-scheduler'
|
37
|
+
|
40
38
|
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
|
41
39
|
s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
|
42
40
|
s.add_runtime_dependency "logstash-mixin-event_support", '~> 1.0'
|
41
|
+
s.add_runtime_dependency "logstash-mixin-scheduler", '~> 1.0'
|
43
42
|
|
44
43
|
s.add_development_dependency "childprocess"
|
45
44
|
s.add_development_dependency 'logstash-devutils', '>= 2.3'
|
@@ -5,6 +5,7 @@ require "sequel"
|
|
5
5
|
require "sequel/adapters/jdbc"
|
6
6
|
require "stud/temporary"
|
7
7
|
require "timecop"
|
8
|
+
require "pathname"
|
8
9
|
|
9
10
|
# LogStash::Logging::Logger::configure_logging("WARN")
|
10
11
|
|
@@ -85,6 +86,15 @@ module LogStash module Filters
|
|
85
86
|
|
86
87
|
let(:ipaddr) { ".3.1.1" }
|
87
88
|
|
89
|
+
describe "verify derby path property" do
|
90
|
+
it "should be set into Logstash data path" do
|
91
|
+
plugin.register
|
92
|
+
|
93
|
+
expected = Pathname.new(LogStash::SETTINGS.get_value("path.data")).join("plugins", "shared", "derby_home").to_path
|
94
|
+
expect(java.lang.System.getProperty("derby.system.home")).to eq(expected)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
88
98
|
describe "non scheduled operation" do
|
89
99
|
after { plugin.close }
|
90
100
|
|
@@ -41,6 +41,31 @@ describe LogStash::Inputs::Jdbc, :integration => true do
|
|
41
41
|
expect(event.get('first_name')).to eq("Mark")
|
42
42
|
expect(event.get('last_name')).to eq("Guckenheimer")
|
43
43
|
end
|
44
|
+
|
45
|
+
context 'with paging' do
|
46
|
+
let(:settings) do
|
47
|
+
super().merge 'jdbc_paging_enabled' => true, 'jdbc_page_size' => 1,
|
48
|
+
"statement" => 'SELECT * FROM "employee" WHERE EMP_NO >= :p1 ORDER BY EMP_NO',
|
49
|
+
'parameters' => { 'p1' => 0 }
|
50
|
+
end
|
51
|
+
|
52
|
+
before do # change plugin logger level to debug - to exercise logging
|
53
|
+
logger = plugin.class.name.gsub('::', '.').downcase
|
54
|
+
logger = org.apache.logging.log4j.LogManager.getLogger(logger)
|
55
|
+
@prev_logger_level = [ logger.getName, logger.getLevel ]
|
56
|
+
org.apache.logging.log4j.core.config.Configurator.setLevel logger.getName, org.apache.logging.log4j.Level::DEBUG
|
57
|
+
end
|
58
|
+
|
59
|
+
after do
|
60
|
+
org.apache.logging.log4j.core.config.Configurator.setLevel *@prev_logger_level
|
61
|
+
end
|
62
|
+
|
63
|
+
it "should populate the event with database entries" do
|
64
|
+
plugin.run(queue)
|
65
|
+
event = queue.pop
|
66
|
+
expect(event.get('first_name')).to eq('David')
|
67
|
+
end
|
68
|
+
end
|
44
69
|
end
|
45
70
|
|
46
71
|
context "when supplying a non-existent library" do
|
data/spec/inputs/jdbc_spec.rb
CHANGED
@@ -9,6 +9,7 @@ require "timecop"
|
|
9
9
|
require "stud/temporary"
|
10
10
|
require "time"
|
11
11
|
require "date"
|
12
|
+
require "pathname"
|
12
13
|
|
13
14
|
# We do not need to set TZ env var anymore because we can have 'Sequel.application_timezone' set to utc by default now.
|
14
15
|
|
@@ -51,6 +52,9 @@ describe LogStash::Inputs::Jdbc do
|
|
51
52
|
db.drop_table(:types_table)
|
52
53
|
db.drop_table(:test1_table)
|
53
54
|
end
|
55
|
+
|
56
|
+
last_run_default_path = LogStash::SETTINGS.get_value("path.data")
|
57
|
+
FileUtils.rm_f("#{last_run_default_path}/plugins/inputs/jdbc/logstash_jdbc_last_run")
|
54
58
|
end
|
55
59
|
|
56
60
|
context "when registering and tearing down" do
|
@@ -247,18 +251,6 @@ describe LogStash::Inputs::Jdbc do
|
|
247
251
|
Timecop.return
|
248
252
|
end
|
249
253
|
|
250
|
-
it "cleans up scheduler resources on close" do
|
251
|
-
runner = Thread.new do
|
252
|
-
plugin.run(queue)
|
253
|
-
end
|
254
|
-
sleep 1
|
255
|
-
plugin.do_close
|
256
|
-
|
257
|
-
scheduler = plugin.instance_variable_get(:@scheduler)
|
258
|
-
expect(scheduler).to_not be_nil
|
259
|
-
expect(scheduler.down?).to be_truthy
|
260
|
-
end
|
261
|
-
|
262
254
|
end
|
263
255
|
|
264
256
|
context "when scheduling and previous runs are to be preserved" do
|
@@ -1114,6 +1106,86 @@ describe LogStash::Inputs::Jdbc do
|
|
1114
1106
|
end
|
1115
1107
|
end
|
1116
1108
|
|
1109
|
+
context "when state is persisted" do
|
1110
|
+
context "to file" do
|
1111
|
+
let(:settings) do
|
1112
|
+
{
|
1113
|
+
"statement" => "SELECT * FROM test_table",
|
1114
|
+
"record_last_run" => true
|
1115
|
+
}
|
1116
|
+
end
|
1117
|
+
|
1118
|
+
before do
|
1119
|
+
plugin.register
|
1120
|
+
end
|
1121
|
+
|
1122
|
+
after do
|
1123
|
+
plugin.stop
|
1124
|
+
end
|
1125
|
+
|
1126
|
+
context "with default last_run_metadata_path" do
|
1127
|
+
it "should save state in data.data subpath" do
|
1128
|
+
path = LogStash::SETTINGS.get_value("path.data")
|
1129
|
+
expect(plugin.last_run_metadata_file_path).to start_with(path)
|
1130
|
+
end
|
1131
|
+
end
|
1132
|
+
|
1133
|
+
context "with customized last_run_metadata_path" do
|
1134
|
+
let(:settings) { super().merge({ "last_run_metadata_path" => Stud::Temporary.pathname })}
|
1135
|
+
|
1136
|
+
it "should save state in data.data subpath" do
|
1137
|
+
expect(plugin.last_run_metadata_file_path).to start_with(settings["last_run_metadata_path"])
|
1138
|
+
end
|
1139
|
+
end
|
1140
|
+
end
|
1141
|
+
|
1142
|
+
context "with customized last_run_metadata_path point to directory" do
|
1143
|
+
let(:settings) do
|
1144
|
+
path = Stud::Temporary.pathname
|
1145
|
+
Pathname.new(path).tap {|path| path.mkpath}
|
1146
|
+
super().merge({ "last_run_metadata_path" => path})
|
1147
|
+
end
|
1148
|
+
|
1149
|
+
it "raise configuration error" do
|
1150
|
+
expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
|
1151
|
+
end
|
1152
|
+
end
|
1153
|
+
end
|
1154
|
+
|
1155
|
+
context "update the previous default last_run_metadata_path" do
|
1156
|
+
let(:settings) do
|
1157
|
+
{
|
1158
|
+
"statement" => "SELECT * FROM test_table",
|
1159
|
+
"record_last_run" => true
|
1160
|
+
}
|
1161
|
+
end
|
1162
|
+
|
1163
|
+
let(:fake_home) do
|
1164
|
+
path = Stud::Temporary.pathname
|
1165
|
+
Pathname.new(path).tap {|path| path.mkpath}
|
1166
|
+
path
|
1167
|
+
end
|
1168
|
+
|
1169
|
+
context "when a file exists" do
|
1170
|
+
before do
|
1171
|
+
# in a faked HOME folder save a valid previous last_run metadata file
|
1172
|
+
allow(ENV).to receive(:[]).with('HOME').and_return(fake_home)
|
1173
|
+
File.open("#{ENV['HOME']}/.logstash_jdbc_last_run", 'w') do |file|
|
1174
|
+
file.write("--- !ruby/object:DateTime '2022-03-08 08:10:00.486889000 Z'")
|
1175
|
+
end
|
1176
|
+
end
|
1177
|
+
|
1178
|
+
it "should be moved" do
|
1179
|
+
plugin.register
|
1180
|
+
|
1181
|
+
expect(::File.exist?("#{ENV['HOME']}/.logstash_jdbc_last_run")).to be false
|
1182
|
+
path = LogStash::SETTINGS.get_value("path.data")
|
1183
|
+
full_path = "#{path}/plugins/inputs/jdbc/logstash_jdbc_last_run"
|
1184
|
+
expect(::File.exist?(full_path)).to be true
|
1185
|
+
end
|
1186
|
+
end
|
1187
|
+
end
|
1188
|
+
|
1117
1189
|
context "when setting fetch size" do
|
1118
1190
|
|
1119
1191
|
let(:settings) do
|
@@ -1432,54 +1504,6 @@ describe LogStash::Inputs::Jdbc do
|
|
1432
1504
|
end
|
1433
1505
|
end
|
1434
1506
|
|
1435
|
-
context "when debug logging and a count query raises a count related error" do
|
1436
|
-
let(:settings) do
|
1437
|
-
{ "statement" => "SELECT * from types_table" }
|
1438
|
-
end
|
1439
|
-
let(:logger) { double("logger", :debug? => true) }
|
1440
|
-
let(:statement_logger) { LogStash::PluginMixins::Jdbc::CheckedCountLogger.new(logger) }
|
1441
|
-
let(:value_tracker) { double("value tracker", :set_value => nil, :write => nil) }
|
1442
|
-
let(:msg) { 'Java::JavaSql::SQLSyntaxErrorException: Dynamic SQL Error; SQL error code = -104; Token unknown - line 1, column 105; LIMIT [SQLState:42000, ISC error code:335544634]' }
|
1443
|
-
let(:error_args) do
|
1444
|
-
{"exception" => msg}
|
1445
|
-
end
|
1446
|
-
|
1447
|
-
before do
|
1448
|
-
db << "INSERT INTO types_table (num, string, started_at, custom_time, ranking) VALUES (1, 'A test', '1999-12-31', '1999-12-31 23:59:59', 95.67)"
|
1449
|
-
plugin.register
|
1450
|
-
plugin.set_statement_logger(statement_logger)
|
1451
|
-
plugin.set_value_tracker(value_tracker)
|
1452
|
-
allow(value_tracker).to receive(:value).and_return("bar")
|
1453
|
-
allow(statement_logger).to receive(:execute_count).once.and_raise(StandardError.new(msg))
|
1454
|
-
end
|
1455
|
-
|
1456
|
-
after do
|
1457
|
-
plugin.stop
|
1458
|
-
end
|
1459
|
-
|
1460
|
-
context "if the count query raises an error" do
|
1461
|
-
it "should log a debug line without a count key as its unknown whether a count works at this stage" do
|
1462
|
-
expect(logger).to receive(:warn).once.with("Attempting a count query raised an error, the generated count statement is most likely incorrect but check networking, authentication or your statement syntax", error_args)
|
1463
|
-
expect(logger).to receive(:warn).once.with("Ongoing count statement generation is being prevented")
|
1464
|
-
expect(logger).to receive(:debug).once.with("Executing JDBC query", :statement => settings["statement"], :parameters => {:sql_last_value=>"bar"})
|
1465
|
-
plugin.run(queue)
|
1466
|
-
queue.pop
|
1467
|
-
end
|
1468
|
-
|
1469
|
-
it "should create an event normally" do
|
1470
|
-
allow(logger).to receive(:warn)
|
1471
|
-
allow(logger).to receive(:debug)
|
1472
|
-
plugin.run(queue)
|
1473
|
-
event = queue.pop
|
1474
|
-
expect(event.get("num")).to eq(1)
|
1475
|
-
expect(event.get("string")).to eq("A test")
|
1476
|
-
expect(event.get("started_at")).to be_a_logstash_timestamp_equivalent_to("1999-12-31T00:00:00.000Z")
|
1477
|
-
expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("1999-12-31T23:59:59.000Z")
|
1478
|
-
expect(event.get("ranking").to_f).to eq(95.67)
|
1479
|
-
end
|
1480
|
-
end
|
1481
|
-
end
|
1482
|
-
|
1483
1507
|
context "when an unreadable jdbc_driver_path entry is present" do
|
1484
1508
|
let(:driver_jar_path) do
|
1485
1509
|
jar_file = $CLASSPATH.find { |name| name.index(Jdbc::Derby.driver_jar) }
|
@@ -1681,7 +1705,7 @@ describe LogStash::Inputs::Jdbc do
|
|
1681
1705
|
let(:jdbc_driver_class) { "org.apache.NonExistentDriver" }
|
1682
1706
|
it "raises a loading error" do
|
1683
1707
|
expect { plugin.send(:load_driver) }.to raise_error LogStash::PluginLoadingError,
|
1684
|
-
/
|
1708
|
+
/ClassNotFoundException: org.apache.NonExistentDriver/
|
1685
1709
|
end
|
1686
1710
|
end
|
1687
1711
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-integration-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-06-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -131,38 +131,38 @@ dependencies:
|
|
131
131
|
- !ruby/object:Gem::Dependency
|
132
132
|
requirement: !ruby/object:Gem::Requirement
|
133
133
|
requirements:
|
134
|
-
- - "
|
134
|
+
- - "~>"
|
135
135
|
- !ruby/object:Gem::Version
|
136
|
-
version: '
|
137
|
-
name:
|
136
|
+
version: '1.3'
|
137
|
+
name: logstash-mixin-ecs_compatibility_support
|
138
138
|
prerelease: false
|
139
139
|
type: :runtime
|
140
140
|
version_requirements: !ruby/object:Gem::Requirement
|
141
141
|
requirements:
|
142
|
-
- - "
|
142
|
+
- - "~>"
|
143
143
|
- !ruby/object:Gem::Version
|
144
|
-
version: '
|
144
|
+
version: '1.3'
|
145
145
|
- !ruby/object:Gem::Dependency
|
146
146
|
requirement: !ruby/object:Gem::Requirement
|
147
147
|
requirements:
|
148
148
|
- - "~>"
|
149
149
|
- !ruby/object:Gem::Version
|
150
|
-
version: '1.
|
151
|
-
name: logstash-mixin-
|
150
|
+
version: '1.0'
|
151
|
+
name: logstash-mixin-validator_support
|
152
152
|
prerelease: false
|
153
153
|
type: :runtime
|
154
154
|
version_requirements: !ruby/object:Gem::Requirement
|
155
155
|
requirements:
|
156
156
|
- - "~>"
|
157
157
|
- !ruby/object:Gem::Version
|
158
|
-
version: '1.
|
158
|
+
version: '1.0'
|
159
159
|
- !ruby/object:Gem::Dependency
|
160
160
|
requirement: !ruby/object:Gem::Requirement
|
161
161
|
requirements:
|
162
162
|
- - "~>"
|
163
163
|
- !ruby/object:Gem::Version
|
164
164
|
version: '1.0'
|
165
|
-
name: logstash-mixin-
|
165
|
+
name: logstash-mixin-event_support
|
166
166
|
prerelease: false
|
167
167
|
type: :runtime
|
168
168
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -176,7 +176,7 @@ dependencies:
|
|
176
176
|
- - "~>"
|
177
177
|
- !ruby/object:Gem::Version
|
178
178
|
version: '1.0'
|
179
|
-
name: logstash-mixin-
|
179
|
+
name: logstash-mixin-scheduler
|
180
180
|
prerelease: false
|
181
181
|
type: :runtime
|
182
182
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -275,10 +275,8 @@ files:
|
|
275
275
|
- lib/logstash/filters/jdbc_streaming.rb
|
276
276
|
- lib/logstash/inputs/jdbc.rb
|
277
277
|
- lib/logstash/inputs/tzinfo_jruby_patch.rb
|
278
|
-
- lib/logstash/plugin_mixins/jdbc/checked_count_logger.rb
|
279
278
|
- lib/logstash/plugin_mixins/jdbc/common.rb
|
280
279
|
- lib/logstash/plugin_mixins/jdbc/jdbc.rb
|
281
|
-
- lib/logstash/plugin_mixins/jdbc/scheduler.rb
|
282
280
|
- lib/logstash/plugin_mixins/jdbc/statement_handler.rb
|
283
281
|
- lib/logstash/plugin_mixins/jdbc/value_tracking.rb
|
284
282
|
- lib/logstash/plugin_mixins/jdbc_streaming.rb
|
@@ -307,7 +305,6 @@ files:
|
|
307
305
|
- spec/helpers/derbyrun.jar
|
308
306
|
- spec/inputs/integration/integ_spec.rb
|
309
307
|
- spec/inputs/jdbc_spec.rb
|
310
|
-
- spec/plugin_mixins/jdbc/scheduler_spec.rb
|
311
308
|
- spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb
|
312
309
|
- vendor/jar-dependencies/org/apache/derby/derby/10.14.1.0/derby-10.14.1.0.jar
|
313
310
|
- vendor/jar-dependencies/org/apache/derby/derbyclient/10.14.1.0/derbyclient-10.14.1.0.jar
|
@@ -360,5 +357,4 @@ test_files:
|
|
360
357
|
- spec/helpers/derbyrun.jar
|
361
358
|
- spec/inputs/integration/integ_spec.rb
|
362
359
|
- spec/inputs/jdbc_spec.rb
|
363
|
-
- spec/plugin_mixins/jdbc/scheduler_spec.rb
|
364
360
|
- spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb
|
@@ -1,43 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
|
-
module LogStash module PluginMixins module Jdbc
|
4
|
-
class CheckedCountLogger
|
5
|
-
def initialize(logger)
|
6
|
-
@logger = logger
|
7
|
-
@needs_check = true
|
8
|
-
@count_is_supported = false
|
9
|
-
@in_debug = @logger.debug?
|
10
|
-
end
|
11
|
-
|
12
|
-
def disable_count
|
13
|
-
@needs_check = false
|
14
|
-
@count_is_supported = false
|
15
|
-
end
|
16
|
-
|
17
|
-
def log_statement_parameters(statement, parameters, query)
|
18
|
-
return unless @in_debug
|
19
|
-
check_count_query(query) if @needs_check && query
|
20
|
-
if @count_is_supported
|
21
|
-
@logger.debug("Executing JDBC query", :statement => statement, :parameters => parameters, :count => execute_count(query))
|
22
|
-
else
|
23
|
-
@logger.debug("Executing JDBC query", :statement => statement, :parameters => parameters)
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
def check_count_query(query)
|
28
|
-
@needs_check = false
|
29
|
-
begin
|
30
|
-
execute_count(query)
|
31
|
-
@count_is_supported = true
|
32
|
-
rescue Exception => e
|
33
|
-
@logger.warn("Attempting a count query raised an error, the generated count statement is most likely incorrect but check networking, authentication or your statement syntax", "exception" => e.message)
|
34
|
-
@logger.warn("Ongoing count statement generation is being prevented")
|
35
|
-
@count_is_supported = false
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
def execute_count(query)
|
40
|
-
query.count
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end end end
|
@@ -1,175 +0,0 @@
|
|
1
|
-
require 'rufus/scheduler'
|
2
|
-
|
3
|
-
require 'logstash/util/loggable'
|
4
|
-
|
5
|
-
module LogStash module PluginMixins module Jdbc
|
6
|
-
class Scheduler < Rufus::Scheduler
|
7
|
-
|
8
|
-
include LogStash::Util::Loggable
|
9
|
-
|
10
|
-
# Rufus::Scheduler >= 3.4 moved the Time impl into a gem EoTime = ::EtOrbi::EoTime`
|
11
|
-
# Rufus::Scheduler 3.1 - 3.3 using it's own Time impl `Rufus::Scheduler::ZoTime`
|
12
|
-
TimeImpl = defined?(Rufus::Scheduler::EoTime) ? Rufus::Scheduler::EoTime :
|
13
|
-
(defined?(Rufus::Scheduler::ZoTime) ? Rufus::Scheduler::ZoTime : ::Time)
|
14
|
-
|
15
|
-
# @param cron [String] cron-line
|
16
|
-
# @param opts [Hash] scheduler options
|
17
|
-
# @return scheduler instance
|
18
|
-
def self.start_cron_scheduler(cron, opts = {}, &block)
|
19
|
-
unless block_given?
|
20
|
-
raise ArgumentError, 'missing (cron scheduler) block - worker task to execute'
|
21
|
-
end
|
22
|
-
scheduler = new_scheduler(opts)
|
23
|
-
scheduler.schedule_cron(cron, &block)
|
24
|
-
scheduler
|
25
|
-
end
|
26
|
-
|
27
|
-
# @param opts [Hash] scheduler options
|
28
|
-
# @return scheduler instance
|
29
|
-
def self.new_scheduler(opts)
|
30
|
-
unless opts.key?(:thread_name)
|
31
|
-
raise ArgumentError, 'thread_name: option is required to be able to distinguish multiple scheduler threads'
|
32
|
-
end
|
33
|
-
opts[:max_work_threads] ||= 1
|
34
|
-
# amount the scheduler thread sleeps between checking whether jobs
|
35
|
-
# should trigger, default is 0.3 which is a bit too often ...
|
36
|
-
# in theory the cron expression '* * * * * *' supports running jobs
|
37
|
-
# every second but this is very rare, we could potentially go higher
|
38
|
-
opts[:frequency] ||= 1.0
|
39
|
-
|
40
|
-
new(opts)
|
41
|
-
end
|
42
|
-
|
43
|
-
# @overload
|
44
|
-
def timeout_jobs
|
45
|
-
# Rufus relies on `Thread.list` which is a blocking operation and with many schedulers
|
46
|
-
# (and threads) within LS will have a negative impact on performance as scheduler
|
47
|
-
# threads will end up waiting to obtain the `Thread.list` lock.
|
48
|
-
#
|
49
|
-
# However, this isn't necessary we can easily detect whether there are any jobs
|
50
|
-
# that might need to timeout: only when `@opts[:timeout]` is set causes worker thread(s)
|
51
|
-
# to have a `Thread.current[:rufus_scheduler_timeout]` that is not nil
|
52
|
-
return unless @opts[:timeout]
|
53
|
-
super
|
54
|
-
end
|
55
|
-
|
56
|
-
# @overload
|
57
|
-
def work_threads(query = :all)
|
58
|
-
if query == :__all_no_cache__ # special case from JobDecorator#start_work_thread
|
59
|
-
@_work_threads = nil # when a new worker thread is being added reset
|
60
|
-
return super(:all)
|
61
|
-
end
|
62
|
-
|
63
|
-
# Gets executed every time a job is triggered, we're going to cache the
|
64
|
-
# worker threads for this scheduler (to avoid `Thread.list`) - they only
|
65
|
-
# change when a new thread is being started from #start_work_thread ...
|
66
|
-
work_threads = @_work_threads
|
67
|
-
if work_threads.nil?
|
68
|
-
work_threads = threads.select { |t| t[:rufus_scheduler_work_thread] }
|
69
|
-
@_work_threads = work_threads
|
70
|
-
end
|
71
|
-
|
72
|
-
case query
|
73
|
-
when :active then work_threads.select { |t| t[:rufus_scheduler_job] }
|
74
|
-
when :vacant then work_threads.reject { |t| t[:rufus_scheduler_job] }
|
75
|
-
else work_threads
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
# @overload
|
80
|
-
def on_error(job, err)
|
81
|
-
details = { exception: err.class, message: err.message, backtrace: err.backtrace }
|
82
|
-
details[:cause] = err.cause if err.cause
|
83
|
-
|
84
|
-
details[:now] = debug_format_time(TimeImpl.now)
|
85
|
-
details[:last_time] = (debug_format_time(job.last_time) rescue nil)
|
86
|
-
details[:next_time] = (debug_format_time(job.next_time) rescue nil)
|
87
|
-
details[:job] = job
|
88
|
-
|
89
|
-
details[:opts] = @opts
|
90
|
-
details[:started_at] = started_at
|
91
|
-
details[:thread] = thread.inspect
|
92
|
-
details[:jobs_size] = @jobs.size
|
93
|
-
details[:work_threads_size] = work_threads.size
|
94
|
-
details[:work_queue_size] = work_queue.size
|
95
|
-
|
96
|
-
logger.error("Scheduler intercepted an error:", details)
|
97
|
-
|
98
|
-
rescue => e
|
99
|
-
logger.error("Scheduler failed in #on_error #{e.inspect}")
|
100
|
-
end
|
101
|
-
|
102
|
-
def debug_format_time(time)
|
103
|
-
# EtOrbi::EoTime used by (newer) Rufus::Scheduler has to_debug_s https://git.io/JyiPj
|
104
|
-
time.respond_to?(:to_debug_s) ? time.to_debug_s : time.strftime("%Y-%m-%dT%H:%M:%S.%L")
|
105
|
-
end
|
106
|
-
private :debug_format_time
|
107
|
-
|
108
|
-
# @private helper used by JobDecorator
|
109
|
-
def work_thread_name_prefix
|
110
|
-
( @opts[:thread_name] || "#{@thread_key}_scheduler" ) + '_worker-'
|
111
|
-
end
|
112
|
-
|
113
|
-
protected
|
114
|
-
|
115
|
-
# @overload
|
116
|
-
def start
|
117
|
-
ret = super() # @thread[:name] = @opts[:thread_name] || "#{@thread_key}_scheduler"
|
118
|
-
|
119
|
-
# at least set thread.name for easier thread dump analysis
|
120
|
-
if @thread.is_a?(Thread) && @thread.respond_to?(:name=)
|
121
|
-
@thread.name = @thread[:name] if @thread[:name]
|
122
|
-
end
|
123
|
-
|
124
|
-
ret
|
125
|
-
end
|
126
|
-
|
127
|
-
# @overload
|
128
|
-
def do_schedule(job_type, t, callable, opts, return_job_instance, block)
|
129
|
-
job_or_id = super
|
130
|
-
|
131
|
-
job_or_id.extend JobDecorator if return_job_instance
|
132
|
-
|
133
|
-
job_or_id
|
134
|
-
end
|
135
|
-
|
136
|
-
module JobDecorator
|
137
|
-
|
138
|
-
def start_work_thread
|
139
|
-
prev_thread_count = @scheduler.work_threads.size
|
140
|
-
|
141
|
-
ret = super() # does not return Thread instance in 3.0
|
142
|
-
|
143
|
-
work_threads = @scheduler.work_threads(:__all_no_cache__)
|
144
|
-
while prev_thread_count == work_threads.size # very unlikely
|
145
|
-
Thread.pass
|
146
|
-
work_threads = @scheduler.work_threads(:__all_no_cache__)
|
147
|
-
end
|
148
|
-
|
149
|
-
work_thread_name_prefix = @scheduler.work_thread_name_prefix
|
150
|
-
|
151
|
-
work_threads.sort! do |t1, t2|
|
152
|
-
if t1[:name].nil?
|
153
|
-
t2[:name].nil? ? 0 : +1 # nils at the end
|
154
|
-
elsif t2[:name].nil?
|
155
|
-
t1[:name].nil? ? 0 : -1
|
156
|
-
else
|
157
|
-
t1[:name] <=> t2[:name]
|
158
|
-
end
|
159
|
-
end
|
160
|
-
|
161
|
-
work_threads.each_with_index do |thread, i|
|
162
|
-
unless thread[:name]
|
163
|
-
thread[:name] = "#{work_thread_name_prefix}#{sprintf('%02i', i)}"
|
164
|
-
thread.name = thread[:name] if thread.respond_to?(:name=)
|
165
|
-
# e.g. "[oracle]<jdbc_scheduler_worker-00"
|
166
|
-
end
|
167
|
-
end
|
168
|
-
|
169
|
-
ret
|
170
|
-
end
|
171
|
-
|
172
|
-
end
|
173
|
-
|
174
|
-
end
|
175
|
-
end end end
|
@@ -1,78 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require "logstash/devutils/rspec/spec_helper"
|
3
|
-
require "logstash/plugin_mixins/jdbc/scheduler"
|
4
|
-
|
5
|
-
describe LogStash::PluginMixins::Jdbc::Scheduler do
|
6
|
-
|
7
|
-
let(:thread_name) { '[test]<jdbc_scheduler' }
|
8
|
-
|
9
|
-
let(:opts) do
|
10
|
-
{ :max_work_threads => 2, :thread_name => thread_name }
|
11
|
-
end
|
12
|
-
|
13
|
-
subject(:scheduler) { LogStash::PluginMixins::Jdbc::Scheduler.new(opts) }
|
14
|
-
|
15
|
-
after { scheduler.stop(:wait) }
|
16
|
-
|
17
|
-
it "sets scheduler thread name" do
|
18
|
-
expect( scheduler.thread.name ).to include thread_name
|
19
|
-
end
|
20
|
-
|
21
|
-
context 'cron schedule' do
|
22
|
-
|
23
|
-
before do
|
24
|
-
scheduler.schedule_cron('* * * * * *') { sleep 1.25 } # every second
|
25
|
-
end
|
26
|
-
|
27
|
-
it "sets worker thread names" do
|
28
|
-
sleep 3.0
|
29
|
-
threads = scheduler.work_threads
|
30
|
-
threads.sort! { |t1, t2| (t1.name || '') <=> (t2.name || '') }
|
31
|
-
|
32
|
-
expect( threads.size ).to eql 2
|
33
|
-
expect( threads.first.name ).to eql "#{thread_name}_worker-00"
|
34
|
-
expect( threads.last.name ).to eql "#{thread_name}_worker-01"
|
35
|
-
end
|
36
|
-
|
37
|
-
end
|
38
|
-
|
39
|
-
context 'every 1s' do
|
40
|
-
|
41
|
-
before do
|
42
|
-
scheduler.schedule_in('1s') { raise 'TEST' } # every second
|
43
|
-
end
|
44
|
-
|
45
|
-
it "logs errors handled" do
|
46
|
-
expect( scheduler.logger ).to receive(:error).with /Scheduler intercepted an error/, hash_including(:message => 'TEST')
|
47
|
-
sleep 1.5
|
48
|
-
end
|
49
|
-
|
50
|
-
end
|
51
|
-
|
52
|
-
context 'work threads' do
|
53
|
-
|
54
|
-
let(:opts) { super().merge :max_work_threads => 3 }
|
55
|
-
|
56
|
-
let(:counter) { java.util.concurrent.atomic.AtomicLong.new(0) }
|
57
|
-
|
58
|
-
before do
|
59
|
-
scheduler.schedule_cron('* * * * * *') { counter.increment_and_get; sleep 3.25 } # every second
|
60
|
-
end
|
61
|
-
|
62
|
-
it "are working" do
|
63
|
-
sleep(0.05) while counter.get == 0
|
64
|
-
expect( scheduler.work_threads.size ).to eql 1
|
65
|
-
sleep(0.05) while counter.get == 1
|
66
|
-
expect( scheduler.work_threads.size ).to eql 2
|
67
|
-
sleep(0.05) while counter.get == 2
|
68
|
-
expect( scheduler.work_threads.size ).to eql 3
|
69
|
-
|
70
|
-
sleep 1.25
|
71
|
-
expect( scheduler.work_threads.size ).to eql 3
|
72
|
-
sleep 1.25
|
73
|
-
expect( scheduler.work_threads.size ).to eql 3
|
74
|
-
end
|
75
|
-
|
76
|
-
end
|
77
|
-
|
78
|
-
end
|