logstash-integration-jdbc 5.2.6 → 5.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c9cc41a07a8cb2f7e4b9a6993dd4d0cd69761b3fa95ba81238d0dd6267367311
4
- data.tar.gz: 32866a340d87d91734586b3324a1349437abc624deb6ff9883ab998b0a4524a6
3
+ metadata.gz: d9937a8c1c5ab26acb4cd887f4bdb9dfff51e9348c7187ef2d44ff6055d137ee
4
+ data.tar.gz: 0042c3e29b5b08abe8baa4f84e4c240960bbc615778df9c77415f8db7985c831
5
5
  SHA512:
6
- metadata.gz: b3513ba2d9ade38a829ec0b0c47ad6c0978d0256e03fb4ac26e136a064db65b08a19b7ee8dd7dabe26851739877b82f0f9a550b2328473b8ffd62f194d569ebb
7
- data.tar.gz: bf71a208f07bc00c213724cbcf175a9ede2f87bf4eaf85a926ee8e9f87b322e2dd79164db1db84ec60633a0b1cfa961e8fc6d3889c8ca25ea220fa437285af5d
6
+ metadata.gz: dbc38871c106b045a500645cf6c4517b30ce15e52bfc68520845e22276efdb4a21df3e4a7709ae682cd6b55ff76e063a615b9fe397795272535aa7c5298a9a92
7
+ data.tar.gz: 70b8e1dc84ba60d478affb22c93400cb153d4d0adc4904667970220ac03cb9d22b94e28f1b1a5504a30b8592c543d3234ee441d0ed57a31aefef70dfb6c2b9a3
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## 5.3.0
2
+ - Refactor: start using scheduler mixin [#110](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/110)
3
+
1
4
  ## 5.2.6
2
5
  - Fix: change default path of 'last_run_metadata_path' to be rooted in the LS data.path folder and not in $HOME [#106](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/106)
3
6
 
@@ -3,7 +3,7 @@ require "logstash-integration-jdbc_jars"
3
3
  require "logstash/filters/base"
4
4
  require "logstash/namespace"
5
5
  require "logstash/plugin_mixins/ecs_compatibility_support"
6
- require "logstash/plugin_mixins/jdbc/scheduler"
6
+ require "logstash/plugin_mixins/scheduler"
7
7
  require_relative "jdbc/loader"
8
8
  require_relative "jdbc/loader_schedule"
9
9
  require_relative "jdbc/repeating_load_runner"
@@ -19,6 +19,8 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
19
19
  # adds ecs_compatibility config which could be :disabled or :v1
20
20
  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
21
21
 
22
+ include LogStash::PluginMixins::Scheduler
23
+
22
24
  config_name "jdbc_static"
23
25
 
24
26
  # Define the loaders, an Array of Hashes, to fetch remote data and create local tables.
@@ -162,7 +164,6 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
162
164
  end
163
165
 
164
166
  def close
165
- @scheduler.stop if @scheduler
166
167
  @parsed_loaders.each(&:close)
167
168
  @processor.close
168
169
  end
@@ -208,13 +209,7 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
208
209
  if @loader_schedule
209
210
  @loader_runner = Jdbc::RepeatingLoadRunner.new(*runner_args)
210
211
  @loader_runner.initial_load
211
- @scheduler = LogStash::PluginMixins::Jdbc::Scheduler.
212
- start_cron_scheduler(@loader_schedule, thread_name: "[#{id}]-jdbc_static__scheduler") { @loader_runner.repeated_load }
213
- cron_job = @scheduler.cron_jobs.first
214
- if cron_job
215
- frequency = cron_job.respond_to?(:rough_frequency) ? cron_job.rough_frequency : cron_job.frequency
216
- logger.info("Loaders will execute every #{frequency} seconds", loader_schedule: @loader_schedule)
217
- end
212
+ scheduler.cron(@loader_schedule) { @loader_runner.repeated_load }
218
213
  else
219
214
  @loader_runner = Jdbc::SingleLoadRunner.new(*runner_args)
220
215
  @loader_runner.initial_load
@@ -3,12 +3,11 @@ require "logstash/inputs/base"
3
3
  require "logstash/namespace"
4
4
  require "logstash/plugin_mixins/jdbc/common"
5
5
  require "logstash/plugin_mixins/jdbc/jdbc"
6
- require "logstash/plugin_mixins/jdbc/scheduler"
7
6
  require "logstash/plugin_mixins/ecs_compatibility_support"
8
7
  require "logstash/plugin_mixins/ecs_compatibility_support/target_check"
9
8
  require "logstash/plugin_mixins/validator_support/field_reference_validation_adapter"
10
-
11
9
  require "logstash/plugin_mixins/event_support/event_factory_adapter"
10
+ require "logstash/plugin_mixins/scheduler"
12
11
  require "fileutils"
13
12
 
14
13
  # this require_relative returns early unless the JRuby version is between 9.2.0.0 and 9.2.8.0
@@ -147,6 +146,8 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
147
146
  # adds :field_reference validator adapter
148
147
  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
149
148
 
149
+ include LogStash::PluginMixins::Scheduler
150
+
150
151
  config_name "jdbc"
151
152
 
152
153
  # If undefined, Logstash will complain, even if codec is unused.
@@ -260,7 +261,6 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
260
261
  end
261
262
  end
262
263
 
263
- require "rufus/scheduler"
264
264
  prepare_jdbc_connection
265
265
 
266
266
  if @use_column_value
@@ -319,22 +319,16 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
319
319
  def run(queue)
320
320
  load_driver
321
321
  if @schedule
322
- # input thread (Java) name example "[my-oracle]<jdbc"
323
- @scheduler = LogStash::PluginMixins::Jdbc::Scheduler.
324
- start_cron_scheduler(@schedule, thread_name: "[#{id}]<jdbc__scheduler") { execute_query(queue) }
325
- @scheduler.join
322
+ # scheduler input thread name example: "[my-oracle]|input|jdbc|scheduler"
323
+ scheduler.cron(@schedule) { execute_query(queue) }
324
+ scheduler.join
326
325
  else
327
326
  execute_query(queue)
328
327
  end
329
328
  end # def run
330
329
 
331
- def close
332
- @scheduler.shutdown if @scheduler
333
- end
334
-
335
330
  def stop
336
331
  close_jdbc_connection
337
- @scheduler.shutdown(:wait) if @scheduler
338
332
  end
339
333
 
340
334
  private
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-integration-jdbc'
3
- s.version = '5.2.6'
3
+ s.version = '5.3.0'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Integration with JDBC - input and filter plugins"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -34,12 +34,11 @@ Gem::Specification.new do |s|
34
34
 
35
35
  s.add_runtime_dependency 'tzinfo'
36
36
  s.add_runtime_dependency 'tzinfo-data'
37
- # plugin maintains compatibility with < 3.5 (3.0.9)
38
- # but works with newer rufus-scheduler >= 3.5 as well
39
- s.add_runtime_dependency 'rufus-scheduler'
37
+
40
38
  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
41
39
  s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
42
40
  s.add_runtime_dependency "logstash-mixin-event_support", '~> 1.0'
41
+ s.add_runtime_dependency "logstash-mixin-scheduler", '~> 1.0'
43
42
 
44
43
  s.add_development_dependency "childprocess"
45
44
  s.add_development_dependency 'logstash-devutils', '>= 2.3'
@@ -251,18 +251,6 @@ describe LogStash::Inputs::Jdbc do
251
251
  Timecop.return
252
252
  end
253
253
 
254
- it "cleans up scheduler resources on close" do
255
- runner = Thread.new do
256
- plugin.run(queue)
257
- end
258
- sleep 1
259
- plugin.do_close
260
-
261
- scheduler = plugin.instance_variable_get(:@scheduler)
262
- expect(scheduler).to_not be_nil
263
- expect(scheduler.down?).to be_truthy
264
- end
265
-
266
254
  end
267
255
 
268
256
  context "when scheduling and previous runs are to be preserved" do
@@ -1717,7 +1705,7 @@ describe LogStash::Inputs::Jdbc do
1717
1705
  let(:jdbc_driver_class) { "org.apache.NonExistentDriver" }
1718
1706
  it "raises a loading error" do
1719
1707
  expect { plugin.send(:load_driver) }.to raise_error LogStash::PluginLoadingError,
1720
- /java.lang.ClassNotFoundException: org.apache.NonExistentDriver/
1708
+ /ClassNotFoundException: org.apache.NonExistentDriver/
1721
1709
  end
1722
1710
  end
1723
1711
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-integration-jdbc
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.2.6
4
+ version: 5.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-05-19 00:00:00.000000000 Z
11
+ date: 2022-06-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -131,38 +131,38 @@ dependencies:
131
131
  - !ruby/object:Gem::Dependency
132
132
  requirement: !ruby/object:Gem::Requirement
133
133
  requirements:
134
- - - ">="
134
+ - - "~>"
135
135
  - !ruby/object:Gem::Version
136
- version: '0'
137
- name: rufus-scheduler
136
+ version: '1.3'
137
+ name: logstash-mixin-ecs_compatibility_support
138
138
  prerelease: false
139
139
  type: :runtime
140
140
  version_requirements: !ruby/object:Gem::Requirement
141
141
  requirements:
142
- - - ">="
142
+ - - "~>"
143
143
  - !ruby/object:Gem::Version
144
- version: '0'
144
+ version: '1.3'
145
145
  - !ruby/object:Gem::Dependency
146
146
  requirement: !ruby/object:Gem::Requirement
147
147
  requirements:
148
148
  - - "~>"
149
149
  - !ruby/object:Gem::Version
150
- version: '1.3'
151
- name: logstash-mixin-ecs_compatibility_support
150
+ version: '1.0'
151
+ name: logstash-mixin-validator_support
152
152
  prerelease: false
153
153
  type: :runtime
154
154
  version_requirements: !ruby/object:Gem::Requirement
155
155
  requirements:
156
156
  - - "~>"
157
157
  - !ruby/object:Gem::Version
158
- version: '1.3'
158
+ version: '1.0'
159
159
  - !ruby/object:Gem::Dependency
160
160
  requirement: !ruby/object:Gem::Requirement
161
161
  requirements:
162
162
  - - "~>"
163
163
  - !ruby/object:Gem::Version
164
164
  version: '1.0'
165
- name: logstash-mixin-validator_support
165
+ name: logstash-mixin-event_support
166
166
  prerelease: false
167
167
  type: :runtime
168
168
  version_requirements: !ruby/object:Gem::Requirement
@@ -176,7 +176,7 @@ dependencies:
176
176
  - - "~>"
177
177
  - !ruby/object:Gem::Version
178
178
  version: '1.0'
179
- name: logstash-mixin-event_support
179
+ name: logstash-mixin-scheduler
180
180
  prerelease: false
181
181
  type: :runtime
182
182
  version_requirements: !ruby/object:Gem::Requirement
@@ -277,7 +277,6 @@ files:
277
277
  - lib/logstash/inputs/tzinfo_jruby_patch.rb
278
278
  - lib/logstash/plugin_mixins/jdbc/common.rb
279
279
  - lib/logstash/plugin_mixins/jdbc/jdbc.rb
280
- - lib/logstash/plugin_mixins/jdbc/scheduler.rb
281
280
  - lib/logstash/plugin_mixins/jdbc/statement_handler.rb
282
281
  - lib/logstash/plugin_mixins/jdbc/value_tracking.rb
283
282
  - lib/logstash/plugin_mixins/jdbc_streaming.rb
@@ -306,7 +305,6 @@ files:
306
305
  - spec/helpers/derbyrun.jar
307
306
  - spec/inputs/integration/integ_spec.rb
308
307
  - spec/inputs/jdbc_spec.rb
309
- - spec/plugin_mixins/jdbc/scheduler_spec.rb
310
308
  - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb
311
309
  - vendor/jar-dependencies/org/apache/derby/derby/10.14.1.0/derby-10.14.1.0.jar
312
310
  - vendor/jar-dependencies/org/apache/derby/derbyclient/10.14.1.0/derbyclient-10.14.1.0.jar
@@ -359,5 +357,4 @@ test_files:
359
357
  - spec/helpers/derbyrun.jar
360
358
  - spec/inputs/integration/integ_spec.rb
361
359
  - spec/inputs/jdbc_spec.rb
362
- - spec/plugin_mixins/jdbc/scheduler_spec.rb
363
360
  - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb
@@ -1,175 +0,0 @@
1
- require 'rufus/scheduler'
2
-
3
- require 'logstash/util/loggable'
4
-
5
- module LogStash module PluginMixins module Jdbc
6
- class Scheduler < Rufus::Scheduler
7
-
8
- include LogStash::Util::Loggable
9
-
10
- # Rufus::Scheduler >= 3.4 moved the Time impl into a gem EoTime = ::EtOrbi::EoTime`
11
- # Rufus::Scheduler 3.1 - 3.3 using it's own Time impl `Rufus::Scheduler::ZoTime`
12
- TimeImpl = defined?(Rufus::Scheduler::EoTime) ? Rufus::Scheduler::EoTime :
13
- (defined?(Rufus::Scheduler::ZoTime) ? Rufus::Scheduler::ZoTime : ::Time)
14
-
15
- # @param cron [String] cron-line
16
- # @param opts [Hash] scheduler options
17
- # @return scheduler instance
18
- def self.start_cron_scheduler(cron, opts = {}, &block)
19
- unless block_given?
20
- raise ArgumentError, 'missing (cron scheduler) block - worker task to execute'
21
- end
22
- scheduler = new_scheduler(opts)
23
- scheduler.schedule_cron(cron, &block)
24
- scheduler
25
- end
26
-
27
- # @param opts [Hash] scheduler options
28
- # @return scheduler instance
29
- def self.new_scheduler(opts)
30
- unless opts.key?(:thread_name)
31
- raise ArgumentError, 'thread_name: option is required to be able to distinguish multiple scheduler threads'
32
- end
33
- opts[:max_work_threads] ||= 1
34
- # amount the scheduler thread sleeps between checking whether jobs
35
- # should trigger, default is 0.3 which is a bit too often ...
36
- # in theory the cron expression '* * * * * *' supports running jobs
37
- # every second but this is very rare, we could potentially go higher
38
- opts[:frequency] ||= 1.0
39
-
40
- new(opts)
41
- end
42
-
43
- # @overload
44
- def timeout_jobs
45
- # Rufus relies on `Thread.list` which is a blocking operation and with many schedulers
46
- # (and threads) within LS will have a negative impact on performance as scheduler
47
- # threads will end up waiting to obtain the `Thread.list` lock.
48
- #
49
- # However, this isn't necessary we can easily detect whether there are any jobs
50
- # that might need to timeout: only when `@opts[:timeout]` is set causes worker thread(s)
51
- # to have a `Thread.current[:rufus_scheduler_timeout]` that is not nil
52
- return unless @opts[:timeout]
53
- super
54
- end
55
-
56
- # @overload
57
- def work_threads(query = :all)
58
- if query == :__all_no_cache__ # special case from JobDecorator#start_work_thread
59
- @_work_threads = nil # when a new worker thread is being added reset
60
- return super(:all)
61
- end
62
-
63
- # Gets executed every time a job is triggered, we're going to cache the
64
- # worker threads for this scheduler (to avoid `Thread.list`) - they only
65
- # change when a new thread is being started from #start_work_thread ...
66
- work_threads = @_work_threads
67
- if work_threads.nil?
68
- work_threads = threads.select { |t| t[:rufus_scheduler_work_thread] }
69
- @_work_threads = work_threads
70
- end
71
-
72
- case query
73
- when :active then work_threads.select { |t| t[:rufus_scheduler_job] }
74
- when :vacant then work_threads.reject { |t| t[:rufus_scheduler_job] }
75
- else work_threads
76
- end
77
- end
78
-
79
- # @overload
80
- def on_error(job, err)
81
- details = { exception: err.class, message: err.message, backtrace: err.backtrace }
82
- details[:cause] = err.cause if err.cause
83
-
84
- details[:now] = debug_format_time(TimeImpl.now)
85
- details[:last_time] = (debug_format_time(job.last_time) rescue nil)
86
- details[:next_time] = (debug_format_time(job.next_time) rescue nil)
87
- details[:job] = job
88
-
89
- details[:opts] = @opts
90
- details[:started_at] = started_at
91
- details[:thread] = thread.inspect
92
- details[:jobs_size] = @jobs.size
93
- details[:work_threads_size] = work_threads.size
94
- details[:work_queue_size] = work_queue.size
95
-
96
- logger.error("Scheduler intercepted an error:", details)
97
-
98
- rescue => e
99
- logger.error("Scheduler failed in #on_error #{e.inspect}")
100
- end
101
-
102
- def debug_format_time(time)
103
- # EtOrbi::EoTime used by (newer) Rufus::Scheduler has to_debug_s https://git.io/JyiPj
104
- time.respond_to?(:to_debug_s) ? time.to_debug_s : time.strftime("%Y-%m-%dT%H:%M:%S.%L")
105
- end
106
- private :debug_format_time
107
-
108
- # @private helper used by JobDecorator
109
- def work_thread_name_prefix
110
- ( @opts[:thread_name] || "#{@thread_key}_scheduler" ) + '_worker-'
111
- end
112
-
113
- protected
114
-
115
- # @overload
116
- def start
117
- ret = super() # @thread[:name] = @opts[:thread_name] || "#{@thread_key}_scheduler"
118
-
119
- # at least set thread.name for easier thread dump analysis
120
- if @thread.is_a?(Thread) && @thread.respond_to?(:name=)
121
- @thread.name = @thread[:name] if @thread[:name]
122
- end
123
-
124
- ret
125
- end
126
-
127
- # @overload
128
- def do_schedule(job_type, t, callable, opts, return_job_instance, block)
129
- job_or_id = super
130
-
131
- job_or_id.extend JobDecorator if return_job_instance
132
-
133
- job_or_id
134
- end
135
-
136
- module JobDecorator
137
-
138
- def start_work_thread
139
- prev_thread_count = @scheduler.work_threads.size
140
-
141
- ret = super() # does not return Thread instance in 3.0
142
-
143
- work_threads = @scheduler.work_threads(:__all_no_cache__)
144
- while prev_thread_count == work_threads.size # very unlikely
145
- Thread.pass
146
- work_threads = @scheduler.work_threads(:__all_no_cache__)
147
- end
148
-
149
- work_thread_name_prefix = @scheduler.work_thread_name_prefix
150
-
151
- work_threads.sort! do |t1, t2|
152
- if t1[:name].nil?
153
- t2[:name].nil? ? 0 : +1 # nils at the end
154
- elsif t2[:name].nil?
155
- t1[:name].nil? ? 0 : -1
156
- else
157
- t1[:name] <=> t2[:name]
158
- end
159
- end
160
-
161
- work_threads.each_with_index do |thread, i|
162
- unless thread[:name]
163
- thread[:name] = "#{work_thread_name_prefix}#{sprintf('%02i', i)}"
164
- thread.name = thread[:name] if thread.respond_to?(:name=)
165
- # e.g. "[oracle]<jdbc_scheduler_worker-00"
166
- end
167
- end
168
-
169
- ret
170
- end
171
-
172
- end
173
-
174
- end
175
- end end end
@@ -1,78 +0,0 @@
1
- # encoding: utf-8
2
- require "logstash/devutils/rspec/spec_helper"
3
- require "logstash/plugin_mixins/jdbc/scheduler"
4
-
5
- describe LogStash::PluginMixins::Jdbc::Scheduler do
6
-
7
- let(:thread_name) { '[test]<jdbc_scheduler' }
8
-
9
- let(:opts) do
10
- { :max_work_threads => 2, :thread_name => thread_name }
11
- end
12
-
13
- subject(:scheduler) { LogStash::PluginMixins::Jdbc::Scheduler.new(opts) }
14
-
15
- after { scheduler.stop(:wait) }
16
-
17
- it "sets scheduler thread name" do
18
- expect( scheduler.thread.name ).to include thread_name
19
- end
20
-
21
- context 'cron schedule' do
22
-
23
- before do
24
- scheduler.schedule_cron('* * * * * *') { sleep 1.25 } # every second
25
- end
26
-
27
- it "sets worker thread names" do
28
- sleep 3.0
29
- threads = scheduler.work_threads
30
- threads.sort! { |t1, t2| (t1.name || '') <=> (t2.name || '') }
31
-
32
- expect( threads.size ).to eql 2
33
- expect( threads.first.name ).to eql "#{thread_name}_worker-00"
34
- expect( threads.last.name ).to eql "#{thread_name}_worker-01"
35
- end
36
-
37
- end
38
-
39
- context 'every 1s' do
40
-
41
- before do
42
- scheduler.schedule_in('1s') { raise 'TEST' } # every second
43
- end
44
-
45
- it "logs errors handled" do
46
- expect( scheduler.logger ).to receive(:error).with /Scheduler intercepted an error/, hash_including(:message => 'TEST')
47
- sleep 1.5
48
- end
49
-
50
- end
51
-
52
- context 'work threads' do
53
-
54
- let(:opts) { super().merge :max_work_threads => 3 }
55
-
56
- let(:counter) { java.util.concurrent.atomic.AtomicLong.new(0) }
57
-
58
- before do
59
- scheduler.schedule_cron('* * * * * *') { counter.increment_and_get; sleep 3.25 } # every second
60
- end
61
-
62
- it "are working" do
63
- sleep(0.05) while counter.get == 0
64
- expect( scheduler.work_threads.size ).to eql 1
65
- sleep(0.05) while counter.get == 1
66
- expect( scheduler.work_threads.size ).to eql 2
67
- sleep(0.05) while counter.get == 2
68
- expect( scheduler.work_threads.size ).to eql 3
69
-
70
- sleep 1.25
71
- expect( scheduler.work_threads.size ).to eql 3
72
- sleep 1.25
73
- expect( scheduler.work_threads.size ).to eql 3
74
- end
75
-
76
- end
77
-
78
- end