logstash-core 6.8.22-java → 7.0.0.alpha1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/lib/logstash/agent.rb +69 -85
  3. data/lib/logstash/api/modules/stats.rb +1 -1
  4. data/lib/logstash/compiler/lscl.rb +7 -7
  5. data/lib/logstash/config/config_ast.rb +1 -1
  6. data/lib/logstash/config/mixin.rb +1 -1
  7. data/lib/logstash/config/modules_common.rb +3 -3
  8. data/lib/logstash/dependency_report.rb +1 -2
  9. data/lib/logstash/environment.rb +4 -9
  10. data/lib/logstash/event.rb +1 -24
  11. data/lib/logstash/filter_delegator.rb +69 -2
  12. data/lib/logstash/filters/base.rb +2 -0
  13. data/lib/logstash/instrument/metric_store.rb +1 -1
  14. data/lib/logstash/instrument/periodic_poller/dlq.rb +7 -5
  15. data/lib/logstash/instrument/periodic_poller/jvm.rb +3 -3
  16. data/lib/logstash/instrument/periodic_poller/pq.rb +8 -6
  17. data/lib/logstash/instrument/periodic_pollers.rb +3 -3
  18. data/lib/logstash/java_pipeline.rb +11 -38
  19. data/lib/logstash/modules/kibana_config.rb +1 -1
  20. data/lib/logstash/modules/logstash_config.rb +1 -1
  21. data/lib/logstash/patches/resolv.rb +32 -17
  22. data/lib/logstash/pipeline.rb +11 -28
  23. data/lib/logstash/pipeline_action/base.rb +1 -1
  24. data/lib/logstash/pipeline_action/create.rb +13 -7
  25. data/lib/logstash/pipeline_action/reload.rb +12 -35
  26. data/lib/logstash/pipeline_action/stop.rb +6 -4
  27. data/lib/logstash/pipeline_settings.rb +1 -2
  28. data/lib/logstash/plugins/registry.rb +2 -5
  29. data/lib/logstash/runner.rb +0 -24
  30. data/lib/logstash/settings.rb +5 -5
  31. data/lib/logstash/state_resolver.rb +5 -5
  32. data/lib/logstash/util/duration_formatter.rb +1 -1
  33. data/lib/logstash/util/safe_uri.rb +0 -1
  34. data/lib/logstash/util/substitution_variables.rb +1 -22
  35. data/lib/logstash/util/thread_dump.rb +1 -1
  36. data/lib/logstash/util.rb +1 -11
  37. data/lib/logstash-core/version.rb +3 -1
  38. data/locales/en.yml +7 -16
  39. data/logstash-core.gemspec +11 -2
  40. data/spec/logstash/acked_queue_concurrent_stress_spec.rb +2 -2
  41. data/spec/logstash/agent/converge_spec.rb +31 -25
  42. data/spec/logstash/agent/metrics_spec.rb +1 -1
  43. data/spec/logstash/agent_spec.rb +7 -6
  44. data/spec/logstash/compiler/compiler_spec.rb +0 -28
  45. data/spec/logstash/config/config_ast_spec.rb +0 -15
  46. data/spec/logstash/config/mixin_spec.rb +2 -3
  47. data/spec/logstash/converge_result_spec.rb +1 -1
  48. data/spec/logstash/environment_spec.rb +4 -4
  49. data/spec/logstash/event_spec.rb +2 -10
  50. data/spec/logstash/filter_delegator_spec.rb +12 -2
  51. data/spec/logstash/filters/base_spec.rb +9 -45
  52. data/spec/logstash/instrument/periodic_poller/cgroup_spec.rb +2 -0
  53. data/spec/logstash/instrument/wrapped_write_client_spec.rb +1 -1
  54. data/spec/logstash/java_filter_delegator_spec.rb +11 -1
  55. data/spec/logstash/legacy_ruby_event_spec.rb +5 -6
  56. data/spec/logstash/patches_spec.rb +3 -1
  57. data/spec/logstash/pipeline_action/create_spec.rb +8 -14
  58. data/spec/logstash/pipeline_action/reload_spec.rb +9 -16
  59. data/spec/logstash/pipeline_action/stop_spec.rb +3 -4
  60. data/spec/logstash/queue_factory_spec.rb +1 -2
  61. data/spec/logstash/runner_spec.rb +0 -2
  62. data/spec/logstash/settings/array_coercible_spec.rb +1 -1
  63. data/spec/logstash/settings/bytes_spec.rb +2 -2
  64. data/spec/logstash/settings/port_range_spec.rb +1 -1
  65. data/spec/logstash/settings_spec.rb +0 -10
  66. data/spec/logstash/state_resolver_spec.rb +22 -26
  67. data/spec/logstash/util/safe_uri_spec.rb +0 -40
  68. data/spec/logstash/util/secretstore_spec.rb +1 -1
  69. data/spec/logstash/util/time_value_spec.rb +1 -1
  70. data/spec/logstash/util/wrapped_acked_queue_spec.rb +1 -1
  71. data/spec/logstash/webserver_spec.rb +5 -9
  72. data/spec/support/matchers.rb +19 -25
  73. data/spec/support/shared_contexts.rb +3 -3
  74. data/versions-gem-copy.yml +9 -9
  75. metadata +31 -44
  76. data/lib/logstash/patches/resolv_9270.rb +0 -2903
  77. data/lib/logstash/pipelines_registry.rb +0 -166
  78. data/lib/logstash/util/lazy_singleton.rb +0 -33
  79. data/spec/logstash/jruby_version_spec.rb +0 -15
  80. data/spec/logstash/pipelines_registry_spec.rb +0 -220
@@ -1,166 +0,0 @@
1
- # encoding: utf-8
2
-
3
- module LogStash
4
- class PipelineState
5
- attr_reader :pipeline_id, :pipeline
6
-
7
- def initialize(pipeline_id, pipeline)
8
- @pipeline_id = pipeline_id
9
- @pipeline = pipeline
10
- @reloading = Concurrent::AtomicBoolean.new(false)
11
- end
12
-
13
- def terminated?
14
- # a reloading pipeline is never considered terminated
15
- @reloading.false? && @pipeline.finished_execution?
16
- end
17
-
18
- def set_reloading(is_reloading)
19
- @reloading.value = is_reloading
20
- end
21
-
22
- def set_pipeline(pipeline)
23
- raise(ArgumentError, "invalid nil pipeline") if pipeline.nil?
24
- @pipeline = pipeline
25
- end
26
- end
27
-
28
- class PipelinesRegistry
29
- attr_reader :states
30
- include LogStash::Util::Loggable
31
-
32
- def initialize
33
- # we leverage the semantic of the Java ConcurrentHashMap for the
34
- # compute() method which is atomic; calling compute() concurrently
35
- # will block until the other compute finishes so no mutex is necessary
36
- # for synchronizing compute calls
37
- @states = java.util.concurrent.ConcurrentHashMap.new
38
- end
39
-
40
- # Execute the passed creation logic block and create a new state upon success
41
- # @param pipeline_id [String, Symbol] the pipeline id
42
- # @param pipeline [Pipeline] the new pipeline to create
43
- # @param create_block [Block] the creation execution logic
44
- #
45
- # @yieldreturn [Boolean] the new pipeline creation success
46
- #
47
- # @return [Boolean] new pipeline creation success
48
- def create_pipeline(pipeline_id, pipeline, &create_block)
49
- success = false
50
-
51
- @states.compute(pipeline_id) do |_, state|
52
- if state
53
- if state.terminated?
54
- success = yield
55
- state.set_pipeline(pipeline)
56
- else
57
- logger.error("Attempted to create a pipeline that already exists", :pipeline_id => pipeline_id)
58
- end
59
- state
60
- else
61
- success = yield
62
- success ? PipelineState.new(pipeline_id, pipeline) : nil
63
- end
64
- end
65
-
66
- success
67
- end
68
-
69
- # Execute the passed termination logic block
70
- # @param pipeline_id [String, Symbol] the pipeline id
71
- # @param stop_block [Block] the termination execution logic
72
- #
73
- # @yieldparam [Pipeline] the pipeline to terminate
74
- def terminate_pipeline(pipeline_id, &stop_block)
75
- @states.compute(pipeline_id) do |_, state|
76
- if state.nil?
77
- logger.error("Attempted to terminate a pipeline that does not exists", :pipeline_id => pipeline_id)
78
- nil
79
- else
80
- yield(state.pipeline)
81
- state
82
- end
83
- end
84
- end
85
-
86
- # Execute the passed reloading logic block in the context of the reloading state and set new pipeline in state
87
- # @param pipeline_id [String, Symbol] the pipeline id
88
- # @param reload_block [Block] the reloading execution logic
89
- #
90
- # @yieldreturn [Array<Boolean, Pipeline>] the new pipeline creation success and new pipeline object
91
- #
92
- # @return [Boolean] new pipeline creation success
93
- def reload_pipeline(pipeline_id, &reload_block)
94
- success = false
95
-
96
- @states.compute(pipeline_id) do |_, state|
97
- if state.nil?
98
- logger.error("Attempted to reload a pipeline that does not exists", :pipeline_id => pipeline_id)
99
- nil
100
- else
101
- state.set_reloading(true)
102
- begin
103
- success, new_pipeline = yield
104
- state.set_pipeline(new_pipeline)
105
- ensure
106
- state.set_reloading(false)
107
- end
108
- state
109
- end
110
- end
111
-
112
- success
113
- end
114
-
115
- # @param pipeline_id [String, Symbol] the pipeline id
116
- # @return [Pipeline] the pipeline object or nil if none for pipeline_id
117
- def get_pipeline(pipeline_id)
118
- state = @states.get(pipeline_id)
119
- state.nil? ? nil : state.pipeline
120
- end
121
-
122
- # @return [Fixnum] number of items in the states collection
123
- def size
124
- @states.size
125
- end
126
-
127
- # @return [Boolean] true if the states collection is empty.
128
- def empty?
129
- @states.isEmpty
130
- end
131
-
132
- # @return [Hash{String=>Pipeline}]
133
- def running_pipelines
134
- select_pipelines { |state| !state.terminated? }
135
- end
136
-
137
- # @return [Hash{String=>Pipeline}]
138
- def non_running_pipelines
139
- select_pipelines { |state| state.terminated? }
140
- end
141
-
142
- # @return [Hash{String=>Pipeline}]
143
- def running_user_defined_pipelines
144
- select_pipelines { |state | !state.terminated? && !state.pipeline.system? }
145
- end
146
-
147
- private
148
-
149
- # Returns a mapping of pipelines by their ids.
150
- # Pipelines can optionally be filtered by their `PipelineState` by passing
151
- # a block that returns truthy when a pipeline should be included in the
152
- # result.
153
- #
154
- # @yieldparam [PipelineState]
155
- # @yieldreturn [Boolean]
156
- #
157
- # @return [Hash{String=>Pipeline}]
158
- def select_pipelines(&optional_state_filter)
159
- @states.each_with_object({}) do |(id, state), memo|
160
- if state && (!block_given? || yield(state))
161
- memo[id] = state.pipeline
162
- end
163
- end
164
- end
165
- end
166
- end
@@ -1,33 +0,0 @@
1
- require 'thread' # Mutex
2
-
3
- # A [LazySingleton] wraps the result of the provided block,
4
- # which is guaranteed to be called at-most-once, even if the
5
- # block's return value is nil.
6
- class ::LogStash::Util::LazySingleton
7
-
8
- def initialize(&block)
9
- @mutex = Mutex.new
10
- @block = block
11
- @instantiated = false
12
- end
13
-
14
- def instance
15
- unless @instantiated
16
- @mutex.synchronize do
17
- unless @instantiated
18
- @instance = @block.call
19
- @instantiated = true
20
- end
21
- end
22
- end
23
-
24
- return @instance
25
- end
26
-
27
- def reset!
28
- @mutex.synchronize do
29
- @instantiated = false
30
- @instance = nil
31
- end
32
- end
33
- end
@@ -1,15 +0,0 @@
1
- require 'spec_helper'
2
-
3
-
4
- describe "JRuby Version" do
5
- it "should break on JRuby version change" do
6
- # This spec will break upon JRuby version change to make sure we
7
- # verify if resolv.rb has been fixed in Jruby so we can get rid of
8
- # lib/logstash/patches/resolv.rb.
9
- # ref:
10
- # https://github.com/logstash-plugins/logstash-filter-dns/issues/51
11
- # https://github.com/jruby/jruby/pull/5722
12
- expect(JRUBY_VERSION).to eq("9.2.7.0")
13
- end
14
- end
15
-
@@ -1,220 +0,0 @@
1
- # encoding: utf-8
2
- require "spec_helper"
3
- require "logstash/pipelines_registry"
4
-
5
- describe LogStash::PipelinesRegistry do
6
-
7
- let(:pipeline_id) { "test" }
8
- let(:pipeline) { double("Pipeline") }
9
- let (:logger) { double("Logger") }
10
-
11
- context "at object creation" do
12
- it "should be empty" do
13
- expect(subject.size).to eq(0)
14
- expect(subject.empty?).to be_truthy
15
- expect(subject.running_pipelines).to be_empty
16
- expect(subject.non_running_pipelines).to be_empty
17
- expect(subject.running_user_defined_pipelines).to be_empty
18
- end
19
- end
20
-
21
- context "creating a pipeline" do
22
- context "without existing same pipeline id" do
23
- it "registry should not have a state for pipeline_id" do
24
- expect(subject.get_pipeline(pipeline_id)).to be_nil
25
- end
26
-
27
- it "should return block return value" do
28
- expect(subject.create_pipeline(pipeline_id, pipeline) { "dummy" }).to eq("dummy")
29
- end
30
-
31
- it "should register the new pipeline upon successful create block" do
32
- subject.create_pipeline(pipeline_id, pipeline) { true }
33
- expect(subject.get_pipeline(pipeline_id)).to eq(pipeline)
34
- end
35
-
36
- it "should not register the new pipeline upon unsuccessful create block" do
37
- subject.create_pipeline(pipeline_id, pipeline) { false }
38
- expect(subject.get_pipeline(pipeline_id)).to be_nil
39
- end
40
- end
41
-
42
- context "with existing pipeline id" do
43
- before :each do
44
- subject.create_pipeline(pipeline_id, pipeline) { true }
45
- end
46
-
47
- it "registry should have a state for pipeline_id" do
48
- expect(subject.get_pipeline(pipeline_id)).to eq(pipeline)
49
- end
50
-
51
- context "when existing pipeline is not terminated" do
52
- before :each do
53
- expect(pipeline).to receive(:finished_execution?).and_return(false)
54
- end
55
-
56
- it "should return false" do
57
- expect(subject.create_pipeline(pipeline_id, pipeline) { "dummy" }).to be_falsey
58
- end
59
-
60
- it "should not call block and log error if pipeline is not terminated" do
61
- expect(LogStash::PipelinesRegistry).to receive(:logger).and_return(logger)
62
- expect(logger).to receive(:error)
63
- expect { |b| subject.create_pipeline(pipeline_id, pipeline, &b) }.not_to yield_control
64
- end
65
- end
66
-
67
- context "when existing pipeline is terminated" do
68
- let (:new_pipeline) { double("New Pipeline") }
69
-
70
- before :each do
71
- expect(pipeline).to receive(:finished_execution?).and_return(true)
72
- end
73
-
74
- it "should return block value" do
75
- expect(subject.create_pipeline(pipeline_id, new_pipeline) { "dummy" }).to eq("dummy")
76
- end
77
-
78
- it "should return block value" do
79
- expect(subject.create_pipeline(pipeline_id, new_pipeline) { "dummy" }).to eq("dummy")
80
- end
81
-
82
- it "should register new pipeline" do
83
- subject.create_pipeline(pipeline_id, new_pipeline) { true }
84
- expect(subject.get_pipeline(pipeline_id)).to eq(new_pipeline)
85
- end
86
- end
87
- end
88
- end
89
-
90
- context "terminating a pipeline" do
91
- context "without existing pipeline id" do
92
- it "should log error" do
93
- expect(LogStash::PipelinesRegistry).to receive(:logger).and_return(logger)
94
- expect(logger).to receive(:error)
95
- subject.terminate_pipeline(pipeline_id) { "dummy" }
96
- end
97
-
98
- it "should not yield to block" do
99
- expect { |b| subject.terminate_pipeline(pipeline_id, &b) }.not_to yield_control
100
- end
101
- end
102
-
103
- context "with existing pipeline id" do
104
- before :each do
105
- subject.create_pipeline(pipeline_id, pipeline) { true }
106
- end
107
-
108
- it "should yield to block" do
109
- expect { |b| subject.terminate_pipeline(pipeline_id, &b) }.to yield_control
110
- end
111
-
112
- it "should keep pipeline id" do
113
- subject.terminate_pipeline(pipeline_id) { "dummy" }
114
- expect(subject.get_pipeline(pipeline_id)).to eq(pipeline)
115
- end
116
- end
117
- end
118
-
119
- context "reloading a pipeline" do
120
- it "should log error with inexistent pipeline id" do
121
- expect(LogStash::PipelinesRegistry).to receive(:logger).and_return(logger)
122
- expect(logger).to receive(:error)
123
- subject.reload_pipeline(pipeline_id) { }
124
- end
125
-
126
- context "with existing pipeline id" do
127
- before :each do
128
- subject.create_pipeline(pipeline_id, pipeline) { true }
129
- end
130
-
131
- it "should return block value" do
132
- expect(subject.reload_pipeline(pipeline_id) { ["dummy", pipeline] }).to eq("dummy")
133
- end
134
-
135
- it "should not be terminated while reloading" do
136
- expect(pipeline).to receive(:finished_execution?).and_return(false, true, true)
137
-
138
- # 1st call: finished_execution? is false
139
- expect(subject.running_pipelines).not_to be_empty
140
-
141
- # 2nd call: finished_execution? is true
142
- expect(subject.running_pipelines).to be_empty
143
-
144
-
145
- queue = Queue.new # threadsafe queue
146
- in_block = Concurrent::AtomicBoolean.new(false)
147
-
148
- thread = Thread.new(subject, pipeline_id, pipeline, queue, in_block) do |subject, pipeline_id, pipeline, queue, in_block|
149
- subject.reload_pipeline(pipeline_id) do
150
- in_block.make_true
151
- queue.pop
152
- [true, pipeline]
153
- end
154
- end
155
-
156
- # make sure we entered the block executioin
157
- wait(10).for {in_block.true?}.to be_truthy
158
-
159
- # at this point the thread is suspended waiting on queue
160
-
161
- # since in reloading state, running_pipelines is not empty
162
- expect(subject.running_pipelines).not_to be_empty
163
-
164
- # unblock thread
165
- queue.push(:dummy)
166
- thread.join
167
-
168
- # 3rd call: finished_execution? is true
169
- expect(subject.running_pipelines).to be_empty
170
- end
171
- end
172
- end
173
-
174
- context "pipelines collections" do
175
- context "with a non terminated pipelines" do
176
- before :each do
177
- subject.create_pipeline(pipeline_id, pipeline) { true }
178
- expect(pipeline).to receive(:finished_execution?).and_return(false)
179
- end
180
-
181
- it "should find running pipelines" do
182
- expect(subject.running_pipelines).not_to be_empty
183
- end
184
-
185
- it "should not find non_running pipelines" do
186
- expect(subject.non_running_pipelines).to be_empty
187
- end
188
-
189
- it "should find running_user_defined_pipelines" do
190
- expect(pipeline).to receive(:system?).and_return(false)
191
- expect(subject.running_user_defined_pipelines).not_to be_empty
192
- end
193
-
194
- it "should not find running_user_defined_pipelines" do
195
- expect(pipeline).to receive(:system?).and_return(true)
196
- expect(subject.running_user_defined_pipelines).to be_empty
197
- end
198
- end
199
-
200
- context "with a terminated pipelines" do
201
- before :each do
202
- subject.create_pipeline(pipeline_id, pipeline) { true }
203
- expect(pipeline).to receive(:finished_execution?).and_return(true)
204
- end
205
-
206
- it "should not find running pipelines" do
207
- expect(subject.running_pipelines).to be_empty
208
- end
209
-
210
- it "should find non_running pipelines" do
211
- expect(subject.non_running_pipelines).not_to be_empty
212
- end
213
-
214
- it "should not find running_user_defined_pipelines" do
215
- expect(subject.running_user_defined_pipelines).to be_empty
216
- end
217
- end
218
-
219
- end
220
- end