logstash-core 6.8.20-java → 7.0.0.alpha1-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/lib/logstash/agent.rb +69 -85
  3. data/lib/logstash/api/modules/stats.rb +1 -1
  4. data/lib/logstash/compiler/lscl.rb +7 -7
  5. data/lib/logstash/config/config_ast.rb +1 -1
  6. data/lib/logstash/config/mixin.rb +1 -1
  7. data/lib/logstash/config/modules_common.rb +3 -3
  8. data/lib/logstash/dependency_report.rb +1 -2
  9. data/lib/logstash/environment.rb +4 -9
  10. data/lib/logstash/event.rb +1 -24
  11. data/lib/logstash/filter_delegator.rb +69 -2
  12. data/lib/logstash/filters/base.rb +2 -0
  13. data/lib/logstash/instrument/metric_store.rb +1 -1
  14. data/lib/logstash/instrument/periodic_poller/dlq.rb +7 -5
  15. data/lib/logstash/instrument/periodic_poller/jvm.rb +3 -3
  16. data/lib/logstash/instrument/periodic_poller/pq.rb +8 -6
  17. data/lib/logstash/instrument/periodic_pollers.rb +3 -3
  18. data/lib/logstash/java_pipeline.rb +11 -38
  19. data/lib/logstash/modules/kibana_config.rb +1 -1
  20. data/lib/logstash/modules/logstash_config.rb +1 -1
  21. data/lib/logstash/patches/resolv.rb +32 -17
  22. data/lib/logstash/pipeline.rb +11 -28
  23. data/lib/logstash/pipeline_action/base.rb +1 -1
  24. data/lib/logstash/pipeline_action/create.rb +13 -7
  25. data/lib/logstash/pipeline_action/reload.rb +12 -35
  26. data/lib/logstash/pipeline_action/stop.rb +6 -4
  27. data/lib/logstash/pipeline_settings.rb +1 -2
  28. data/lib/logstash/plugins/registry.rb +2 -5
  29. data/lib/logstash/runner.rb +0 -24
  30. data/lib/logstash/settings.rb +5 -5
  31. data/lib/logstash/state_resolver.rb +5 -5
  32. data/lib/logstash/util/duration_formatter.rb +1 -1
  33. data/lib/logstash/util/safe_uri.rb +0 -1
  34. data/lib/logstash/util/substitution_variables.rb +1 -22
  35. data/lib/logstash/util/thread_dump.rb +1 -1
  36. data/lib/logstash/util.rb +1 -11
  37. data/lib/logstash-core/version.rb +3 -1
  38. data/locales/en.yml +7 -16
  39. data/logstash-core.gemspec +11 -2
  40. data/spec/logstash/acked_queue_concurrent_stress_spec.rb +2 -2
  41. data/spec/logstash/agent/converge_spec.rb +31 -25
  42. data/spec/logstash/agent/metrics_spec.rb +1 -1
  43. data/spec/logstash/agent_spec.rb +7 -6
  44. data/spec/logstash/compiler/compiler_spec.rb +0 -28
  45. data/spec/logstash/config/config_ast_spec.rb +0 -15
  46. data/spec/logstash/config/mixin_spec.rb +2 -3
  47. data/spec/logstash/converge_result_spec.rb +1 -1
  48. data/spec/logstash/environment_spec.rb +4 -4
  49. data/spec/logstash/event_spec.rb +2 -10
  50. data/spec/logstash/filter_delegator_spec.rb +12 -2
  51. data/spec/logstash/filters/base_spec.rb +9 -45
  52. data/spec/logstash/instrument/periodic_poller/cgroup_spec.rb +2 -0
  53. data/spec/logstash/instrument/wrapped_write_client_spec.rb +1 -1
  54. data/spec/logstash/java_filter_delegator_spec.rb +11 -1
  55. data/spec/logstash/legacy_ruby_event_spec.rb +5 -6
  56. data/spec/logstash/patches_spec.rb +3 -1
  57. data/spec/logstash/pipeline_action/create_spec.rb +8 -14
  58. data/spec/logstash/pipeline_action/reload_spec.rb +9 -16
  59. data/spec/logstash/pipeline_action/stop_spec.rb +3 -4
  60. data/spec/logstash/queue_factory_spec.rb +1 -2
  61. data/spec/logstash/runner_spec.rb +0 -2
  62. data/spec/logstash/settings/array_coercible_spec.rb +1 -1
  63. data/spec/logstash/settings/bytes_spec.rb +2 -2
  64. data/spec/logstash/settings/port_range_spec.rb +1 -1
  65. data/spec/logstash/settings_spec.rb +0 -10
  66. data/spec/logstash/state_resolver_spec.rb +22 -26
  67. data/spec/logstash/util/safe_uri_spec.rb +0 -40
  68. data/spec/logstash/util/secretstore_spec.rb +1 -1
  69. data/spec/logstash/util/time_value_spec.rb +1 -1
  70. data/spec/logstash/util/wrapped_acked_queue_spec.rb +1 -1
  71. data/spec/logstash/webserver_spec.rb +5 -9
  72. data/spec/support/matchers.rb +19 -25
  73. data/spec/support/shared_contexts.rb +3 -3
  74. data/versions-gem-copy.yml +9 -9
  75. metadata +31 -44
  76. data/lib/logstash/patches/resolv_9270.rb +0 -2903
  77. data/lib/logstash/pipelines_registry.rb +0 -166
  78. data/lib/logstash/util/lazy_singleton.rb +0 -33
  79. data/spec/logstash/jruby_version_spec.rb +0 -15
  80. data/spec/logstash/pipelines_registry_spec.rb +0 -220
@@ -1,166 +0,0 @@
1
- # encoding: utf-8
2
-
3
- module LogStash
4
- class PipelineState
5
- attr_reader :pipeline_id, :pipeline
6
-
7
- def initialize(pipeline_id, pipeline)
8
- @pipeline_id = pipeline_id
9
- @pipeline = pipeline
10
- @reloading = Concurrent::AtomicBoolean.new(false)
11
- end
12
-
13
- def terminated?
14
- # a reloading pipeline is never considered terminated
15
- @reloading.false? && @pipeline.finished_execution?
16
- end
17
-
18
- def set_reloading(is_reloading)
19
- @reloading.value = is_reloading
20
- end
21
-
22
- def set_pipeline(pipeline)
23
- raise(ArgumentError, "invalid nil pipeline") if pipeline.nil?
24
- @pipeline = pipeline
25
- end
26
- end
27
-
28
- class PipelinesRegistry
29
- attr_reader :states
30
- include LogStash::Util::Loggable
31
-
32
- def initialize
33
- # we leverage the semantic of the Java ConcurrentHashMap for the
34
- # compute() method which is atomic; calling compute() concurrently
35
- # will block until the other compute finishes so no mutex is necessary
36
- # for synchronizing compute calls
37
- @states = java.util.concurrent.ConcurrentHashMap.new
38
- end
39
-
40
- # Execute the passed creation logic block and create a new state upon success
41
- # @param pipeline_id [String, Symbol] the pipeline id
42
- # @param pipeline [Pipeline] the new pipeline to create
43
- # @param create_block [Block] the creation execution logic
44
- #
45
- # @yieldreturn [Boolean] the new pipeline creation success
46
- #
47
- # @return [Boolean] new pipeline creation success
48
- def create_pipeline(pipeline_id, pipeline, &create_block)
49
- success = false
50
-
51
- @states.compute(pipeline_id) do |_, state|
52
- if state
53
- if state.terminated?
54
- success = yield
55
- state.set_pipeline(pipeline)
56
- else
57
- logger.error("Attempted to create a pipeline that already exists", :pipeline_id => pipeline_id)
58
- end
59
- state
60
- else
61
- success = yield
62
- success ? PipelineState.new(pipeline_id, pipeline) : nil
63
- end
64
- end
65
-
66
- success
67
- end
68
-
69
- # Execute the passed termination logic block
70
- # @param pipeline_id [String, Symbol] the pipeline id
71
- # @param stop_block [Block] the termination execution logic
72
- #
73
- # @yieldparam [Pipeline] the pipeline to terminate
74
- def terminate_pipeline(pipeline_id, &stop_block)
75
- @states.compute(pipeline_id) do |_, state|
76
- if state.nil?
77
- logger.error("Attempted to terminate a pipeline that does not exists", :pipeline_id => pipeline_id)
78
- nil
79
- else
80
- yield(state.pipeline)
81
- state
82
- end
83
- end
84
- end
85
-
86
- # Execute the passed reloading logic block in the context of the reloading state and set new pipeline in state
87
- # @param pipeline_id [String, Symbol] the pipeline id
88
- # @param reload_block [Block] the reloading execution logic
89
- #
90
- # @yieldreturn [Array<Boolean, Pipeline>] the new pipeline creation success and new pipeline object
91
- #
92
- # @return [Boolean] new pipeline creation success
93
- def reload_pipeline(pipeline_id, &reload_block)
94
- success = false
95
-
96
- @states.compute(pipeline_id) do |_, state|
97
- if state.nil?
98
- logger.error("Attempted to reload a pipeline that does not exists", :pipeline_id => pipeline_id)
99
- nil
100
- else
101
- state.set_reloading(true)
102
- begin
103
- success, new_pipeline = yield
104
- state.set_pipeline(new_pipeline)
105
- ensure
106
- state.set_reloading(false)
107
- end
108
- state
109
- end
110
- end
111
-
112
- success
113
- end
114
-
115
- # @param pipeline_id [String, Symbol] the pipeline id
116
- # @return [Pipeline] the pipeline object or nil if none for pipeline_id
117
- def get_pipeline(pipeline_id)
118
- state = @states.get(pipeline_id)
119
- state.nil? ? nil : state.pipeline
120
- end
121
-
122
- # @return [Fixnum] number of items in the states collection
123
- def size
124
- @states.size
125
- end
126
-
127
- # @return [Boolean] true if the states collection is empty.
128
- def empty?
129
- @states.isEmpty
130
- end
131
-
132
- # @return [Hash{String=>Pipeline}]
133
- def running_pipelines
134
- select_pipelines { |state| !state.terminated? }
135
- end
136
-
137
- # @return [Hash{String=>Pipeline}]
138
- def non_running_pipelines
139
- select_pipelines { |state| state.terminated? }
140
- end
141
-
142
- # @return [Hash{String=>Pipeline}]
143
- def running_user_defined_pipelines
144
- select_pipelines { |state | !state.terminated? && !state.pipeline.system? }
145
- end
146
-
147
- private
148
-
149
- # Returns a mapping of pipelines by their ids.
150
- # Pipelines can optionally be filtered by their `PipelineState` by passing
151
- # a block that returns truthy when a pipeline should be included in the
152
- # result.
153
- #
154
- # @yieldparam [PipelineState]
155
- # @yieldreturn [Boolean]
156
- #
157
- # @return [Hash{String=>Pipeline}]
158
- def select_pipelines(&optional_state_filter)
159
- @states.each_with_object({}) do |(id, state), memo|
160
- if state && (!block_given? || yield(state))
161
- memo[id] = state.pipeline
162
- end
163
- end
164
- end
165
- end
166
- end
@@ -1,33 +0,0 @@
1
- require 'thread' # Mutex
2
-
3
- # A [LazySingleton] wraps the result of the provided block,
4
- # which is guaranteed to be called at-most-once, even if the
5
- # block's return value is nil.
6
- class ::LogStash::Util::LazySingleton
7
-
8
- def initialize(&block)
9
- @mutex = Mutex.new
10
- @block = block
11
- @instantiated = false
12
- end
13
-
14
- def instance
15
- unless @instantiated
16
- @mutex.synchronize do
17
- unless @instantiated
18
- @instance = @block.call
19
- @instantiated = true
20
- end
21
- end
22
- end
23
-
24
- return @instance
25
- end
26
-
27
- def reset!
28
- @mutex.synchronize do
29
- @instantiated = false
30
- @instance = nil
31
- end
32
- end
33
- end
@@ -1,15 +0,0 @@
1
- require 'spec_helper'
2
-
3
-
4
- describe "JRuby Version" do
5
- it "should break on JRuby version change" do
6
- # This spec will break upon JRuby version change to make sure we
7
- # verify if resolv.rb has been fixed in Jruby so we can get rid of
8
- # lib/logstash/patches/resolv.rb.
9
- # ref:
10
- # https://github.com/logstash-plugins/logstash-filter-dns/issues/51
11
- # https://github.com/jruby/jruby/pull/5722
12
- expect(JRUBY_VERSION).to eq("9.2.7.0")
13
- end
14
- end
15
-
@@ -1,220 +0,0 @@
1
- # encoding: utf-8
2
- require "spec_helper"
3
- require "logstash/pipelines_registry"
4
-
5
- describe LogStash::PipelinesRegistry do
6
-
7
- let(:pipeline_id) { "test" }
8
- let(:pipeline) { double("Pipeline") }
9
- let (:logger) { double("Logger") }
10
-
11
- context "at object creation" do
12
- it "should be empty" do
13
- expect(subject.size).to eq(0)
14
- expect(subject.empty?).to be_truthy
15
- expect(subject.running_pipelines).to be_empty
16
- expect(subject.non_running_pipelines).to be_empty
17
- expect(subject.running_user_defined_pipelines).to be_empty
18
- end
19
- end
20
-
21
- context "creating a pipeline" do
22
- context "without existing same pipeline id" do
23
- it "registry should not have a state for pipeline_id" do
24
- expect(subject.get_pipeline(pipeline_id)).to be_nil
25
- end
26
-
27
- it "should return block return value" do
28
- expect(subject.create_pipeline(pipeline_id, pipeline) { "dummy" }).to eq("dummy")
29
- end
30
-
31
- it "should register the new pipeline upon successful create block" do
32
- subject.create_pipeline(pipeline_id, pipeline) { true }
33
- expect(subject.get_pipeline(pipeline_id)).to eq(pipeline)
34
- end
35
-
36
- it "should not register the new pipeline upon unsuccessful create block" do
37
- subject.create_pipeline(pipeline_id, pipeline) { false }
38
- expect(subject.get_pipeline(pipeline_id)).to be_nil
39
- end
40
- end
41
-
42
- context "with existing pipeline id" do
43
- before :each do
44
- subject.create_pipeline(pipeline_id, pipeline) { true }
45
- end
46
-
47
- it "registry should have a state for pipeline_id" do
48
- expect(subject.get_pipeline(pipeline_id)).to eq(pipeline)
49
- end
50
-
51
- context "when existing pipeline is not terminated" do
52
- before :each do
53
- expect(pipeline).to receive(:finished_execution?).and_return(false)
54
- end
55
-
56
- it "should return false" do
57
- expect(subject.create_pipeline(pipeline_id, pipeline) { "dummy" }).to be_falsey
58
- end
59
-
60
- it "should not call block and log error if pipeline is not terminated" do
61
- expect(LogStash::PipelinesRegistry).to receive(:logger).and_return(logger)
62
- expect(logger).to receive(:error)
63
- expect { |b| subject.create_pipeline(pipeline_id, pipeline, &b) }.not_to yield_control
64
- end
65
- end
66
-
67
- context "when existing pipeline is terminated" do
68
- let (:new_pipeline) { double("New Pipeline") }
69
-
70
- before :each do
71
- expect(pipeline).to receive(:finished_execution?).and_return(true)
72
- end
73
-
74
- it "should return block value" do
75
- expect(subject.create_pipeline(pipeline_id, new_pipeline) { "dummy" }).to eq("dummy")
76
- end
77
-
78
- it "should return block value" do
79
- expect(subject.create_pipeline(pipeline_id, new_pipeline) { "dummy" }).to eq("dummy")
80
- end
81
-
82
- it "should register new pipeline" do
83
- subject.create_pipeline(pipeline_id, new_pipeline) { true }
84
- expect(subject.get_pipeline(pipeline_id)).to eq(new_pipeline)
85
- end
86
- end
87
- end
88
- end
89
-
90
- context "terminating a pipeline" do
91
- context "without existing pipeline id" do
92
- it "should log error" do
93
- expect(LogStash::PipelinesRegistry).to receive(:logger).and_return(logger)
94
- expect(logger).to receive(:error)
95
- subject.terminate_pipeline(pipeline_id) { "dummy" }
96
- end
97
-
98
- it "should not yield to block" do
99
- expect { |b| subject.terminate_pipeline(pipeline_id, &b) }.not_to yield_control
100
- end
101
- end
102
-
103
- context "with existing pipeline id" do
104
- before :each do
105
- subject.create_pipeline(pipeline_id, pipeline) { true }
106
- end
107
-
108
- it "should yield to block" do
109
- expect { |b| subject.terminate_pipeline(pipeline_id, &b) }.to yield_control
110
- end
111
-
112
- it "should keep pipeline id" do
113
- subject.terminate_pipeline(pipeline_id) { "dummy" }
114
- expect(subject.get_pipeline(pipeline_id)).to eq(pipeline)
115
- end
116
- end
117
- end
118
-
119
- context "reloading a pipeline" do
120
- it "should log error with inexistent pipeline id" do
121
- expect(LogStash::PipelinesRegistry).to receive(:logger).and_return(logger)
122
- expect(logger).to receive(:error)
123
- subject.reload_pipeline(pipeline_id) { }
124
- end
125
-
126
- context "with existing pipeline id" do
127
- before :each do
128
- subject.create_pipeline(pipeline_id, pipeline) { true }
129
- end
130
-
131
- it "should return block value" do
132
- expect(subject.reload_pipeline(pipeline_id) { ["dummy", pipeline] }).to eq("dummy")
133
- end
134
-
135
- it "should not be terminated while reloading" do
136
- expect(pipeline).to receive(:finished_execution?).and_return(false, true, true)
137
-
138
- # 1st call: finished_execution? is false
139
- expect(subject.running_pipelines).not_to be_empty
140
-
141
- # 2nd call: finished_execution? is true
142
- expect(subject.running_pipelines).to be_empty
143
-
144
-
145
- queue = Queue.new # threadsafe queue
146
- in_block = Concurrent::AtomicBoolean.new(false)
147
-
148
- thread = Thread.new(subject, pipeline_id, pipeline, queue, in_block) do |subject, pipeline_id, pipeline, queue, in_block|
149
- subject.reload_pipeline(pipeline_id) do
150
- in_block.make_true
151
- queue.pop
152
- [true, pipeline]
153
- end
154
- end
155
-
156
- # make sure we entered the block executioin
157
- wait(10).for {in_block.true?}.to be_truthy
158
-
159
- # at this point the thread is suspended waiting on queue
160
-
161
- # since in reloading state, running_pipelines is not empty
162
- expect(subject.running_pipelines).not_to be_empty
163
-
164
- # unblock thread
165
- queue.push(:dummy)
166
- thread.join
167
-
168
- # 3rd call: finished_execution? is true
169
- expect(subject.running_pipelines).to be_empty
170
- end
171
- end
172
- end
173
-
174
- context "pipelines collections" do
175
- context "with a non terminated pipelines" do
176
- before :each do
177
- subject.create_pipeline(pipeline_id, pipeline) { true }
178
- expect(pipeline).to receive(:finished_execution?).and_return(false)
179
- end
180
-
181
- it "should find running pipelines" do
182
- expect(subject.running_pipelines).not_to be_empty
183
- end
184
-
185
- it "should not find non_running pipelines" do
186
- expect(subject.non_running_pipelines).to be_empty
187
- end
188
-
189
- it "should find running_user_defined_pipelines" do
190
- expect(pipeline).to receive(:system?).and_return(false)
191
- expect(subject.running_user_defined_pipelines).not_to be_empty
192
- end
193
-
194
- it "should not find running_user_defined_pipelines" do
195
- expect(pipeline).to receive(:system?).and_return(true)
196
- expect(subject.running_user_defined_pipelines).to be_empty
197
- end
198
- end
199
-
200
- context "with a terminated pipelines" do
201
- before :each do
202
- subject.create_pipeline(pipeline_id, pipeline) { true }
203
- expect(pipeline).to receive(:finished_execution?).and_return(true)
204
- end
205
-
206
- it "should not find running pipelines" do
207
- expect(subject.running_pipelines).to be_empty
208
- end
209
-
210
- it "should find non_running pipelines" do
211
- expect(subject.non_running_pipelines).not_to be_empty
212
- end
213
-
214
- it "should not find running_user_defined_pipelines" do
215
- expect(subject.running_user_defined_pipelines).to be_empty
216
- end
217
- end
218
-
219
- end
220
- end