logstash-devutils 1.3.6-java → 2.0.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/Gemfile +4 -0
- data/Rakefile +2 -0
- data/lib/logstash/devutils/rspec/logstash_helpers.rb +101 -55
- data/lib/logstash/devutils/rspec/spec_helper.rb +12 -18
- data/lib/logstash/outputs/test_sink.rb +90 -0
- data/lib/logstash/test_pipeline.rb +166 -0
- data/logstash-devutils.gemspec +17 -16
- metadata +14 -29
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9529292d4be6bf7ae845758206433d9fff38a8fdf068d4eee49581a595022563
|
|
4
|
+
data.tar.gz: '09bac2f78a40a15e156f893929a0113bb815234f5e24dacd0b806d94a7db7784'
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2c1f07ee224a06900492e5421031b1f8e4a15705edf0feb91bfe4279853cb30a3bd248933209e20e16d91e19c65f7b8dd6e29b5a75ee293b80c5e3c5032aa4b9
|
|
7
|
+
data.tar.gz: 1b151df23dd77671425a0874a4b0903de5460d10f93a8b521c8019ff4f71965b8ff52089d2978eebe3dfd0b91e1342dac9ecaffbde6a3fa3735a7a938482b81e
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
## 2.0.0
|
|
2
|
+
- Reinvented helpers using Java pipeline, only LS >= 6.x (JRuby >= 9.1) is supported.
|
|
3
|
+
- [BREAKING] changes:
|
|
4
|
+
* `plugin_input` helper no longer works - simply fails with a not implemented error
|
|
5
|
+
* `type` and `tags` helpers have no effect - they will print a deprecation warning
|
|
6
|
+
* using gem **insist** is discouraged and has to be pulled in manually
|
|
7
|
+
(in *plugin.gemspec* `add_development_dependency 'insist'` and `require "insist"`)
|
|
8
|
+
* shared examples need to be explicitly required, as they are not re-used that much
|
|
9
|
+
(in spec_helper.rb `require "logstash/devutils/rspec/shared_examples"'`)
|
|
10
|
+
* `input` helper now yields a Queue-like collection (with `Queue#pop` blocking semantics)
|
|
11
|
+
with a default timeout polling mechanism to guard against potential dead-locks
|
|
12
|
+
|
|
1
13
|
## 1.3.6
|
|
2
14
|
- Revert the removal (e.g. add back) of the log4j spec helper. It is still needed for 5.x builds.
|
|
3
15
|
|
data/Gemfile
CHANGED
data/Rakefile
ADDED
|
@@ -1,16 +1,37 @@
|
|
|
1
1
|
require "logstash/agent"
|
|
2
|
-
require "logstash/pipeline"
|
|
3
2
|
require "logstash/event"
|
|
3
|
+
require "logstash/test_pipeline"
|
|
4
|
+
|
|
4
5
|
require "stud/try"
|
|
5
6
|
require "rspec/expectations"
|
|
6
|
-
|
|
7
|
+
|
|
8
|
+
require "logstash/environment"
|
|
7
9
|
|
|
8
10
|
module LogStashHelper
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
11
|
+
|
|
12
|
+
@@excluded_tags = {
|
|
13
|
+
:integration => true,
|
|
14
|
+
:redis => true,
|
|
15
|
+
:socket => true,
|
|
16
|
+
:performance => true,
|
|
17
|
+
:couchdb => true,
|
|
18
|
+
:elasticsearch => true,
|
|
19
|
+
:elasticsearch_secure => true,
|
|
20
|
+
:export_cypher => true
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if LogStash::Environment.windows?
|
|
24
|
+
@@excluded_tags[:unix] = true
|
|
25
|
+
else
|
|
26
|
+
@@excluded_tags[:windows] = true
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.excluded_tags
|
|
30
|
+
@@excluded_tags
|
|
12
31
|
end
|
|
13
32
|
|
|
33
|
+
TestPipeline = LogStash::TestPipeline
|
|
34
|
+
|
|
14
35
|
DEFAULT_NUMBER_OF_TRY = 5
|
|
15
36
|
DEFAULT_EXCEPTIONS_FOR_TRY = [RSpec::Expectations::ExpectationNotMetError]
|
|
16
37
|
|
|
@@ -23,12 +44,12 @@ module LogStashHelper
|
|
|
23
44
|
end # def config
|
|
24
45
|
|
|
25
46
|
def type(default_type)
|
|
26
|
-
|
|
47
|
+
deprecated "type(#{default_type.inspect}) no longer has any effect"
|
|
27
48
|
end
|
|
28
49
|
|
|
29
50
|
def tags(*tags)
|
|
30
51
|
let(:default_tags) { tags }
|
|
31
|
-
|
|
52
|
+
deprecated "tags(#{tags.inspect}) - let(:default_tags) are not used"
|
|
32
53
|
end
|
|
33
54
|
|
|
34
55
|
def sample(sample_event, &block)
|
|
@@ -46,18 +67,14 @@ module LogStashHelper
|
|
|
46
67
|
end
|
|
47
68
|
|
|
48
69
|
let(:results) do
|
|
49
|
-
|
|
50
|
-
pipeline.instance_eval { @filters.each(&:register) }
|
|
70
|
+
pipeline.filters.each(&:register)
|
|
51
71
|
|
|
52
|
-
event
|
|
53
|
-
# filter call the block on all filtered events, included new events added by the filter
|
|
54
|
-
pipeline.filter(e) { |filtered_event| results << filtered_event }
|
|
55
|
-
end
|
|
72
|
+
pipeline.run_with(event)
|
|
56
73
|
|
|
57
74
|
# flush makes sure to empty any buffered events in the filter
|
|
58
75
|
pipeline.flush_filters(:final => true) { |flushed_event| results << flushed_event }
|
|
59
76
|
|
|
60
|
-
|
|
77
|
+
pipeline.filter_queue_client.processed_events
|
|
61
78
|
end
|
|
62
79
|
|
|
63
80
|
# starting at logstash-core 5.3 an initialized pipeline need to be closed
|
|
@@ -71,68 +88,97 @@ module LogStashHelper
|
|
|
71
88
|
end
|
|
72
89
|
end # def sample
|
|
73
90
|
|
|
74
|
-
def input(
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
@output_func = lambda { |event| queue << event }
|
|
81
|
-
|
|
82
|
-
# output_func is now a method, call closure
|
|
83
|
-
def output_func(event)
|
|
84
|
-
@output_func.call(event)
|
|
85
|
-
# We want to return nil or [] since outputs aren't used here
|
|
86
|
-
# NOTE: In Ruby 1.9.x, Queue#<< returned nil, but in 2.x it returns the queue itself
|
|
87
|
-
# So we need to be explicit about the return
|
|
88
|
-
[]
|
|
89
|
-
end
|
|
90
|
-
end
|
|
91
|
+
def input(config_string, test_sink: {}, &block); require 'logstash/outputs/test_sink'
|
|
92
|
+
config_parts = [ config_source(config_string), test_sink_output_source(**test_sink) ]
|
|
93
|
+
|
|
94
|
+
# TODO unwrapping output from LogStash::OutputDelegator is cumbersome
|
|
95
|
+
instances = LogStash::Outputs::TestSink::TRACKER.keys.to_a
|
|
96
|
+
pipeline = new_pipeline(config_parts)
|
|
91
97
|
|
|
92
|
-
|
|
93
|
-
sleep 0.1 while !pipeline.ready?
|
|
98
|
+
start_thread = pipeline.start_and_wait
|
|
94
99
|
|
|
100
|
+
queue = (LogStash::Outputs::TestSink::TRACKER.keys.to_a - instances).first.event_store
|
|
101
|
+
|
|
102
|
+
# NOTE: we used to pass a Queue here, now its a Java List/Queue collection
|
|
95
103
|
result = block.call(pipeline, queue)
|
|
96
104
|
|
|
97
105
|
pipeline.shutdown
|
|
98
|
-
|
|
106
|
+
start_thread.join if start_thread.alive?
|
|
99
107
|
|
|
100
108
|
result
|
|
101
|
-
end
|
|
109
|
+
end
|
|
102
110
|
|
|
111
|
+
# @deprecated
|
|
103
112
|
def plugin_input(plugin, &block)
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
input_thread = Thread.new do
|
|
107
|
-
plugin.run(queue)
|
|
108
|
-
end
|
|
109
|
-
result = block.call(queue)
|
|
110
|
-
|
|
111
|
-
plugin.do_stop
|
|
112
|
-
input_thread.join
|
|
113
|
-
result
|
|
113
|
+
raise NotImplementedError.new("#{__method__} no longer supported; please refactor")
|
|
114
114
|
end
|
|
115
115
|
|
|
116
116
|
def agent(&block)
|
|
117
|
-
|
|
118
117
|
it("agent(#{caller[0].gsub(/ .*/, "")}) runs") do
|
|
119
118
|
pipeline = new_pipeline_from_string(config)
|
|
120
119
|
pipeline.run
|
|
121
120
|
block.call
|
|
122
121
|
end
|
|
123
|
-
end
|
|
122
|
+
end
|
|
124
123
|
|
|
125
|
-
def new_pipeline_from_string(
|
|
126
|
-
|
|
127
|
-
settings = ::LogStash::SETTINGS.clone
|
|
124
|
+
def new_pipeline_from_string(config_string, pipeline_id: :main, test_sink: {})
|
|
125
|
+
config_parts = [ config_source(config_string) ]
|
|
128
126
|
|
|
129
|
-
|
|
127
|
+
# include a default test_sink output if no outputs given -> we're using it to track processed events
|
|
128
|
+
# NOTE: a output is required with the JavaPipeline otherwise no processing happen (despite filters being defined)
|
|
129
|
+
if !OUTPUT_BLOCK_RE.match(config_string)
|
|
130
|
+
config_parts << test_sink_output_source(**test_sink)
|
|
131
|
+
elsif test_sink && !test_sink.empty?
|
|
132
|
+
warn "#{__method__} test_sink: #{test_sink.inspect} options have no effect as config_string has an output"
|
|
133
|
+
end
|
|
130
134
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
else
|
|
134
|
-
TestPipeline.new(string)
|
|
135
|
+
if !INPUT_BLOCK_RE.match(config_string)
|
|
136
|
+
# NOTE: currently using manual batch to push events down the pipe, so an input isn't required
|
|
135
137
|
end
|
|
138
|
+
|
|
139
|
+
new_pipeline(config_parts, pipeline_id)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def new_pipeline(config_parts, pipeline_id = :main, settings = ::LogStash::SETTINGS.clone)
|
|
143
|
+
pipeline_config = LogStash::Config::PipelineConfig.new(LogStash::Config::Source::Local, pipeline_id, config_parts, settings)
|
|
144
|
+
TestPipeline.new(pipeline_config)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
OUTPUT_BLOCK_RE = defined?(LogStash::Config::Source::Local::ConfigStringLoader::OUTPUT_BLOCK_RE) ?
|
|
148
|
+
LogStash::Config::Source::Local::ConfigStringLoader::OUTPUT_BLOCK_RE : /output *{/
|
|
149
|
+
private_constant :OUTPUT_BLOCK_RE
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
INPUT_BLOCK_RE = defined?(LogStash::Config::Source::Local::ConfigStringLoader::INPUT_BLOCK_RE) ?
|
|
153
|
+
LogStash::Config::Source::Local::ConfigStringLoader::INPUT_BLOCK_RE : /input *{/
|
|
154
|
+
private_constant :INPUT_BLOCK_RE
|
|
155
|
+
|
|
156
|
+
private
|
|
157
|
+
|
|
158
|
+
def config_source(config_string)
|
|
159
|
+
org.logstash.common.SourceWithMetadata.new("string", "config_string", config_string)
|
|
136
160
|
end
|
|
161
|
+
|
|
162
|
+
def test_sink_output_source(**config)
|
|
163
|
+
config = { id: current_spec_id }.merge(config).map { |k, v| "#{k} => #{v.is_a?(String) ? v.inspect : v}" }.join(' ')
|
|
164
|
+
output_string = "output { test_sink { #{config} } }" # TODO opts for performance store_events: false
|
|
165
|
+
org.logstash.common.SourceWithMetadata.new("string", "test_sink_output", output_string)
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def current_spec_id
|
|
169
|
+
@__current_example_metadata&.[](:location) || 'spec-sample'
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
if RUBY_VERSION > '2.5'
|
|
173
|
+
def deprecated(msg)
|
|
174
|
+
Kernel.warn(msg, uplevel: 1)
|
|
175
|
+
end
|
|
176
|
+
else # due JRuby 9.1 (Ruby 2.3)
|
|
177
|
+
def deprecated(msg)
|
|
178
|
+
loc = caller_locations[1]
|
|
179
|
+
Kernel.warn("#{loc.path}:#{loc.lineno}: warning: #{msg}")
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
137
183
|
end # module LogStash
|
|
138
184
|
|
|
@@ -14,10 +14,7 @@ end
|
|
|
14
14
|
|
|
15
15
|
require "logstash-core"
|
|
16
16
|
require "logstash/logging"
|
|
17
|
-
require "logstash/environment"
|
|
18
17
|
require "logstash/devutils/rspec/logstash_helpers"
|
|
19
|
-
require "logstash/devutils/rspec/shared_examples"
|
|
20
|
-
require "insist"
|
|
21
18
|
|
|
22
19
|
Thread.abort_on_exception = true
|
|
23
20
|
|
|
@@ -28,35 +25,32 @@ unless java.lang.System.getProperty("log4j.configurationFile")
|
|
|
28
25
|
end
|
|
29
26
|
|
|
30
27
|
$TESTING = true
|
|
31
|
-
if RUBY_VERSION < "
|
|
32
|
-
|
|
33
|
-
raise LoadError
|
|
28
|
+
if RUBY_VERSION < "2.3"
|
|
29
|
+
raise LoadError.new("Ruby >= 2.3.0 or later is required. (You are running: " + RUBY_VERSION + ")")
|
|
34
30
|
end
|
|
35
31
|
|
|
36
|
-
if ENV["TEST_DEBUG"]
|
|
37
|
-
|
|
32
|
+
if level = (ENV["TEST_DEBUG"] || ENV['LOGGER_LEVEL'])
|
|
33
|
+
logger, level = level.split('=') # 'logstash.filters.grok=DEBUG'
|
|
34
|
+
level, logger = logger, nil if level.nil? # only level given e.g. 'DEBUG'
|
|
35
|
+
level = org.apache.logging.log4j.Level.toLevel(level, org.apache.logging.log4j.Level::WARN)
|
|
36
|
+
LogStash::Logging::Logger::configure_logging(level.to_s, logger)
|
|
38
37
|
else
|
|
39
|
-
LogStash::Logging::Logger::configure_logging(
|
|
38
|
+
LogStash::Logging::Logger::configure_logging('ERROR')
|
|
40
39
|
end
|
|
41
40
|
|
|
42
|
-
# removed the strictness check, it did not seem to catch anything
|
|
43
|
-
|
|
44
41
|
RSpec.configure do |config|
|
|
45
42
|
# for now both include and extend are required because the newly refactored "input" helper method need to be visible in a "it" block
|
|
46
43
|
# and this is only possible by calling include on LogStashHelper
|
|
47
44
|
config.include LogStashHelper
|
|
48
45
|
config.extend LogStashHelper
|
|
49
46
|
|
|
50
|
-
|
|
47
|
+
config.filter_run_excluding LogStashHelper.excluded_tags
|
|
51
48
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
exclude_tags[:windows] = true
|
|
49
|
+
config.around(:each) do |example|
|
|
50
|
+
@__current_example_metadata = example.metadata
|
|
51
|
+
example.run
|
|
56
52
|
end
|
|
57
53
|
|
|
58
|
-
config.filter_run_excluding exclude_tags
|
|
59
|
-
|
|
60
54
|
# Run specs in random order to surface order dependencies. If you find an
|
|
61
55
|
# order dependency and want to debug it, you can fix the order by providing
|
|
62
56
|
# the seed, which is printed after each run.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
require "logstash/namespace"
|
|
3
|
+
require "logstash/outputs/base"
|
|
4
|
+
require "logstash/errors"
|
|
5
|
+
|
|
6
|
+
# This output simply discards (but tracks) received events.
|
|
7
|
+
class LogStash::Outputs::TestSink < LogStash::Outputs::Base
|
|
8
|
+
|
|
9
|
+
concurrency :shared
|
|
10
|
+
|
|
11
|
+
config_name "test_sink"
|
|
12
|
+
|
|
13
|
+
# Whether we're tracking events received or simply act as a true sink.
|
|
14
|
+
config :store_events, :validate => :boolean, :default => true
|
|
15
|
+
# Plugin could not release itself (on close) if needed to keep its events around.
|
|
16
|
+
config :release_on_close, :validate => :boolean, :default => true
|
|
17
|
+
# Even poll timeout in milliseconds (for input plugin tests consuming events).
|
|
18
|
+
# HINT: set to 0 (null) to wait forever.
|
|
19
|
+
config :event_poll_timeout, :default => 5 * 1000
|
|
20
|
+
|
|
21
|
+
TRACKER = java.util.WeakHashMap.new
|
|
22
|
+
|
|
23
|
+
# @override plugin hook
|
|
24
|
+
def register
|
|
25
|
+
TRACKER[self] = Queue.new(@event_poll_timeout.to_f * 1000)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# @override plugin impl
|
|
29
|
+
def receive(event)
|
|
30
|
+
event_store << event if store_events?
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# @override plugin hook
|
|
34
|
+
def close
|
|
35
|
+
TRACKER.delete(self) if release_on_close?
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def store_events?
|
|
39
|
+
!!@store_events
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def release_on_close?
|
|
43
|
+
!!@release_on_close
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Clears the event store.
|
|
47
|
+
def clear!
|
|
48
|
+
event_store.clear
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# @return [Queue] (enumerable) event store
|
|
52
|
+
def event_store
|
|
53
|
+
TRACKER[self] || raise("#{self} not registered; please call plugin.register before use")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# TODO refactor to java.util.concurrent.ConcurrentLinkedQueue
|
|
57
|
+
# Interestingly, using a ConcurrentLinkedQueue gets specs that depend on pop-ing events
|
|
58
|
+
# from the output (e.g. syslog input plugin) passing fine with the old Ruby pipeline.
|
|
59
|
+
# The Java pipeline seems to reach a shutdown before the input yielded events are to be consumed.
|
|
60
|
+
|
|
61
|
+
class Queue < java.util.concurrent.LinkedBlockingQueue
|
|
62
|
+
|
|
63
|
+
java_import java.util.concurrent.TimeoutException
|
|
64
|
+
|
|
65
|
+
MILLISECONDS = java.util.concurrent.TimeUnit::MILLISECONDS
|
|
66
|
+
private_constant :MILLISECONDS
|
|
67
|
+
|
|
68
|
+
def initialize(poll_timeout = nil)
|
|
69
|
+
super()
|
|
70
|
+
@timeout = poll_timeout.to_i
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Ruby Queue like pop-er with (default) blocking.
|
|
74
|
+
# @see Queue#pop
|
|
75
|
+
def pop(non_block = nil)
|
|
76
|
+
# for compatibility we're making it behave like Ruby's Queue
|
|
77
|
+
return poll if non_block
|
|
78
|
+
@timeout.zero? ? take :
|
|
79
|
+
(poll(@timeout, MILLISECONDS) || timeout!(__method__))
|
|
80
|
+
end
|
|
81
|
+
alias deq pop
|
|
82
|
+
alias shift pop
|
|
83
|
+
|
|
84
|
+
private
|
|
85
|
+
|
|
86
|
+
def timeout!(op)
|
|
87
|
+
raise TimeoutException.new("#{op.inspect} did not complete (in #{@timeout}ms)")
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
require "logstash/pipeline"
|
|
2
|
+
require "logstash/java_pipeline"
|
|
3
|
+
|
|
4
|
+
module LogStash
|
|
5
|
+
class TestPipeline < LogStash::JavaPipeline
|
|
6
|
+
public :flush_filters
|
|
7
|
+
|
|
8
|
+
def run_with(events)
|
|
9
|
+
if inputs&.any? # will work but might be unintended
|
|
10
|
+
config = "\n #{config_str}" if $VERBOSE
|
|
11
|
+
warn "#{self} pipeline is getting events pushed manually while having inputs: #{inputs.inspect} #{config}"
|
|
12
|
+
end
|
|
13
|
+
# TODO could we handle a generator (Enumerator) ?
|
|
14
|
+
queue.write_client.push_batch events.to_a
|
|
15
|
+
queue_read_client = filter_queue_client
|
|
16
|
+
# potentially handle run_with called multiple times (re-use the same instance) :
|
|
17
|
+
if queue_read_client.is_a?(EventTrackingQueueReadClientDelegator)
|
|
18
|
+
queue_read_client.reset_events!
|
|
19
|
+
else
|
|
20
|
+
# start_worker using @filter_queue_client on 6.3, since 6.4 a reader method is used
|
|
21
|
+
# to make things compatible with 6.3 we're assigning the @filter_queue_client ivar
|
|
22
|
+
@filter_queue_client = EventTrackingQueueReadClientDelegator.new queue_read_client
|
|
23
|
+
end
|
|
24
|
+
run
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @override for WorkerLoop to pick it up
|
|
28
|
+
# @note only works since LS 6.4 (need to use tha actual ivar for 6.3)
|
|
29
|
+
def filter_queue_client
|
|
30
|
+
@filter_queue_client || super
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
java_import org.apache.logging.log4j.ThreadContext unless const_defined?(:ThreadContext)
|
|
34
|
+
|
|
35
|
+
def start_and_wait
|
|
36
|
+
parent_thread = Thread.current
|
|
37
|
+
@finished_execution.make_false
|
|
38
|
+
@finished_run&.make_false # only since 6.5
|
|
39
|
+
|
|
40
|
+
@thread = Thread.new do
|
|
41
|
+
begin
|
|
42
|
+
LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
|
|
43
|
+
ThreadContext.put("pipeline.id", pipeline_id)
|
|
44
|
+
run
|
|
45
|
+
@finished_run&.make_true
|
|
46
|
+
rescue => e
|
|
47
|
+
close
|
|
48
|
+
parent_thread.raise(e)
|
|
49
|
+
ensure
|
|
50
|
+
@finished_execution.make_true
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
unless wait_until_started
|
|
55
|
+
raise "failed to start pipeline: #{self}\n with config: #{config_str.inspect}"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
@thread
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
class EventTrackingQueueReadClientDelegator
|
|
62
|
+
include org.logstash.execution.QueueReadClient
|
|
63
|
+
java_import org.logstash.execution.QueueReadClient
|
|
64
|
+
|
|
65
|
+
attr_reader :processed_events
|
|
66
|
+
|
|
67
|
+
def initialize(delegate)
|
|
68
|
+
# NOTE: can not use LogStash::MemoryReadClient#read_batch due its JavaObject wrapping
|
|
69
|
+
@delegate = delegate.to_java(QueueReadClient)
|
|
70
|
+
@processed_events = []
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# @override QueueBatch readBatch() throws InterruptedException;
|
|
74
|
+
def readBatch
|
|
75
|
+
QueueBatchDelegator.new(self, @delegate.read_batch)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# @override void closeBatch(QueueBatch batch) throws IOException;
|
|
79
|
+
def closeBatch(batch)
|
|
80
|
+
@delegate.close_batch(batch)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# @override boolean isEmpty();
|
|
84
|
+
def isEmpty
|
|
85
|
+
@delegate.empty?
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# @override QueueBatch newBatch();
|
|
89
|
+
def newBatch
|
|
90
|
+
@delegate.new_batch
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# @override void startMetrics(QueueBatch batch);
|
|
94
|
+
def startMetrics(batch)
|
|
95
|
+
@delegate.start_metrics(batch)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# @override void addOutputMetrics(int filteredSize);
|
|
99
|
+
def addOutputMetrics(filteredSize)
|
|
100
|
+
@delegate.add_output_metrics(filteredSize)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# @override void addFilteredMetrics(int filteredSize);
|
|
104
|
+
def addFilteredMetrics(filteredSize)
|
|
105
|
+
@delegate.add_filtered_metrics(filteredSize)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# @override
|
|
109
|
+
def set_batch_dimensions(batch_size, batch_delay)
|
|
110
|
+
@delegate.set_batch_dimensions(batch_size, batch_delay)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# @override
|
|
114
|
+
def close
|
|
115
|
+
@delegate.close
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# @override dispatch to delegate
|
|
119
|
+
def method_missing(method, *args)
|
|
120
|
+
@delegate.public_send(method, *args)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def filtered_events(events)
|
|
124
|
+
@processed_events.concat(events)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def reset_events!
|
|
128
|
+
@processed_events = []
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
class QueueBatchDelegator
|
|
134
|
+
include org.logstash.execution.QueueBatch
|
|
135
|
+
|
|
136
|
+
def initialize(event_tracker, delegate)
|
|
137
|
+
@event_tracker = event_tracker
|
|
138
|
+
@delegate = delegate
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# @override RubyArray to_a();
|
|
142
|
+
def to_a
|
|
143
|
+
@delegate.to_a.tap do |events|
|
|
144
|
+
# filters out rogue (cancelled) events
|
|
145
|
+
@event_tracker.filtered_events events
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# @override int filteredSize();
|
|
150
|
+
def filteredSize
|
|
151
|
+
@delegate.to_java.filtered_size
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# @override void merge(IRubyObject event);
|
|
155
|
+
def merge(event)
|
|
156
|
+
@delegate.merge(event)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# @override void close() throws IOException;
|
|
160
|
+
def close
|
|
161
|
+
@delegate.close
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
data/logstash-devutils.gemspec
CHANGED
|
@@ -1,28 +1,30 @@
|
|
|
1
|
-
if RUBY_PLATFORM != "java"
|
|
2
|
-
raise "Only JRuby is supported"
|
|
3
|
-
end
|
|
1
|
+
raise "Only JRuby is supported" if RUBY_PLATFORM != "java"
|
|
4
2
|
|
|
5
3
|
Gem::Specification.new do |spec|
|
|
6
4
|
files = %x{git ls-files}.split("\n")
|
|
7
5
|
|
|
8
6
|
spec.name = "logstash-devutils"
|
|
9
|
-
spec.version = "
|
|
10
|
-
spec.
|
|
11
|
-
spec.summary = "logstash-devutils"
|
|
12
|
-
spec.description = "logstash-devutils"
|
|
7
|
+
spec.version = "2.0.0"
|
|
8
|
+
spec.license = "Apache-2.0"
|
|
13
9
|
spec.authors = ["Elastic"]
|
|
14
10
|
spec.email = "info@elastic.co"
|
|
15
11
|
spec.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
|
16
12
|
|
|
13
|
+
spec.summary = %q{An assortment of tooling/libraries to make Logstash plugin development and releasing a bit easier.}
|
|
14
|
+
spec.description = %q{logstash-devutils gem is meant to be used as a development dependency from other plugins/gems.}
|
|
15
|
+
|
|
17
16
|
spec.files = files
|
|
18
17
|
spec.require_paths << "lib"
|
|
19
18
|
|
|
20
19
|
spec.platform = "java"
|
|
21
20
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
#
|
|
25
|
-
#
|
|
21
|
+
spec.required_ruby_version = '>= 2.3'
|
|
22
|
+
|
|
23
|
+
# Please note that devutils is meant to be used as a development dependency from other plugins/gems.
|
|
24
|
+
# As such, devutils' OWN development dependencies (any add_development_dependency) will be ignored when
|
|
25
|
+
# bundling a plugin/gem which uses/depends on devutils.
|
|
26
|
+
#
|
|
27
|
+
# This is why all devutils own dependencies should normally be specified as add_runtime_dependency.
|
|
26
28
|
|
|
27
29
|
# It is important to specify rspec "~> 3.0" and not "~> 3.0.0" (or any other version to the patch level)
|
|
28
30
|
# otherwise the version constrain will have to be met up to the minor release version and only allow
|
|
@@ -35,12 +37,11 @@ Gem::Specification.new do |spec|
|
|
|
35
37
|
spec.add_runtime_dependency "rake" # MIT License
|
|
36
38
|
spec.add_runtime_dependency "gem_publisher" # MIT License
|
|
37
39
|
spec.add_runtime_dependency "minitar" # GPL2|Ruby License
|
|
38
|
-
spec.add_runtime_dependency "logstash-core-plugin-api", "<= 2.99", ">= 2.0"
|
|
39
40
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
#
|
|
43
|
-
spec.add_runtime_dependency "insist"
|
|
41
|
+
spec.add_runtime_dependency "logstash-core", ">= 6.3"
|
|
42
|
+
|
|
43
|
+
# Some plugins are (still) using insist by their own, but we no longer force this dependency on others.
|
|
44
|
+
#spec.add_runtime_dependency "insist" # (Apache 2.0 license)
|
|
44
45
|
spec.add_runtime_dependency "kramdown", '1.14.0'
|
|
45
46
|
spec.add_runtime_dependency "stud", " >= 0.0.20"
|
|
46
47
|
spec.add_runtime_dependency "fivemat"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-devutils
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 2.0.0
|
|
5
5
|
platform: java
|
|
6
6
|
authors:
|
|
7
7
|
- Elastic
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2020-02-11 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -83,37 +83,17 @@ dependencies:
|
|
|
83
83
|
- !ruby/object:Gem::Dependency
|
|
84
84
|
requirement: !ruby/object:Gem::Requirement
|
|
85
85
|
requirements:
|
|
86
|
-
- - "<="
|
|
87
|
-
- !ruby/object:Gem::Version
|
|
88
|
-
version: '2.99'
|
|
89
86
|
- - ">="
|
|
90
87
|
- !ruby/object:Gem::Version
|
|
91
|
-
version: '
|
|
92
|
-
name: logstash-core
|
|
88
|
+
version: '6.3'
|
|
89
|
+
name: logstash-core
|
|
93
90
|
prerelease: false
|
|
94
91
|
type: :runtime
|
|
95
92
|
version_requirements: !ruby/object:Gem::Requirement
|
|
96
93
|
requirements:
|
|
97
|
-
- - "<="
|
|
98
|
-
- !ruby/object:Gem::Version
|
|
99
|
-
version: '2.99'
|
|
100
94
|
- - ">="
|
|
101
95
|
- !ruby/object:Gem::Version
|
|
102
|
-
version: '
|
|
103
|
-
- !ruby/object:Gem::Dependency
|
|
104
|
-
requirement: !ruby/object:Gem::Requirement
|
|
105
|
-
requirements:
|
|
106
|
-
- - '='
|
|
107
|
-
- !ruby/object:Gem::Version
|
|
108
|
-
version: 1.0.0
|
|
109
|
-
name: insist
|
|
110
|
-
prerelease: false
|
|
111
|
-
type: :runtime
|
|
112
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
113
|
-
requirements:
|
|
114
|
-
- - '='
|
|
115
|
-
- !ruby/object:Gem::Version
|
|
116
|
-
version: 1.0.0
|
|
96
|
+
version: '6.3'
|
|
117
97
|
- !ruby/object:Gem::Dependency
|
|
118
98
|
requirement: !ruby/object:Gem::Requirement
|
|
119
99
|
requirements:
|
|
@@ -156,7 +136,8 @@ dependencies:
|
|
|
156
136
|
- - ">="
|
|
157
137
|
- !ruby/object:Gem::Version
|
|
158
138
|
version: '0'
|
|
159
|
-
description: logstash-devutils
|
|
139
|
+
description: logstash-devutils gem is meant to be used as a development dependency
|
|
140
|
+
from other plugins/gems.
|
|
160
141
|
email: info@elastic.co
|
|
161
142
|
executables: []
|
|
162
143
|
extensions: []
|
|
@@ -165,6 +146,7 @@ files:
|
|
|
165
146
|
- CHANGELOG.md
|
|
166
147
|
- Gemfile
|
|
167
148
|
- LICENSE
|
|
149
|
+
- Rakefile
|
|
168
150
|
- lib/logstash/devutils/rake.rb
|
|
169
151
|
- lib/logstash/devutils/rake/publish.rake
|
|
170
152
|
- lib/logstash/devutils/rake/vendor.rake
|
|
@@ -172,10 +154,12 @@ files:
|
|
|
172
154
|
- lib/logstash/devutils/rspec/logstash_helpers.rb
|
|
173
155
|
- lib/logstash/devutils/rspec/shared_examples.rb
|
|
174
156
|
- lib/logstash/devutils/rspec/spec_helper.rb
|
|
157
|
+
- lib/logstash/outputs/test_sink.rb
|
|
158
|
+
- lib/logstash/test_pipeline.rb
|
|
175
159
|
- logstash-devutils.gemspec
|
|
176
160
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
|
177
161
|
licenses:
|
|
178
|
-
- Apache
|
|
162
|
+
- Apache-2.0
|
|
179
163
|
metadata: {}
|
|
180
164
|
post_install_message:
|
|
181
165
|
rdoc_options: []
|
|
@@ -186,7 +170,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
186
170
|
requirements:
|
|
187
171
|
- - ">="
|
|
188
172
|
- !ruby/object:Gem::Version
|
|
189
|
-
version: '
|
|
173
|
+
version: '2.3'
|
|
190
174
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
191
175
|
requirements:
|
|
192
176
|
- - ">="
|
|
@@ -197,5 +181,6 @@ rubyforge_project:
|
|
|
197
181
|
rubygems_version: 2.6.13
|
|
198
182
|
signing_key:
|
|
199
183
|
specification_version: 4
|
|
200
|
-
summary:
|
|
184
|
+
summary: An assortment of tooling/libraries to make Logstash plugin development and
|
|
185
|
+
releasing a bit easier.
|
|
201
186
|
test_files: []
|