logstash-devutils 1.3.6-java → 2.0.4-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -0
- data/Gemfile +5 -0
- data/Rakefile +2 -0
- data/lib/logstash/devutils/rspec/logstash_helpers.rb +106 -56
- data/lib/logstash/devutils/rspec/spec_helper.rb +12 -18
- data/lib/logstash/outputs/test_sink.rb +108 -0
- data/lib/logstash/test_pipeline.rb +173 -0
- data/logstash-devutils.gemspec +18 -16
- metadata +26 -27
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1d34e698203e399da96e7498c7582dd598768b295943c23ee4030affcf1ca4f6
|
|
4
|
+
data.tar.gz: 2f92614519c0663e305f5c346cce5a6438eed765b4e429f814fc3317b91031a0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7fb5f164892910850f3240d31461de6b357820871f90845c2815f6320ef4599b9be2e9c8bb902aa0229de29b07a5c745599283d675f94d9048ad22eb355063bb
|
|
7
|
+
data.tar.gz: 71db99f7105959af5e641b75cfb55306dbb2afe677f352a4f85ccb6fabb0e0c074d64137b0fd2d001b2c2165346c63d7242b3f5894d31acc5487052de976fcab
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,28 @@
|
|
|
1
|
+
## 2.0.4
|
|
2
|
+
- Fix: avoid double registering filters on `sample` spec helper
|
|
3
|
+
|
|
4
|
+
## 2.0.3
|
|
5
|
+
- Fix: add missing `events` method to QueuedBatchDelegator, which was causing test failures
|
|
6
|
+
after https://github.com/elastic/logstash/pull/11737 was committed.
|
|
7
|
+
|
|
8
|
+
## 2.0.2
|
|
9
|
+
- Fix: add plain codec as runtime dependency for TestPipeline helper
|
|
10
|
+
|
|
11
|
+
## 2.0.1
|
|
12
|
+
- Fix: unwrap output and refactor test sink (#82)
|
|
13
|
+
|
|
14
|
+
## 2.0.0
|
|
15
|
+
- Reinvented helpers using Java pipeline, only LS >= 6.x (JRuby >= 9.1) is supported.
|
|
16
|
+
- [BREAKING] changes:
|
|
17
|
+
* `plugin_input` helper no longer works - simply fails with a not implemented error
|
|
18
|
+
* `type` and `tags` helpers have no effect - they will print a deprecation warning
|
|
19
|
+
* using gem **insist** is discouraged and has to be pulled in manually
|
|
20
|
+
(in *plugin.gemspec* `add_development_dependency 'insist'` and `require "insist"`)
|
|
21
|
+
* shared examples need to be explicitly required, as they are not re-used that much
|
|
22
|
+
(in spec_helper.rb `require "logstash/devutils/rspec/shared_examples"'`)
|
|
23
|
+
* `input` helper now yields a Queue-like collection (with `Queue#pop` blocking semantics)
|
|
24
|
+
with a default timeout polling mechanism to guard against potential dead-locks
|
|
25
|
+
|
|
1
26
|
## 1.3.6
|
|
2
27
|
- Revert the removal (e.g. add back) of the log4j spec helper. It is still needed for 5.x builds.
|
|
3
28
|
|
data/Gemfile
CHANGED
data/Rakefile
ADDED
|
@@ -1,16 +1,37 @@
|
|
|
1
1
|
require "logstash/agent"
|
|
2
|
-
require "logstash/pipeline"
|
|
3
2
|
require "logstash/event"
|
|
3
|
+
require "logstash/test_pipeline"
|
|
4
|
+
|
|
4
5
|
require "stud/try"
|
|
5
6
|
require "rspec/expectations"
|
|
6
|
-
|
|
7
|
+
|
|
8
|
+
require "logstash/environment"
|
|
7
9
|
|
|
8
10
|
module LogStashHelper
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
11
|
+
|
|
12
|
+
@@excluded_tags = {
|
|
13
|
+
:integration => true,
|
|
14
|
+
:redis => true,
|
|
15
|
+
:socket => true,
|
|
16
|
+
:performance => true,
|
|
17
|
+
:couchdb => true,
|
|
18
|
+
:elasticsearch => true,
|
|
19
|
+
:elasticsearch_secure => true,
|
|
20
|
+
:export_cypher => true
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if LogStash::Environment.windows?
|
|
24
|
+
@@excluded_tags[:unix] = true
|
|
25
|
+
else
|
|
26
|
+
@@excluded_tags[:windows] = true
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.excluded_tags
|
|
30
|
+
@@excluded_tags
|
|
12
31
|
end
|
|
13
32
|
|
|
33
|
+
TestPipeline = LogStash::TestPipeline
|
|
34
|
+
|
|
14
35
|
DEFAULT_NUMBER_OF_TRY = 5
|
|
15
36
|
DEFAULT_EXCEPTIONS_FOR_TRY = [RSpec::Expectations::ExpectationNotMetError]
|
|
16
37
|
|
|
@@ -23,12 +44,12 @@ module LogStashHelper
|
|
|
23
44
|
end # def config
|
|
24
45
|
|
|
25
46
|
def type(default_type)
|
|
26
|
-
|
|
47
|
+
deprecated "type(#{default_type.inspect}) no longer has any effect"
|
|
27
48
|
end
|
|
28
49
|
|
|
29
50
|
def tags(*tags)
|
|
30
51
|
let(:default_tags) { tags }
|
|
31
|
-
|
|
52
|
+
deprecated "tags(#{tags.inspect}) - let(:default_tags) are not used"
|
|
32
53
|
end
|
|
33
54
|
|
|
34
55
|
def sample(sample_event, &block)
|
|
@@ -46,18 +67,13 @@ module LogStashHelper
|
|
|
46
67
|
end
|
|
47
68
|
|
|
48
69
|
let(:results) do
|
|
49
|
-
|
|
50
|
-
pipeline.
|
|
51
|
-
|
|
52
|
-
event.each do |e|
|
|
53
|
-
# filter call the block on all filtered events, included new events added by the filter
|
|
54
|
-
pipeline.filter(e) { |filtered_event| results << filtered_event }
|
|
55
|
-
end
|
|
70
|
+
# Java pipeline (since 6.1) registers filters from start_workers
|
|
71
|
+
pipeline.run_with(event)
|
|
56
72
|
|
|
57
73
|
# flush makes sure to empty any buffered events in the filter
|
|
58
74
|
pipeline.flush_filters(:final => true) { |flushed_event| results << flushed_event }
|
|
59
75
|
|
|
60
|
-
|
|
76
|
+
pipeline.filter_queue_client.processed_events
|
|
61
77
|
end
|
|
62
78
|
|
|
63
79
|
# starting at logstash-core 5.3 an initialized pipeline need to be closed
|
|
@@ -71,68 +87,102 @@ module LogStashHelper
|
|
|
71
87
|
end
|
|
72
88
|
end # def sample
|
|
73
89
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
# output_func is now a method, call closure
|
|
83
|
-
def output_func(event)
|
|
84
|
-
@output_func.call(event)
|
|
85
|
-
# We want to return nil or [] since outputs aren't used here
|
|
86
|
-
# NOTE: In Ruby 1.9.x, Queue#<< returned nil, but in 2.x it returns the queue itself
|
|
87
|
-
# So we need to be explicit about the return
|
|
88
|
-
[]
|
|
89
|
-
end
|
|
90
|
-
end
|
|
90
|
+
org.logstash.config.ir.compiler.OutputStrategyExt::SimpleAbstractOutputStrategyExt.class_eval do
|
|
91
|
+
field_reader :output # available since LS 6.3
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def input(config_string, test_sink: {}, &block); require 'logstash/outputs/test_sink'
|
|
95
|
+
config_parts = [ config_source(config_string), test_sink_output_source(**test_sink) ]
|
|
96
|
+
|
|
97
|
+
pipeline = new_pipeline(config_parts)
|
|
91
98
|
|
|
92
|
-
|
|
93
|
-
|
|
99
|
+
output_delegator = pipeline.outputs.last # LogStash::OutputDelegator
|
|
100
|
+
fail('test_sink output expected') unless output_delegator.config_name.eql?('test_sink')
|
|
101
|
+
test_sink_output = output_delegator.strategy.to_java.output
|
|
102
|
+
queue = test_sink_output.init_event_store
|
|
94
103
|
|
|
104
|
+
start_thread = pipeline.start_and_wait
|
|
105
|
+
|
|
106
|
+
# NOTE: we used to pass a Queue here, now its a Java List/Queue collection
|
|
95
107
|
result = block.call(pipeline, queue)
|
|
96
108
|
|
|
97
109
|
pipeline.shutdown
|
|
98
|
-
|
|
110
|
+
start_thread.join if start_thread.alive?
|
|
99
111
|
|
|
100
112
|
result
|
|
101
|
-
end
|
|
113
|
+
end
|
|
102
114
|
|
|
115
|
+
# @deprecated
|
|
103
116
|
def plugin_input(plugin, &block)
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
input_thread = Thread.new do
|
|
107
|
-
plugin.run(queue)
|
|
108
|
-
end
|
|
109
|
-
result = block.call(queue)
|
|
110
|
-
|
|
111
|
-
plugin.do_stop
|
|
112
|
-
input_thread.join
|
|
113
|
-
result
|
|
117
|
+
raise NotImplementedError.new("#{__method__} no longer supported; please refactor")
|
|
114
118
|
end
|
|
115
119
|
|
|
116
120
|
def agent(&block)
|
|
117
|
-
|
|
118
121
|
it("agent(#{caller[0].gsub(/ .*/, "")}) runs") do
|
|
119
122
|
pipeline = new_pipeline_from_string(config)
|
|
120
123
|
pipeline.run
|
|
121
124
|
block.call
|
|
122
125
|
end
|
|
123
|
-
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def new_pipeline_from_string(config_string, pipeline_id: :main, test_sink: {})
|
|
129
|
+
config_parts = [ config_source(config_string) ]
|
|
130
|
+
|
|
131
|
+
# include a default test_sink output if no outputs given -> we're using it to track processed events
|
|
132
|
+
# NOTE: a output is required with the JavaPipeline otherwise no processing happen (despite filters being defined)
|
|
133
|
+
if !OUTPUT_BLOCK_RE.match(config_string)
|
|
134
|
+
config_parts << test_sink_output_source(**test_sink)
|
|
135
|
+
elsif test_sink && !test_sink.empty?
|
|
136
|
+
warn "#{__method__} test_sink: #{test_sink.inspect} options have no effect as config_string has an output"
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
if !INPUT_BLOCK_RE.match(config_string)
|
|
140
|
+
# NOTE: currently using manual batch to push events down the pipe, so an input isn't required
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
new_pipeline(config_parts, pipeline_id)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def new_pipeline(config_parts, pipeline_id = :main, settings = ::LogStash::SETTINGS.clone)
|
|
147
|
+
pipeline_config = LogStash::Config::PipelineConfig.new(LogStash::Config::Source::Local, pipeline_id, config_parts, settings)
|
|
148
|
+
TestPipeline.new(pipeline_config)
|
|
149
|
+
end
|
|
124
150
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
151
|
+
OUTPUT_BLOCK_RE = defined?(LogStash::Config::Source::Local::ConfigStringLoader::OUTPUT_BLOCK_RE) ?
|
|
152
|
+
LogStash::Config::Source::Local::ConfigStringLoader::OUTPUT_BLOCK_RE : /output *{/
|
|
153
|
+
private_constant :OUTPUT_BLOCK_RE
|
|
128
154
|
|
|
129
|
-
config_part = org.logstash.common.SourceWithMetadata.new("config_string", "config_string", string)
|
|
130
155
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
156
|
+
INPUT_BLOCK_RE = defined?(LogStash::Config::Source::Local::ConfigStringLoader::INPUT_BLOCK_RE) ?
|
|
157
|
+
LogStash::Config::Source::Local::ConfigStringLoader::INPUT_BLOCK_RE : /input *{/
|
|
158
|
+
private_constant :INPUT_BLOCK_RE
|
|
159
|
+
|
|
160
|
+
private
|
|
161
|
+
|
|
162
|
+
def config_source(config_string)
|
|
163
|
+
org.logstash.common.SourceWithMetadata.new("string", "config_string", config_string)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def test_sink_output_source(**config)
|
|
167
|
+
config = { id: current_spec_id }.merge(config).map { |k, v| "#{k} => #{v.is_a?(String) ? v.inspect : v}" }.join(' ')
|
|
168
|
+
output_string = "output { test_sink { #{config} } }" # TODO opts for performance store_events: false
|
|
169
|
+
org.logstash.common.SourceWithMetadata.new("string", "test_sink_output", output_string)
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def current_spec_id
|
|
173
|
+
@__current_example_metadata&.[](:location) || 'spec-sample'
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
if RUBY_VERSION > '2.5'
|
|
177
|
+
def deprecated(msg)
|
|
178
|
+
Kernel.warn(msg, uplevel: 1)
|
|
179
|
+
end
|
|
180
|
+
else # due JRuby 9.1 (Ruby 2.3)
|
|
181
|
+
def deprecated(msg)
|
|
182
|
+
loc = caller_locations[1]
|
|
183
|
+
Kernel.warn("#{loc.path}:#{loc.lineno}: warning: #{msg}")
|
|
135
184
|
end
|
|
136
185
|
end
|
|
186
|
+
|
|
137
187
|
end # module LogStash
|
|
138
188
|
|
|
@@ -14,10 +14,7 @@ end
|
|
|
14
14
|
|
|
15
15
|
require "logstash-core"
|
|
16
16
|
require "logstash/logging"
|
|
17
|
-
require "logstash/environment"
|
|
18
17
|
require "logstash/devutils/rspec/logstash_helpers"
|
|
19
|
-
require "logstash/devutils/rspec/shared_examples"
|
|
20
|
-
require "insist"
|
|
21
18
|
|
|
22
19
|
Thread.abort_on_exception = true
|
|
23
20
|
|
|
@@ -28,35 +25,32 @@ unless java.lang.System.getProperty("log4j.configurationFile")
|
|
|
28
25
|
end
|
|
29
26
|
|
|
30
27
|
$TESTING = true
|
|
31
|
-
if RUBY_VERSION < "
|
|
32
|
-
|
|
33
|
-
raise LoadError
|
|
28
|
+
if RUBY_VERSION < "2.3"
|
|
29
|
+
raise LoadError.new("Ruby >= 2.3.0 or later is required. (You are running: " + RUBY_VERSION + ")")
|
|
34
30
|
end
|
|
35
31
|
|
|
36
|
-
if ENV["TEST_DEBUG"]
|
|
37
|
-
|
|
32
|
+
if level = (ENV['LOG_LEVEL'] || ENV['LOGGER_LEVEL'] || ENV["TEST_DEBUG"])
|
|
33
|
+
logger, level = level.split('=') # 'logstash.filters.grok=DEBUG'
|
|
34
|
+
level, logger = logger, nil if level.nil? # only level given e.g. 'DEBUG'
|
|
35
|
+
level = org.apache.logging.log4j.Level.toLevel(level, org.apache.logging.log4j.Level::WARN)
|
|
36
|
+
LogStash::Logging::Logger::configure_logging(level.to_s, logger)
|
|
38
37
|
else
|
|
39
|
-
LogStash::Logging::Logger::configure_logging(
|
|
38
|
+
LogStash::Logging::Logger::configure_logging('ERROR')
|
|
40
39
|
end
|
|
41
40
|
|
|
42
|
-
# removed the strictness check, it did not seem to catch anything
|
|
43
|
-
|
|
44
41
|
RSpec.configure do |config|
|
|
45
42
|
# for now both include and extend are required because the newly refactored "input" helper method need to be visible in a "it" block
|
|
46
43
|
# and this is only possible by calling include on LogStashHelper
|
|
47
44
|
config.include LogStashHelper
|
|
48
45
|
config.extend LogStashHelper
|
|
49
46
|
|
|
50
|
-
|
|
47
|
+
config.filter_run_excluding LogStashHelper.excluded_tags
|
|
51
48
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
exclude_tags[:windows] = true
|
|
49
|
+
config.around(:each) do |example|
|
|
50
|
+
@__current_example_metadata = example.metadata
|
|
51
|
+
example.run
|
|
56
52
|
end
|
|
57
53
|
|
|
58
|
-
config.filter_run_excluding exclude_tags
|
|
59
|
-
|
|
60
54
|
# Run specs in random order to surface order dependencies. If you find an
|
|
61
55
|
# order dependency and want to debug it, you can fix the order by providing
|
|
62
56
|
# the seed, which is printed after each run.
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
require "logstash/namespace"
|
|
3
|
+
require "logstash/outputs/base"
|
|
4
|
+
require "logstash/errors"
|
|
5
|
+
|
|
6
|
+
# This output simply discards (but tracks) received events.
|
|
7
|
+
class LogStash::Outputs::TestSink < LogStash::Outputs::Base
|
|
8
|
+
|
|
9
|
+
concurrency :shared
|
|
10
|
+
|
|
11
|
+
config_name "test_sink"
|
|
12
|
+
|
|
13
|
+
# Whether we're tracking events received or simply act as a true sink.
|
|
14
|
+
config :store_events, :validate => :boolean, :default => true
|
|
15
|
+
# Plugin could not release itself (on close) if needed to keep its events around.
|
|
16
|
+
config :release_on_close, :validate => :boolean, :default => true
|
|
17
|
+
# Even poll timeout in milliseconds (for input plugin tests consuming events).
|
|
18
|
+
# HINT: set to 0 (null) to wait forever.
|
|
19
|
+
config :event_poll_timeout, :default => 5 * 1000
|
|
20
|
+
|
|
21
|
+
TRACKER = java.util.WeakHashMap.new
|
|
22
|
+
|
|
23
|
+
# @override plugin hook
|
|
24
|
+
def register
|
|
25
|
+
TRACKER[self] = @_event_store || init_event_store
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# @override plugin impl
|
|
29
|
+
def receive(event)
|
|
30
|
+
event_store << event if store_events?
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# @override plugin hook
|
|
34
|
+
def close
|
|
35
|
+
TRACKER.delete(self)
|
|
36
|
+
@_event_store = false if release_on_close?
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def store_events?
|
|
40
|
+
!!@store_events
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def release_on_close?
|
|
44
|
+
!!@release_on_close
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Clears the event store.
|
|
48
|
+
def clear!
|
|
49
|
+
event_store.clear
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# @return [Queue] (enumerable) event store
|
|
53
|
+
def event_store
|
|
54
|
+
if @_event_store.nil?
|
|
55
|
+
warn("#{self} event store not initialized (call plugin.register to initialize)", caller_locations(2))
|
|
56
|
+
return init_event_store
|
|
57
|
+
elsif @_event_store.eql?(false)
|
|
58
|
+
warn("#{self} closed - event store no longer available (release_on_close => false if you need to retain events)", caller_locations(2))
|
|
59
|
+
return nil
|
|
60
|
+
end
|
|
61
|
+
@_event_store
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def init_event_store
|
|
65
|
+
@_event_store = Queue.new(@event_poll_timeout.to_f * 1000)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private
|
|
69
|
+
|
|
70
|
+
def warn(msg, callstack = nil)
|
|
71
|
+
Kernel.warn("#{msg}#{callstack ? " called from #{callstack.first}" : nil}")
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# TODO refactor to java.util.concurrent.ConcurrentLinkedQueue
|
|
75
|
+
# Interestingly, using a ConcurrentLinkedQueue gets specs that depend on pop-ing events
|
|
76
|
+
# from the output (e.g. syslog input plugin) passing fine with the old Ruby pipeline.
|
|
77
|
+
# The Java pipeline seems to reach a shutdown before the input yielded events are to be consumed.
|
|
78
|
+
|
|
79
|
+
class Queue < java.util.concurrent.LinkedBlockingQueue
|
|
80
|
+
|
|
81
|
+
java_import java.util.concurrent.TimeoutException
|
|
82
|
+
|
|
83
|
+
MILLISECONDS = java.util.concurrent.TimeUnit::MILLISECONDS
|
|
84
|
+
private_constant :MILLISECONDS
|
|
85
|
+
|
|
86
|
+
def initialize(poll_timeout = nil)
|
|
87
|
+
super()
|
|
88
|
+
@timeout = poll_timeout.to_i
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Ruby Queue like pop-er with (default) blocking.
|
|
92
|
+
# @see Queue#pop
|
|
93
|
+
def pop(non_block = nil)
|
|
94
|
+
# for compatibility we're making it behave like Ruby's Queue
|
|
95
|
+
return poll if non_block
|
|
96
|
+
@timeout.zero? ? take :
|
|
97
|
+
(poll(@timeout, MILLISECONDS) || timeout!(__method__))
|
|
98
|
+
end
|
|
99
|
+
alias deq pop
|
|
100
|
+
alias shift pop
|
|
101
|
+
|
|
102
|
+
private
|
|
103
|
+
|
|
104
|
+
def timeout!(op)
|
|
105
|
+
raise TimeoutException.new("#{op.inspect} did not complete (in #{@timeout}ms)")
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
require "logstash/pipeline"
|
|
2
|
+
require "logstash/java_pipeline"
|
|
3
|
+
|
|
4
|
+
module LogStash
|
|
5
|
+
class TestPipeline < LogStash::JavaPipeline
|
|
6
|
+
public :flush_filters
|
|
7
|
+
|
|
8
|
+
def run_with(events)
|
|
9
|
+
if inputs&.any? # will work but might be unintended
|
|
10
|
+
config = "\n #{config_str}" if $VERBOSE
|
|
11
|
+
warn "#{self} pipeline is getting events pushed manually while having inputs: #{inputs.inspect} #{config}"
|
|
12
|
+
end
|
|
13
|
+
# TODO could we handle a generator (Enumerator) ?
|
|
14
|
+
queue.write_client.push_batch events.to_a
|
|
15
|
+
queue_read_client = filter_queue_client
|
|
16
|
+
# potentially handle run_with called multiple times (re-use the same instance) :
|
|
17
|
+
if queue_read_client.is_a?(EventTrackingQueueReadClientDelegator)
|
|
18
|
+
queue_read_client.reset_events!
|
|
19
|
+
else
|
|
20
|
+
# start_worker using @filter_queue_client on 6.3, since 6.4 a reader method is used
|
|
21
|
+
# to make things compatible with 6.3 we're assigning the @filter_queue_client ivar
|
|
22
|
+
@filter_queue_client = EventTrackingQueueReadClientDelegator.new queue_read_client
|
|
23
|
+
end
|
|
24
|
+
run
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @override for WorkerLoop to pick it up
|
|
28
|
+
# @note only works since LS 6.4 (need to use tha actual ivar for 6.3)
|
|
29
|
+
def filter_queue_client
|
|
30
|
+
@filter_queue_client || super
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
java_import org.apache.logging.log4j.ThreadContext unless const_defined?(:ThreadContext)
|
|
34
|
+
|
|
35
|
+
def start_and_wait
|
|
36
|
+
parent_thread = Thread.current
|
|
37
|
+
@finished_execution.make_false
|
|
38
|
+
@finished_run&.make_false # only since 6.5
|
|
39
|
+
|
|
40
|
+
@thread = Thread.new do
|
|
41
|
+
begin
|
|
42
|
+
LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
|
|
43
|
+
ThreadContext.put("pipeline.id", pipeline_id)
|
|
44
|
+
run
|
|
45
|
+
@finished_run&.make_true
|
|
46
|
+
rescue => e
|
|
47
|
+
close
|
|
48
|
+
parent_thread.raise(e)
|
|
49
|
+
ensure
|
|
50
|
+
@finished_execution.make_true
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
unless wait_until_started
|
|
55
|
+
raise "failed to start pipeline: #{self}\n with config: #{config_str.inspect}"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
@thread
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
class EventTrackingQueueReadClientDelegator
|
|
62
|
+
include org.logstash.execution.QueueReadClient
|
|
63
|
+
java_import org.logstash.execution.QueueReadClient
|
|
64
|
+
|
|
65
|
+
attr_reader :processed_events
|
|
66
|
+
|
|
67
|
+
def initialize(delegate)
|
|
68
|
+
# NOTE: can not use LogStash::MemoryReadClient#read_batch due its JavaObject wrapping
|
|
69
|
+
@delegate = delegate.to_java(QueueReadClient)
|
|
70
|
+
@processed_events = []
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# @override QueueBatch readBatch() throws InterruptedException;
|
|
74
|
+
def readBatch
|
|
75
|
+
QueueBatchDelegator.new(self, @delegate.read_batch)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# @override void closeBatch(QueueBatch batch) throws IOException;
|
|
79
|
+
def closeBatch(batch)
|
|
80
|
+
@delegate.close_batch(batch)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# @override boolean isEmpty();
|
|
84
|
+
def isEmpty
|
|
85
|
+
@delegate.empty?
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# @override QueueBatch newBatch();
|
|
89
|
+
def newBatch
|
|
90
|
+
@delegate.new_batch
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# @override void startMetrics(QueueBatch batch);
|
|
94
|
+
def startMetrics(batch)
|
|
95
|
+
@delegate.start_metrics(batch)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# @override void addOutputMetrics(int filteredSize);
|
|
99
|
+
def addOutputMetrics(filteredSize)
|
|
100
|
+
@delegate.add_output_metrics(filteredSize)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# @override void addFilteredMetrics(int filteredSize);
|
|
104
|
+
def addFilteredMetrics(filteredSize)
|
|
105
|
+
@delegate.add_filtered_metrics(filteredSize)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# @override
|
|
109
|
+
def set_batch_dimensions(batch_size, batch_delay)
|
|
110
|
+
@delegate.set_batch_dimensions(batch_size, batch_delay)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# @override
|
|
114
|
+
def close
|
|
115
|
+
@delegate.close
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# @override dispatch to delegate
|
|
119
|
+
def method_missing(method, *args)
|
|
120
|
+
@delegate.public_send(method, *args)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def filtered_events(events)
|
|
124
|
+
@processed_events.concat(events)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def reset_events!
|
|
128
|
+
@processed_events = []
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
class QueueBatchDelegator
|
|
134
|
+
include org.logstash.execution.QueueBatch
|
|
135
|
+
|
|
136
|
+
def initialize(event_tracker, delegate)
|
|
137
|
+
@event_tracker = event_tracker
|
|
138
|
+
@delegate = delegate
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# @override RubyArray to_a();
|
|
142
|
+
def to_a
|
|
143
|
+
@delegate.to_a.tap do |events|
|
|
144
|
+
# filters out rogue (cancelled) events
|
|
145
|
+
@event_tracker.filtered_events events
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# @override int filteredSize();
|
|
150
|
+
def filteredSize
|
|
151
|
+
@delegate.to_java.filtered_size
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def events
|
|
155
|
+
@delegate.events.tap do |events|
|
|
156
|
+
# filters out rogue (cancelled) events
|
|
157
|
+
@event_tracker.filtered_events events
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# @override void merge(IRubyObject event);
|
|
162
|
+
def merge(event)
|
|
163
|
+
@delegate.merge(event)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# @override void close() throws IOException;
|
|
167
|
+
def close
|
|
168
|
+
@delegate.close
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|
data/logstash-devutils.gemspec
CHANGED
|
@@ -1,28 +1,30 @@
|
|
|
1
|
-
if RUBY_PLATFORM != "java"
|
|
2
|
-
raise "Only JRuby is supported"
|
|
3
|
-
end
|
|
1
|
+
raise "Only JRuby is supported" if RUBY_PLATFORM != "java"
|
|
4
2
|
|
|
5
3
|
Gem::Specification.new do |spec|
|
|
6
4
|
files = %x{git ls-files}.split("\n")
|
|
7
5
|
|
|
8
6
|
spec.name = "logstash-devutils"
|
|
9
|
-
spec.version = "
|
|
10
|
-
spec.
|
|
11
|
-
spec.summary = "logstash-devutils"
|
|
12
|
-
spec.description = "logstash-devutils"
|
|
7
|
+
spec.version = "2.0.4"
|
|
8
|
+
spec.license = "Apache-2.0"
|
|
13
9
|
spec.authors = ["Elastic"]
|
|
14
10
|
spec.email = "info@elastic.co"
|
|
15
11
|
spec.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
|
16
12
|
|
|
13
|
+
spec.summary = %q{An assortment of tooling/libraries to make Logstash plugin development and releasing a bit easier.}
|
|
14
|
+
spec.description = %q{logstash-devutils gem is meant to be used as a development dependency from other plugins/gems.}
|
|
15
|
+
|
|
17
16
|
spec.files = files
|
|
18
17
|
spec.require_paths << "lib"
|
|
19
18
|
|
|
20
19
|
spec.platform = "java"
|
|
21
20
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
#
|
|
25
|
-
#
|
|
21
|
+
spec.required_ruby_version = '>= 2.3'
|
|
22
|
+
|
|
23
|
+
# Please note that devutils is meant to be used as a development dependency from other plugins/gems.
|
|
24
|
+
# As such, devutils' OWN development dependencies (any add_development_dependency) will be ignored when
|
|
25
|
+
# bundling a plugin/gem which uses/depends on devutils.
|
|
26
|
+
#
|
|
27
|
+
# This is why all devutils own dependencies should normally be specified as add_runtime_dependency.
|
|
26
28
|
|
|
27
29
|
# It is important to specify rspec "~> 3.0" and not "~> 3.0.0" (or any other version to the patch level)
|
|
28
30
|
# otherwise the version constrain will have to be met up to the minor release version and only allow
|
|
@@ -35,13 +37,13 @@ Gem::Specification.new do |spec|
|
|
|
35
37
|
spec.add_runtime_dependency "rake" # MIT License
|
|
36
38
|
spec.add_runtime_dependency "gem_publisher" # MIT License
|
|
37
39
|
spec.add_runtime_dependency "minitar" # GPL2|Ruby License
|
|
38
|
-
spec.add_runtime_dependency "logstash-core-plugin-api", "<= 2.99", ">= 2.0"
|
|
39
40
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
#
|
|
43
|
-
spec.add_runtime_dependency "insist"
|
|
41
|
+
spec.add_runtime_dependency "logstash-core", ">= 6.3"
|
|
42
|
+
|
|
43
|
+
# Some plugins are (still) using insist by their own, but we no longer force this dependency on others.
|
|
44
|
+
#spec.add_runtime_dependency "insist" # (Apache 2.0 license)
|
|
44
45
|
spec.add_runtime_dependency "kramdown", '1.14.0'
|
|
45
46
|
spec.add_runtime_dependency "stud", " >= 0.0.20"
|
|
46
47
|
spec.add_runtime_dependency "fivemat"
|
|
48
|
+
spec.add_runtime_dependency "logstash-codec-plain"
|
|
47
49
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-devutils
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 2.0.4
|
|
5
5
|
platform: java
|
|
6
6
|
authors:
|
|
7
7
|
- Elastic
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2020-11-07 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -83,72 +83,66 @@ dependencies:
|
|
|
83
83
|
- !ruby/object:Gem::Dependency
|
|
84
84
|
requirement: !ruby/object:Gem::Requirement
|
|
85
85
|
requirements:
|
|
86
|
-
- - "<="
|
|
87
|
-
- !ruby/object:Gem::Version
|
|
88
|
-
version: '2.99'
|
|
89
86
|
- - ">="
|
|
90
87
|
- !ruby/object:Gem::Version
|
|
91
|
-
version: '
|
|
92
|
-
name: logstash-core
|
|
88
|
+
version: '6.3'
|
|
89
|
+
name: logstash-core
|
|
93
90
|
prerelease: false
|
|
94
91
|
type: :runtime
|
|
95
92
|
version_requirements: !ruby/object:Gem::Requirement
|
|
96
93
|
requirements:
|
|
97
|
-
- - "<="
|
|
98
|
-
- !ruby/object:Gem::Version
|
|
99
|
-
version: '2.99'
|
|
100
94
|
- - ">="
|
|
101
95
|
- !ruby/object:Gem::Version
|
|
102
|
-
version: '
|
|
96
|
+
version: '6.3'
|
|
103
97
|
- !ruby/object:Gem::Dependency
|
|
104
98
|
requirement: !ruby/object:Gem::Requirement
|
|
105
99
|
requirements:
|
|
106
100
|
- - '='
|
|
107
101
|
- !ruby/object:Gem::Version
|
|
108
|
-
version: 1.
|
|
109
|
-
name:
|
|
102
|
+
version: 1.14.0
|
|
103
|
+
name: kramdown
|
|
110
104
|
prerelease: false
|
|
111
105
|
type: :runtime
|
|
112
106
|
version_requirements: !ruby/object:Gem::Requirement
|
|
113
107
|
requirements:
|
|
114
108
|
- - '='
|
|
115
109
|
- !ruby/object:Gem::Version
|
|
116
|
-
version: 1.
|
|
110
|
+
version: 1.14.0
|
|
117
111
|
- !ruby/object:Gem::Dependency
|
|
118
112
|
requirement: !ruby/object:Gem::Requirement
|
|
119
113
|
requirements:
|
|
120
|
-
- -
|
|
114
|
+
- - ">="
|
|
121
115
|
- !ruby/object:Gem::Version
|
|
122
|
-
version:
|
|
123
|
-
name:
|
|
116
|
+
version: 0.0.20
|
|
117
|
+
name: stud
|
|
124
118
|
prerelease: false
|
|
125
119
|
type: :runtime
|
|
126
120
|
version_requirements: !ruby/object:Gem::Requirement
|
|
127
121
|
requirements:
|
|
128
|
-
- -
|
|
122
|
+
- - ">="
|
|
129
123
|
- !ruby/object:Gem::Version
|
|
130
|
-
version:
|
|
124
|
+
version: 0.0.20
|
|
131
125
|
- !ruby/object:Gem::Dependency
|
|
132
126
|
requirement: !ruby/object:Gem::Requirement
|
|
133
127
|
requirements:
|
|
134
128
|
- - ">="
|
|
135
129
|
- !ruby/object:Gem::Version
|
|
136
|
-
version: 0
|
|
137
|
-
name:
|
|
130
|
+
version: '0'
|
|
131
|
+
name: fivemat
|
|
138
132
|
prerelease: false
|
|
139
133
|
type: :runtime
|
|
140
134
|
version_requirements: !ruby/object:Gem::Requirement
|
|
141
135
|
requirements:
|
|
142
136
|
- - ">="
|
|
143
137
|
- !ruby/object:Gem::Version
|
|
144
|
-
version: 0
|
|
138
|
+
version: '0'
|
|
145
139
|
- !ruby/object:Gem::Dependency
|
|
146
140
|
requirement: !ruby/object:Gem::Requirement
|
|
147
141
|
requirements:
|
|
148
142
|
- - ">="
|
|
149
143
|
- !ruby/object:Gem::Version
|
|
150
144
|
version: '0'
|
|
151
|
-
name:
|
|
145
|
+
name: logstash-codec-plain
|
|
152
146
|
prerelease: false
|
|
153
147
|
type: :runtime
|
|
154
148
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -156,7 +150,8 @@ dependencies:
|
|
|
156
150
|
- - ">="
|
|
157
151
|
- !ruby/object:Gem::Version
|
|
158
152
|
version: '0'
|
|
159
|
-
description: logstash-devutils
|
|
153
|
+
description: logstash-devutils gem is meant to be used as a development dependency
|
|
154
|
+
from other plugins/gems.
|
|
160
155
|
email: info@elastic.co
|
|
161
156
|
executables: []
|
|
162
157
|
extensions: []
|
|
@@ -165,6 +160,7 @@ files:
|
|
|
165
160
|
- CHANGELOG.md
|
|
166
161
|
- Gemfile
|
|
167
162
|
- LICENSE
|
|
163
|
+
- Rakefile
|
|
168
164
|
- lib/logstash/devutils/rake.rb
|
|
169
165
|
- lib/logstash/devutils/rake/publish.rake
|
|
170
166
|
- lib/logstash/devutils/rake/vendor.rake
|
|
@@ -172,10 +168,12 @@ files:
|
|
|
172
168
|
- lib/logstash/devutils/rspec/logstash_helpers.rb
|
|
173
169
|
- lib/logstash/devutils/rspec/shared_examples.rb
|
|
174
170
|
- lib/logstash/devutils/rspec/spec_helper.rb
|
|
171
|
+
- lib/logstash/outputs/test_sink.rb
|
|
172
|
+
- lib/logstash/test_pipeline.rb
|
|
175
173
|
- logstash-devutils.gemspec
|
|
176
174
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
|
177
175
|
licenses:
|
|
178
|
-
- Apache
|
|
176
|
+
- Apache-2.0
|
|
179
177
|
metadata: {}
|
|
180
178
|
post_install_message:
|
|
181
179
|
rdoc_options: []
|
|
@@ -186,7 +184,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
186
184
|
requirements:
|
|
187
185
|
- - ">="
|
|
188
186
|
- !ruby/object:Gem::Version
|
|
189
|
-
version: '
|
|
187
|
+
version: '2.3'
|
|
190
188
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
191
189
|
requirements:
|
|
192
190
|
- - ">="
|
|
@@ -197,5 +195,6 @@ rubyforge_project:
|
|
|
197
195
|
rubygems_version: 2.6.13
|
|
198
196
|
signing_key:
|
|
199
197
|
specification_version: 4
|
|
200
|
-
summary:
|
|
198
|
+
summary: An assortment of tooling/libraries to make Logstash plugin development and
|
|
199
|
+
releasing a bit easier.
|
|
201
200
|
test_files: []
|