logstash-devutils 1.3.6-java → 2.0.4-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -0
- data/Gemfile +5 -0
- data/Rakefile +2 -0
- data/lib/logstash/devutils/rspec/logstash_helpers.rb +106 -56
- data/lib/logstash/devutils/rspec/spec_helper.rb +12 -18
- data/lib/logstash/outputs/test_sink.rb +108 -0
- data/lib/logstash/test_pipeline.rb +173 -0
- data/logstash-devutils.gemspec +18 -16
- metadata +26 -27
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1d34e698203e399da96e7498c7582dd598768b295943c23ee4030affcf1ca4f6
|
4
|
+
data.tar.gz: 2f92614519c0663e305f5c346cce5a6438eed765b4e429f814fc3317b91031a0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7fb5f164892910850f3240d31461de6b357820871f90845c2815f6320ef4599b9be2e9c8bb902aa0229de29b07a5c745599283d675f94d9048ad22eb355063bb
|
7
|
+
data.tar.gz: 71db99f7105959af5e641b75cfb55306dbb2afe677f352a4f85ccb6fabb0e0c074d64137b0fd2d001b2c2165346c63d7242b3f5894d31acc5487052de976fcab
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,28 @@
|
|
1
|
+
## 2.0.4
|
2
|
+
- Fix: avoid double registering filters on `sample` spec helper
|
3
|
+
|
4
|
+
## 2.0.3
|
5
|
+
- Fix: add missing `events` method to QueuedBatchDelegator, which was causing test failures
|
6
|
+
after https://github.com/elastic/logstash/pull/11737 was committed.
|
7
|
+
|
8
|
+
## 2.0.2
|
9
|
+
- Fix: add plain codec as runtime dependency for TestPipeline helper
|
10
|
+
|
11
|
+
## 2.0.1
|
12
|
+
- Fix: unwrap output and refactor test sink (#82)
|
13
|
+
|
14
|
+
## 2.0.0
|
15
|
+
- Reinvented helpers using Java pipeline, only LS >= 6.x (JRuby >= 9.1) is supported.
|
16
|
+
- [BREAKING] changes:
|
17
|
+
* `plugin_input` helper no longer works - simply fails with a not implemented error
|
18
|
+
* `type` and `tags` helpers have no effect - they will print a deprecation warning
|
19
|
+
* using gem **insist** is discouraged and has to be pulled in manually
|
20
|
+
(in *plugin.gemspec* `add_development_dependency 'insist'` and `require "insist"`)
|
21
|
+
* shared examples need to be explicitly required, as they are not re-used that much
|
22
|
+
(in spec_helper.rb `require "logstash/devutils/rspec/shared_examples"'`)
|
23
|
+
* `input` helper now yields a Queue-like collection (with `Queue#pop` blocking semantics)
|
24
|
+
with a default timeout polling mechanism to guard against potential dead-locks
|
25
|
+
|
1
26
|
## 1.3.6
|
2
27
|
- Revert the removal (e.g. add back) of the log4j spec helper. It is still needed for 5.x builds.
|
3
28
|
|
data/Gemfile
CHANGED
data/Rakefile
ADDED
@@ -1,16 +1,37 @@
|
|
1
1
|
require "logstash/agent"
|
2
|
-
require "logstash/pipeline"
|
3
2
|
require "logstash/event"
|
3
|
+
require "logstash/test_pipeline"
|
4
|
+
|
4
5
|
require "stud/try"
|
5
6
|
require "rspec/expectations"
|
6
|
-
|
7
|
+
|
8
|
+
require "logstash/environment"
|
7
9
|
|
8
10
|
module LogStashHelper
|
9
|
-
|
10
|
-
|
11
|
-
|
11
|
+
|
12
|
+
@@excluded_tags = {
|
13
|
+
:integration => true,
|
14
|
+
:redis => true,
|
15
|
+
:socket => true,
|
16
|
+
:performance => true,
|
17
|
+
:couchdb => true,
|
18
|
+
:elasticsearch => true,
|
19
|
+
:elasticsearch_secure => true,
|
20
|
+
:export_cypher => true
|
21
|
+
}
|
22
|
+
|
23
|
+
if LogStash::Environment.windows?
|
24
|
+
@@excluded_tags[:unix] = true
|
25
|
+
else
|
26
|
+
@@excluded_tags[:windows] = true
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.excluded_tags
|
30
|
+
@@excluded_tags
|
12
31
|
end
|
13
32
|
|
33
|
+
TestPipeline = LogStash::TestPipeline
|
34
|
+
|
14
35
|
DEFAULT_NUMBER_OF_TRY = 5
|
15
36
|
DEFAULT_EXCEPTIONS_FOR_TRY = [RSpec::Expectations::ExpectationNotMetError]
|
16
37
|
|
@@ -23,12 +44,12 @@ module LogStashHelper
|
|
23
44
|
end # def config
|
24
45
|
|
25
46
|
def type(default_type)
|
26
|
-
|
47
|
+
deprecated "type(#{default_type.inspect}) no longer has any effect"
|
27
48
|
end
|
28
49
|
|
29
50
|
def tags(*tags)
|
30
51
|
let(:default_tags) { tags }
|
31
|
-
|
52
|
+
deprecated "tags(#{tags.inspect}) - let(:default_tags) are not used"
|
32
53
|
end
|
33
54
|
|
34
55
|
def sample(sample_event, &block)
|
@@ -46,18 +67,13 @@ module LogStashHelper
|
|
46
67
|
end
|
47
68
|
|
48
69
|
let(:results) do
|
49
|
-
|
50
|
-
pipeline.
|
51
|
-
|
52
|
-
event.each do |e|
|
53
|
-
# filter call the block on all filtered events, included new events added by the filter
|
54
|
-
pipeline.filter(e) { |filtered_event| results << filtered_event }
|
55
|
-
end
|
70
|
+
# Java pipeline (since 6.1) registers filters from start_workers
|
71
|
+
pipeline.run_with(event)
|
56
72
|
|
57
73
|
# flush makes sure to empty any buffered events in the filter
|
58
74
|
pipeline.flush_filters(:final => true) { |flushed_event| results << flushed_event }
|
59
75
|
|
60
|
-
|
76
|
+
pipeline.filter_queue_client.processed_events
|
61
77
|
end
|
62
78
|
|
63
79
|
# starting at logstash-core 5.3 an initialized pipeline need to be closed
|
@@ -71,68 +87,102 @@ module LogStashHelper
|
|
71
87
|
end
|
72
88
|
end # def sample
|
73
89
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
# output_func is now a method, call closure
|
83
|
-
def output_func(event)
|
84
|
-
@output_func.call(event)
|
85
|
-
# We want to return nil or [] since outputs aren't used here
|
86
|
-
# NOTE: In Ruby 1.9.x, Queue#<< returned nil, but in 2.x it returns the queue itself
|
87
|
-
# So we need to be explicit about the return
|
88
|
-
[]
|
89
|
-
end
|
90
|
-
end
|
90
|
+
org.logstash.config.ir.compiler.OutputStrategyExt::SimpleAbstractOutputStrategyExt.class_eval do
|
91
|
+
field_reader :output # available since LS 6.3
|
92
|
+
end
|
93
|
+
|
94
|
+
def input(config_string, test_sink: {}, &block); require 'logstash/outputs/test_sink'
|
95
|
+
config_parts = [ config_source(config_string), test_sink_output_source(**test_sink) ]
|
96
|
+
|
97
|
+
pipeline = new_pipeline(config_parts)
|
91
98
|
|
92
|
-
|
93
|
-
|
99
|
+
output_delegator = pipeline.outputs.last # LogStash::OutputDelegator
|
100
|
+
fail('test_sink output expected') unless output_delegator.config_name.eql?('test_sink')
|
101
|
+
test_sink_output = output_delegator.strategy.to_java.output
|
102
|
+
queue = test_sink_output.init_event_store
|
94
103
|
|
104
|
+
start_thread = pipeline.start_and_wait
|
105
|
+
|
106
|
+
# NOTE: we used to pass a Queue here, now its a Java List/Queue collection
|
95
107
|
result = block.call(pipeline, queue)
|
96
108
|
|
97
109
|
pipeline.shutdown
|
98
|
-
|
110
|
+
start_thread.join if start_thread.alive?
|
99
111
|
|
100
112
|
result
|
101
|
-
end
|
113
|
+
end
|
102
114
|
|
115
|
+
# @deprecated
|
103
116
|
def plugin_input(plugin, &block)
|
104
|
-
|
105
|
-
|
106
|
-
input_thread = Thread.new do
|
107
|
-
plugin.run(queue)
|
108
|
-
end
|
109
|
-
result = block.call(queue)
|
110
|
-
|
111
|
-
plugin.do_stop
|
112
|
-
input_thread.join
|
113
|
-
result
|
117
|
+
raise NotImplementedError.new("#{__method__} no longer supported; please refactor")
|
114
118
|
end
|
115
119
|
|
116
120
|
def agent(&block)
|
117
|
-
|
118
121
|
it("agent(#{caller[0].gsub(/ .*/, "")}) runs") do
|
119
122
|
pipeline = new_pipeline_from_string(config)
|
120
123
|
pipeline.run
|
121
124
|
block.call
|
122
125
|
end
|
123
|
-
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def new_pipeline_from_string(config_string, pipeline_id: :main, test_sink: {})
|
129
|
+
config_parts = [ config_source(config_string) ]
|
130
|
+
|
131
|
+
# include a default test_sink output if no outputs given -> we're using it to track processed events
|
132
|
+
# NOTE: a output is required with the JavaPipeline otherwise no processing happen (despite filters being defined)
|
133
|
+
if !OUTPUT_BLOCK_RE.match(config_string)
|
134
|
+
config_parts << test_sink_output_source(**test_sink)
|
135
|
+
elsif test_sink && !test_sink.empty?
|
136
|
+
warn "#{__method__} test_sink: #{test_sink.inspect} options have no effect as config_string has an output"
|
137
|
+
end
|
138
|
+
|
139
|
+
if !INPUT_BLOCK_RE.match(config_string)
|
140
|
+
# NOTE: currently using manual batch to push events down the pipe, so an input isn't required
|
141
|
+
end
|
142
|
+
|
143
|
+
new_pipeline(config_parts, pipeline_id)
|
144
|
+
end
|
145
|
+
|
146
|
+
def new_pipeline(config_parts, pipeline_id = :main, settings = ::LogStash::SETTINGS.clone)
|
147
|
+
pipeline_config = LogStash::Config::PipelineConfig.new(LogStash::Config::Source::Local, pipeline_id, config_parts, settings)
|
148
|
+
TestPipeline.new(pipeline_config)
|
149
|
+
end
|
124
150
|
|
125
|
-
|
126
|
-
|
127
|
-
|
151
|
+
OUTPUT_BLOCK_RE = defined?(LogStash::Config::Source::Local::ConfigStringLoader::OUTPUT_BLOCK_RE) ?
|
152
|
+
LogStash::Config::Source::Local::ConfigStringLoader::OUTPUT_BLOCK_RE : /output *{/
|
153
|
+
private_constant :OUTPUT_BLOCK_RE
|
128
154
|
|
129
|
-
config_part = org.logstash.common.SourceWithMetadata.new("config_string", "config_string", string)
|
130
155
|
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
156
|
+
INPUT_BLOCK_RE = defined?(LogStash::Config::Source::Local::ConfigStringLoader::INPUT_BLOCK_RE) ?
|
157
|
+
LogStash::Config::Source::Local::ConfigStringLoader::INPUT_BLOCK_RE : /input *{/
|
158
|
+
private_constant :INPUT_BLOCK_RE
|
159
|
+
|
160
|
+
private
|
161
|
+
|
162
|
+
def config_source(config_string)
|
163
|
+
org.logstash.common.SourceWithMetadata.new("string", "config_string", config_string)
|
164
|
+
end
|
165
|
+
|
166
|
+
def test_sink_output_source(**config)
|
167
|
+
config = { id: current_spec_id }.merge(config).map { |k, v| "#{k} => #{v.is_a?(String) ? v.inspect : v}" }.join(' ')
|
168
|
+
output_string = "output { test_sink { #{config} } }" # TODO opts for performance store_events: false
|
169
|
+
org.logstash.common.SourceWithMetadata.new("string", "test_sink_output", output_string)
|
170
|
+
end
|
171
|
+
|
172
|
+
def current_spec_id
|
173
|
+
@__current_example_metadata&.[](:location) || 'spec-sample'
|
174
|
+
end
|
175
|
+
|
176
|
+
if RUBY_VERSION > '2.5'
|
177
|
+
def deprecated(msg)
|
178
|
+
Kernel.warn(msg, uplevel: 1)
|
179
|
+
end
|
180
|
+
else # due JRuby 9.1 (Ruby 2.3)
|
181
|
+
def deprecated(msg)
|
182
|
+
loc = caller_locations[1]
|
183
|
+
Kernel.warn("#{loc.path}:#{loc.lineno}: warning: #{msg}")
|
135
184
|
end
|
136
185
|
end
|
186
|
+
|
137
187
|
end # module LogStash
|
138
188
|
|
@@ -14,10 +14,7 @@ end
|
|
14
14
|
|
15
15
|
require "logstash-core"
|
16
16
|
require "logstash/logging"
|
17
|
-
require "logstash/environment"
|
18
17
|
require "logstash/devutils/rspec/logstash_helpers"
|
19
|
-
require "logstash/devutils/rspec/shared_examples"
|
20
|
-
require "insist"
|
21
18
|
|
22
19
|
Thread.abort_on_exception = true
|
23
20
|
|
@@ -28,35 +25,32 @@ unless java.lang.System.getProperty("log4j.configurationFile")
|
|
28
25
|
end
|
29
26
|
|
30
27
|
$TESTING = true
|
31
|
-
if RUBY_VERSION < "
|
32
|
-
|
33
|
-
raise LoadError
|
28
|
+
if RUBY_VERSION < "2.3"
|
29
|
+
raise LoadError.new("Ruby >= 2.3.0 or later is required. (You are running: " + RUBY_VERSION + ")")
|
34
30
|
end
|
35
31
|
|
36
|
-
if ENV["TEST_DEBUG"]
|
37
|
-
|
32
|
+
if level = (ENV['LOG_LEVEL'] || ENV['LOGGER_LEVEL'] || ENV["TEST_DEBUG"])
|
33
|
+
logger, level = level.split('=') # 'logstash.filters.grok=DEBUG'
|
34
|
+
level, logger = logger, nil if level.nil? # only level given e.g. 'DEBUG'
|
35
|
+
level = org.apache.logging.log4j.Level.toLevel(level, org.apache.logging.log4j.Level::WARN)
|
36
|
+
LogStash::Logging::Logger::configure_logging(level.to_s, logger)
|
38
37
|
else
|
39
|
-
LogStash::Logging::Logger::configure_logging(
|
38
|
+
LogStash::Logging::Logger::configure_logging('ERROR')
|
40
39
|
end
|
41
40
|
|
42
|
-
# removed the strictness check, it did not seem to catch anything
|
43
|
-
|
44
41
|
RSpec.configure do |config|
|
45
42
|
# for now both include and extend are required because the newly refactored "input" helper method need to be visible in a "it" block
|
46
43
|
# and this is only possible by calling include on LogStashHelper
|
47
44
|
config.include LogStashHelper
|
48
45
|
config.extend LogStashHelper
|
49
46
|
|
50
|
-
|
47
|
+
config.filter_run_excluding LogStashHelper.excluded_tags
|
51
48
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
exclude_tags[:windows] = true
|
49
|
+
config.around(:each) do |example|
|
50
|
+
@__current_example_metadata = example.metadata
|
51
|
+
example.run
|
56
52
|
end
|
57
53
|
|
58
|
-
config.filter_run_excluding exclude_tags
|
59
|
-
|
60
54
|
# Run specs in random order to surface order dependencies. If you find an
|
61
55
|
# order dependency and want to debug it, you can fix the order by providing
|
62
56
|
# the seed, which is printed after each run.
|
@@ -0,0 +1,108 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/namespace"
|
3
|
+
require "logstash/outputs/base"
|
4
|
+
require "logstash/errors"
|
5
|
+
|
6
|
+
# This output simply discards (but tracks) received events.
|
7
|
+
class LogStash::Outputs::TestSink < LogStash::Outputs::Base
|
8
|
+
|
9
|
+
concurrency :shared
|
10
|
+
|
11
|
+
config_name "test_sink"
|
12
|
+
|
13
|
+
# Whether we're tracking events received or simply act as a true sink.
|
14
|
+
config :store_events, :validate => :boolean, :default => true
|
15
|
+
# Plugin could not release itself (on close) if needed to keep its events around.
|
16
|
+
config :release_on_close, :validate => :boolean, :default => true
|
17
|
+
# Even poll timeout in milliseconds (for input plugin tests consuming events).
|
18
|
+
# HINT: set to 0 (null) to wait forever.
|
19
|
+
config :event_poll_timeout, :default => 5 * 1000
|
20
|
+
|
21
|
+
TRACKER = java.util.WeakHashMap.new
|
22
|
+
|
23
|
+
# @override plugin hook
|
24
|
+
def register
|
25
|
+
TRACKER[self] = @_event_store || init_event_store
|
26
|
+
end
|
27
|
+
|
28
|
+
# @override plugin impl
|
29
|
+
def receive(event)
|
30
|
+
event_store << event if store_events?
|
31
|
+
end
|
32
|
+
|
33
|
+
# @override plugin hook
|
34
|
+
def close
|
35
|
+
TRACKER.delete(self)
|
36
|
+
@_event_store = false if release_on_close?
|
37
|
+
end
|
38
|
+
|
39
|
+
def store_events?
|
40
|
+
!!@store_events
|
41
|
+
end
|
42
|
+
|
43
|
+
def release_on_close?
|
44
|
+
!!@release_on_close
|
45
|
+
end
|
46
|
+
|
47
|
+
# Clears the event store.
|
48
|
+
def clear!
|
49
|
+
event_store.clear
|
50
|
+
end
|
51
|
+
|
52
|
+
# @return [Queue] (enumerable) event store
|
53
|
+
def event_store
|
54
|
+
if @_event_store.nil?
|
55
|
+
warn("#{self} event store not initialized (call plugin.register to initialize)", caller_locations(2))
|
56
|
+
return init_event_store
|
57
|
+
elsif @_event_store.eql?(false)
|
58
|
+
warn("#{self} closed - event store no longer available (release_on_close => false if you need to retain events)", caller_locations(2))
|
59
|
+
return nil
|
60
|
+
end
|
61
|
+
@_event_store
|
62
|
+
end
|
63
|
+
|
64
|
+
def init_event_store
|
65
|
+
@_event_store = Queue.new(@event_poll_timeout.to_f * 1000)
|
66
|
+
end
|
67
|
+
|
68
|
+
private
|
69
|
+
|
70
|
+
def warn(msg, callstack = nil)
|
71
|
+
Kernel.warn("#{msg}#{callstack ? " called from #{callstack.first}" : nil}")
|
72
|
+
end
|
73
|
+
|
74
|
+
# TODO refactor to java.util.concurrent.ConcurrentLinkedQueue
|
75
|
+
# Interestingly, using a ConcurrentLinkedQueue gets specs that depend on pop-ing events
|
76
|
+
# from the output (e.g. syslog input plugin) passing fine with the old Ruby pipeline.
|
77
|
+
# The Java pipeline seems to reach a shutdown before the input yielded events are to be consumed.
|
78
|
+
|
79
|
+
class Queue < java.util.concurrent.LinkedBlockingQueue
|
80
|
+
|
81
|
+
java_import java.util.concurrent.TimeoutException
|
82
|
+
|
83
|
+
MILLISECONDS = java.util.concurrent.TimeUnit::MILLISECONDS
|
84
|
+
private_constant :MILLISECONDS
|
85
|
+
|
86
|
+
def initialize(poll_timeout = nil)
|
87
|
+
super()
|
88
|
+
@timeout = poll_timeout.to_i
|
89
|
+
end
|
90
|
+
|
91
|
+
# Ruby Queue like pop-er with (default) blocking.
|
92
|
+
# @see Queue#pop
|
93
|
+
def pop(non_block = nil)
|
94
|
+
# for compatibility we're making it behave like Ruby's Queue
|
95
|
+
return poll if non_block
|
96
|
+
@timeout.zero? ? take :
|
97
|
+
(poll(@timeout, MILLISECONDS) || timeout!(__method__))
|
98
|
+
end
|
99
|
+
alias deq pop
|
100
|
+
alias shift pop
|
101
|
+
|
102
|
+
private
|
103
|
+
|
104
|
+
def timeout!(op)
|
105
|
+
raise TimeoutException.new("#{op.inspect} did not complete (in #{@timeout}ms)")
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
@@ -0,0 +1,173 @@
|
|
1
|
+
require "logstash/pipeline"
|
2
|
+
require "logstash/java_pipeline"
|
3
|
+
|
4
|
+
module LogStash
|
5
|
+
class TestPipeline < LogStash::JavaPipeline
|
6
|
+
public :flush_filters
|
7
|
+
|
8
|
+
def run_with(events)
|
9
|
+
if inputs&.any? # will work but might be unintended
|
10
|
+
config = "\n #{config_str}" if $VERBOSE
|
11
|
+
warn "#{self} pipeline is getting events pushed manually while having inputs: #{inputs.inspect} #{config}"
|
12
|
+
end
|
13
|
+
# TODO could we handle a generator (Enumerator) ?
|
14
|
+
queue.write_client.push_batch events.to_a
|
15
|
+
queue_read_client = filter_queue_client
|
16
|
+
# potentially handle run_with called multiple times (re-use the same instance) :
|
17
|
+
if queue_read_client.is_a?(EventTrackingQueueReadClientDelegator)
|
18
|
+
queue_read_client.reset_events!
|
19
|
+
else
|
20
|
+
# start_worker using @filter_queue_client on 6.3, since 6.4 a reader method is used
|
21
|
+
# to make things compatible with 6.3 we're assigning the @filter_queue_client ivar
|
22
|
+
@filter_queue_client = EventTrackingQueueReadClientDelegator.new queue_read_client
|
23
|
+
end
|
24
|
+
run
|
25
|
+
end
|
26
|
+
|
27
|
+
# @override for WorkerLoop to pick it up
|
28
|
+
# @note only works since LS 6.4 (need to use tha actual ivar for 6.3)
|
29
|
+
def filter_queue_client
|
30
|
+
@filter_queue_client || super
|
31
|
+
end
|
32
|
+
|
33
|
+
java_import org.apache.logging.log4j.ThreadContext unless const_defined?(:ThreadContext)
|
34
|
+
|
35
|
+
def start_and_wait
|
36
|
+
parent_thread = Thread.current
|
37
|
+
@finished_execution.make_false
|
38
|
+
@finished_run&.make_false # only since 6.5
|
39
|
+
|
40
|
+
@thread = Thread.new do
|
41
|
+
begin
|
42
|
+
LogStash::Util.set_thread_name("pipeline.#{pipeline_id}")
|
43
|
+
ThreadContext.put("pipeline.id", pipeline_id)
|
44
|
+
run
|
45
|
+
@finished_run&.make_true
|
46
|
+
rescue => e
|
47
|
+
close
|
48
|
+
parent_thread.raise(e)
|
49
|
+
ensure
|
50
|
+
@finished_execution.make_true
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
unless wait_until_started
|
55
|
+
raise "failed to start pipeline: #{self}\n with config: #{config_str.inspect}"
|
56
|
+
end
|
57
|
+
|
58
|
+
@thread
|
59
|
+
end
|
60
|
+
|
61
|
+
class EventTrackingQueueReadClientDelegator
|
62
|
+
include org.logstash.execution.QueueReadClient
|
63
|
+
java_import org.logstash.execution.QueueReadClient
|
64
|
+
|
65
|
+
attr_reader :processed_events
|
66
|
+
|
67
|
+
def initialize(delegate)
|
68
|
+
# NOTE: can not use LogStash::MemoryReadClient#read_batch due its JavaObject wrapping
|
69
|
+
@delegate = delegate.to_java(QueueReadClient)
|
70
|
+
@processed_events = []
|
71
|
+
end
|
72
|
+
|
73
|
+
# @override QueueBatch readBatch() throws InterruptedException;
|
74
|
+
def readBatch
|
75
|
+
QueueBatchDelegator.new(self, @delegate.read_batch)
|
76
|
+
end
|
77
|
+
|
78
|
+
# @override void closeBatch(QueueBatch batch) throws IOException;
|
79
|
+
def closeBatch(batch)
|
80
|
+
@delegate.close_batch(batch)
|
81
|
+
end
|
82
|
+
|
83
|
+
# @override boolean isEmpty();
|
84
|
+
def isEmpty
|
85
|
+
@delegate.empty?
|
86
|
+
end
|
87
|
+
|
88
|
+
# @override QueueBatch newBatch();
|
89
|
+
def newBatch
|
90
|
+
@delegate.new_batch
|
91
|
+
end
|
92
|
+
|
93
|
+
# @override void startMetrics(QueueBatch batch);
|
94
|
+
def startMetrics(batch)
|
95
|
+
@delegate.start_metrics(batch)
|
96
|
+
end
|
97
|
+
|
98
|
+
# @override void addOutputMetrics(int filteredSize);
|
99
|
+
def addOutputMetrics(filteredSize)
|
100
|
+
@delegate.add_output_metrics(filteredSize)
|
101
|
+
end
|
102
|
+
|
103
|
+
# @override void addFilteredMetrics(int filteredSize);
|
104
|
+
def addFilteredMetrics(filteredSize)
|
105
|
+
@delegate.add_filtered_metrics(filteredSize)
|
106
|
+
end
|
107
|
+
|
108
|
+
# @override
|
109
|
+
def set_batch_dimensions(batch_size, batch_delay)
|
110
|
+
@delegate.set_batch_dimensions(batch_size, batch_delay)
|
111
|
+
end
|
112
|
+
|
113
|
+
# @override
|
114
|
+
def close
|
115
|
+
@delegate.close
|
116
|
+
end
|
117
|
+
|
118
|
+
# @override dispatch to delegate
|
119
|
+
def method_missing(method, *args)
|
120
|
+
@delegate.public_send(method, *args)
|
121
|
+
end
|
122
|
+
|
123
|
+
def filtered_events(events)
|
124
|
+
@processed_events.concat(events)
|
125
|
+
end
|
126
|
+
|
127
|
+
def reset_events!
|
128
|
+
@processed_events = []
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
132
|
+
|
133
|
+
class QueueBatchDelegator
|
134
|
+
include org.logstash.execution.QueueBatch
|
135
|
+
|
136
|
+
def initialize(event_tracker, delegate)
|
137
|
+
@event_tracker = event_tracker
|
138
|
+
@delegate = delegate
|
139
|
+
end
|
140
|
+
|
141
|
+
# @override RubyArray to_a();
|
142
|
+
def to_a
|
143
|
+
@delegate.to_a.tap do |events|
|
144
|
+
# filters out rogue (cancelled) events
|
145
|
+
@event_tracker.filtered_events events
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
# @override int filteredSize();
|
150
|
+
def filteredSize
|
151
|
+
@delegate.to_java.filtered_size
|
152
|
+
end
|
153
|
+
|
154
|
+
def events
|
155
|
+
@delegate.events.tap do |events|
|
156
|
+
# filters out rogue (cancelled) events
|
157
|
+
@event_tracker.filtered_events events
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
# @override void merge(IRubyObject event);
|
162
|
+
def merge(event)
|
163
|
+
@delegate.merge(event)
|
164
|
+
end
|
165
|
+
|
166
|
+
# @override void close() throws IOException;
|
167
|
+
def close
|
168
|
+
@delegate.close
|
169
|
+
end
|
170
|
+
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
data/logstash-devutils.gemspec
CHANGED
@@ -1,28 +1,30 @@
|
|
1
|
-
if RUBY_PLATFORM != "java"
|
2
|
-
raise "Only JRuby is supported"
|
3
|
-
end
|
1
|
+
raise "Only JRuby is supported" if RUBY_PLATFORM != "java"
|
4
2
|
|
5
3
|
Gem::Specification.new do |spec|
|
6
4
|
files = %x{git ls-files}.split("\n")
|
7
5
|
|
8
6
|
spec.name = "logstash-devutils"
|
9
|
-
spec.version = "
|
10
|
-
spec.
|
11
|
-
spec.summary = "logstash-devutils"
|
12
|
-
spec.description = "logstash-devutils"
|
7
|
+
spec.version = "2.0.4"
|
8
|
+
spec.license = "Apache-2.0"
|
13
9
|
spec.authors = ["Elastic"]
|
14
10
|
spec.email = "info@elastic.co"
|
15
11
|
spec.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
16
12
|
|
13
|
+
spec.summary = %q{An assortment of tooling/libraries to make Logstash plugin development and releasing a bit easier.}
|
14
|
+
spec.description = %q{logstash-devutils gem is meant to be used as a development dependency from other plugins/gems.}
|
15
|
+
|
17
16
|
spec.files = files
|
18
17
|
spec.require_paths << "lib"
|
19
18
|
|
20
19
|
spec.platform = "java"
|
21
20
|
|
22
|
-
|
23
|
-
|
24
|
-
#
|
25
|
-
#
|
21
|
+
spec.required_ruby_version = '>= 2.3'
|
22
|
+
|
23
|
+
# Please note that devutils is meant to be used as a development dependency from other plugins/gems.
|
24
|
+
# As such, devutils' OWN development dependencies (any add_development_dependency) will be ignored when
|
25
|
+
# bundling a plugin/gem which uses/depends on devutils.
|
26
|
+
#
|
27
|
+
# This is why all devutils own dependencies should normally be specified as add_runtime_dependency.
|
26
28
|
|
27
29
|
# It is important to specify rspec "~> 3.0" and not "~> 3.0.0" (or any other version to the patch level)
|
28
30
|
# otherwise the version constrain will have to be met up to the minor release version and only allow
|
@@ -35,13 +37,13 @@ Gem::Specification.new do |spec|
|
|
35
37
|
spec.add_runtime_dependency "rake" # MIT License
|
36
38
|
spec.add_runtime_dependency "gem_publisher" # MIT License
|
37
39
|
spec.add_runtime_dependency "minitar" # GPL2|Ruby License
|
38
|
-
spec.add_runtime_dependency "logstash-core-plugin-api", "<= 2.99", ">= 2.0"
|
39
40
|
|
40
|
-
|
41
|
-
|
42
|
-
#
|
43
|
-
spec.add_runtime_dependency "insist"
|
41
|
+
spec.add_runtime_dependency "logstash-core", ">= 6.3"
|
42
|
+
|
43
|
+
# Some plugins are (still) using insist by their own, but we no longer force this dependency on others.
|
44
|
+
#spec.add_runtime_dependency "insist" # (Apache 2.0 license)
|
44
45
|
spec.add_runtime_dependency "kramdown", '1.14.0'
|
45
46
|
spec.add_runtime_dependency "stud", " >= 0.0.20"
|
46
47
|
spec.add_runtime_dependency "fivemat"
|
48
|
+
spec.add_runtime_dependency "logstash-codec-plain"
|
47
49
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-devutils
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.4
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-11-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -83,72 +83,66 @@ dependencies:
|
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
requirement: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
|
-
- - "<="
|
87
|
-
- !ruby/object:Gem::Version
|
88
|
-
version: '2.99'
|
89
86
|
- - ">="
|
90
87
|
- !ruby/object:Gem::Version
|
91
|
-
version: '
|
92
|
-
name: logstash-core
|
88
|
+
version: '6.3'
|
89
|
+
name: logstash-core
|
93
90
|
prerelease: false
|
94
91
|
type: :runtime
|
95
92
|
version_requirements: !ruby/object:Gem::Requirement
|
96
93
|
requirements:
|
97
|
-
- - "<="
|
98
|
-
- !ruby/object:Gem::Version
|
99
|
-
version: '2.99'
|
100
94
|
- - ">="
|
101
95
|
- !ruby/object:Gem::Version
|
102
|
-
version: '
|
96
|
+
version: '6.3'
|
103
97
|
- !ruby/object:Gem::Dependency
|
104
98
|
requirement: !ruby/object:Gem::Requirement
|
105
99
|
requirements:
|
106
100
|
- - '='
|
107
101
|
- !ruby/object:Gem::Version
|
108
|
-
version: 1.
|
109
|
-
name:
|
102
|
+
version: 1.14.0
|
103
|
+
name: kramdown
|
110
104
|
prerelease: false
|
111
105
|
type: :runtime
|
112
106
|
version_requirements: !ruby/object:Gem::Requirement
|
113
107
|
requirements:
|
114
108
|
- - '='
|
115
109
|
- !ruby/object:Gem::Version
|
116
|
-
version: 1.
|
110
|
+
version: 1.14.0
|
117
111
|
- !ruby/object:Gem::Dependency
|
118
112
|
requirement: !ruby/object:Gem::Requirement
|
119
113
|
requirements:
|
120
|
-
- -
|
114
|
+
- - ">="
|
121
115
|
- !ruby/object:Gem::Version
|
122
|
-
version:
|
123
|
-
name:
|
116
|
+
version: 0.0.20
|
117
|
+
name: stud
|
124
118
|
prerelease: false
|
125
119
|
type: :runtime
|
126
120
|
version_requirements: !ruby/object:Gem::Requirement
|
127
121
|
requirements:
|
128
|
-
- -
|
122
|
+
- - ">="
|
129
123
|
- !ruby/object:Gem::Version
|
130
|
-
version:
|
124
|
+
version: 0.0.20
|
131
125
|
- !ruby/object:Gem::Dependency
|
132
126
|
requirement: !ruby/object:Gem::Requirement
|
133
127
|
requirements:
|
134
128
|
- - ">="
|
135
129
|
- !ruby/object:Gem::Version
|
136
|
-
version: 0
|
137
|
-
name:
|
130
|
+
version: '0'
|
131
|
+
name: fivemat
|
138
132
|
prerelease: false
|
139
133
|
type: :runtime
|
140
134
|
version_requirements: !ruby/object:Gem::Requirement
|
141
135
|
requirements:
|
142
136
|
- - ">="
|
143
137
|
- !ruby/object:Gem::Version
|
144
|
-
version: 0
|
138
|
+
version: '0'
|
145
139
|
- !ruby/object:Gem::Dependency
|
146
140
|
requirement: !ruby/object:Gem::Requirement
|
147
141
|
requirements:
|
148
142
|
- - ">="
|
149
143
|
- !ruby/object:Gem::Version
|
150
144
|
version: '0'
|
151
|
-
name:
|
145
|
+
name: logstash-codec-plain
|
152
146
|
prerelease: false
|
153
147
|
type: :runtime
|
154
148
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -156,7 +150,8 @@ dependencies:
|
|
156
150
|
- - ">="
|
157
151
|
- !ruby/object:Gem::Version
|
158
152
|
version: '0'
|
159
|
-
description: logstash-devutils
|
153
|
+
description: logstash-devutils gem is meant to be used as a development dependency
|
154
|
+
from other plugins/gems.
|
160
155
|
email: info@elastic.co
|
161
156
|
executables: []
|
162
157
|
extensions: []
|
@@ -165,6 +160,7 @@ files:
|
|
165
160
|
- CHANGELOG.md
|
166
161
|
- Gemfile
|
167
162
|
- LICENSE
|
163
|
+
- Rakefile
|
168
164
|
- lib/logstash/devutils/rake.rb
|
169
165
|
- lib/logstash/devutils/rake/publish.rake
|
170
166
|
- lib/logstash/devutils/rake/vendor.rake
|
@@ -172,10 +168,12 @@ files:
|
|
172
168
|
- lib/logstash/devutils/rspec/logstash_helpers.rb
|
173
169
|
- lib/logstash/devutils/rspec/shared_examples.rb
|
174
170
|
- lib/logstash/devutils/rspec/spec_helper.rb
|
171
|
+
- lib/logstash/outputs/test_sink.rb
|
172
|
+
- lib/logstash/test_pipeline.rb
|
175
173
|
- logstash-devutils.gemspec
|
176
174
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
177
175
|
licenses:
|
178
|
-
- Apache
|
176
|
+
- Apache-2.0
|
179
177
|
metadata: {}
|
180
178
|
post_install_message:
|
181
179
|
rdoc_options: []
|
@@ -186,7 +184,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
186
184
|
requirements:
|
187
185
|
- - ">="
|
188
186
|
- !ruby/object:Gem::Version
|
189
|
-
version: '
|
187
|
+
version: '2.3'
|
190
188
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
191
189
|
requirements:
|
192
190
|
- - ">="
|
@@ -197,5 +195,6 @@ rubyforge_project:
|
|
197
195
|
rubygems_version: 2.6.13
|
198
196
|
signing_key:
|
199
197
|
specification_version: 4
|
200
|
-
summary:
|
198
|
+
summary: An assortment of tooling/libraries to make Logstash plugin development and
|
199
|
+
releasing a bit easier.
|
201
200
|
test_files: []
|