logstash-core 5.0.0.pre.rc1-java → 5.0.1-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/jars.rb +1 -1
- data/lib/logstash-core/version.rb +1 -1
- data/lib/logstash/config/loader.rb +14 -3
- data/lib/logstash/environment.rb +3 -3
- data/lib/logstash/filters/base.rb +9 -3
- data/lib/logstash/instrument/periodic_poller/base.rb +22 -8
- data/lib/logstash/logging/logger.rb +12 -1
- data/lib/logstash/pipeline.rb +4 -11
- data/lib/logstash/runner.rb +18 -10
- data/lib/logstash/settings.rb +34 -9
- data/lib/logstash/util/decorators.rb +10 -7
- data/lib/logstash/util/wrapped_synchronous_queue.rb +15 -5
- data/lib/logstash/version.rb +1 -1
- data/locales/en.yml +4 -5
- data/logstash-core.gemspec +1 -1
- data/spec/logstash/filters/base_spec.rb +45 -0
- data/spec/logstash/instrument/periodic_poller/base_spec.rb +32 -0
- data/spec/logstash/pipeline_spec.rb +54 -0
- data/spec/logstash/runner_spec.rb +2 -2
- data/spec/logstash/setting_spec.rb +23 -1
- data/spec/logstash/settings/string_spec.rb +1 -1
- data/spec/logstash/settings_spec.rb +27 -0
- data/spec/logstash/util/wrapped_synchronous_queue_spec.rb +8 -7
- data/spec/support/mocks_classes.rb +22 -0
- data/vendor/jars/org/logstash/logstash-core/{5.0.0-rc1/logstash-core-5.0.0-rc1.jar → 5.0.1/logstash-core-5.0.1.jar} +0 -0
- metadata +9 -8
- data/lib/logstash/instrument/periodic_poller/periodic_poller_observer.rb +0 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1d2dff17e7881a7df042ac7b79891f9519331757
|
4
|
+
data.tar.gz: f02d9b9301c6e6b8973b211b2e5f3c8800ed0fd3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f012536aacad34585ae14a4810fc6924e6755f239bc86fdb83a72b258c4379d8f4ca386aba22adbc8805ed31c396c28dbc5a625737aa648fd6d4a8f2f3eb3780
|
7
|
+
data.tar.gz: 2a04623696dec22955b2b44fac4a75bf56525ed0d6247ce95cb751d026d916d55c1c0728825e4baedd9b6a280f882fea784eee34d999af67c820aa5771f036ad
|
data/lib/jars.rb
CHANGED
@@ -4,4 +4,4 @@ require_jar('org.apache.logging.log4j', 'log4j-api', '2.6.2')
|
|
4
4
|
require_jar('org.apache.logging.log4j', 'log4j-core', '2.6.2')
|
5
5
|
require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.7.4')
|
6
6
|
require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.7.4')
|
7
|
-
require_jar('org.logstash', 'logstash-core', '5.0.
|
7
|
+
require_jar('org.logstash', 'logstash-core', '5.0.1')
|
@@ -12,7 +12,18 @@ module LogStash; module Config; class Loader
|
|
12
12
|
# Append the config string.
|
13
13
|
# This allows users to provide both -f and -e flags. The combination
|
14
14
|
# is rare, but useful for debugging.
|
15
|
-
|
15
|
+
loaded_config = load_config(config_path)
|
16
|
+
if loaded_config.empty? && config_string.empty?
|
17
|
+
# If loaded config from `-f` is empty *and* if config string is empty we raise an error
|
18
|
+
fail(I18n.t("logstash.runner.configuration.file-not-found", :path => config_path))
|
19
|
+
end
|
20
|
+
|
21
|
+
# tell the user we are merging, otherwise it is very confusing
|
22
|
+
if !loaded_config.empty? && !config_string.empty?
|
23
|
+
@logger.info("Created final config by merging config string and config path", :path => config_path)
|
24
|
+
end
|
25
|
+
|
26
|
+
config_string = config_string + loaded_config
|
16
27
|
else
|
17
28
|
# include a default stdin input if no inputs given
|
18
29
|
if config_string !~ /input *{/
|
@@ -52,11 +63,11 @@ module LogStash; module Config; class Loader
|
|
52
63
|
path = ::File.expand_path(path)
|
53
64
|
path = ::File.join(path, "*") if ::File.directory?(path)
|
54
65
|
|
66
|
+
config = ""
|
55
67
|
if Dir.glob(path).length == 0
|
56
|
-
|
68
|
+
@logger.info("No config files found in path", :path => path)
|
57
69
|
end
|
58
70
|
|
59
|
-
config = ""
|
60
71
|
encoding_issue_files = []
|
61
72
|
Dir.glob(path).sort.each do |file|
|
62
73
|
next unless ::File.file?(file)
|
data/lib/logstash/environment.rb
CHANGED
@@ -17,9 +17,9 @@ module LogStash
|
|
17
17
|
|
18
18
|
[
|
19
19
|
Setting::String.new("node.name", Socket.gethostname),
|
20
|
-
|
20
|
+
Setting::NullableString.new("path.config", nil, false),
|
21
21
|
Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")),
|
22
|
-
|
22
|
+
Setting::NullableString.new("config.string", nil, false),
|
23
23
|
Setting::Boolean.new("config.test_and_exit", false),
|
24
24
|
Setting::Boolean.new("config.reload.automatic", false),
|
25
25
|
Setting::Numeric.new("config.reload.interval", 3), # in seconds
|
@@ -31,7 +31,7 @@ module LogStash
|
|
31
31
|
Setting::Numeric.new("pipeline.batch.delay", 5), # in milliseconds
|
32
32
|
Setting::Boolean.new("pipeline.unsafe_shutdown", false),
|
33
33
|
Setting.new("path.plugins", Array, []),
|
34
|
-
|
34
|
+
Setting::NullableString.new("interactive", nil, false),
|
35
35
|
Setting::Boolean.new("config.debug", false),
|
36
36
|
Setting::String.new("log.level", "info", true, ["fatal", "error", "warn", "debug", "info", "trace"]),
|
37
37
|
Setting::Boolean.new("version", false),
|
@@ -187,14 +187,20 @@ class LogStash::Filters::Base < LogStash::Plugin
|
|
187
187
|
# note below that the tags array field needs to be updated then reassigned to the event.
|
188
188
|
# this is important because a construct like event["tags"].delete(tag) will not work
|
189
189
|
# in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
|
190
|
+
|
191
|
+
tags = event.get("tags")
|
192
|
+
return unless tags
|
193
|
+
|
194
|
+
tags = Array(tags)
|
190
195
|
@remove_tag.each do |tag|
|
191
|
-
|
192
|
-
|
196
|
+
break if tags.empty?
|
197
|
+
|
193
198
|
tag = event.sprintf(tag)
|
194
199
|
@logger.debug? and @logger.debug("filters/#{self.class.name}: removing tag", :tag => tag)
|
195
200
|
tags.delete(tag)
|
196
|
-
event.set("tags", tags)
|
197
201
|
end
|
202
|
+
|
203
|
+
event.set("tags", tags)
|
198
204
|
end # def filter_matched
|
199
205
|
|
200
206
|
protected
|
@@ -8,8 +8,8 @@ module LogStash module Instrument module PeriodicPoller
|
|
8
8
|
include LogStash::Util::Loggable
|
9
9
|
|
10
10
|
DEFAULT_OPTIONS = {
|
11
|
-
:polling_interval =>
|
12
|
-
:polling_timeout =>
|
11
|
+
:polling_interval => 5,
|
12
|
+
:polling_timeout => 120
|
13
13
|
}
|
14
14
|
|
15
15
|
public
|
@@ -22,11 +22,25 @@ module LogStash module Instrument module PeriodicPoller
|
|
22
22
|
def update(time, result, exception)
|
23
23
|
return unless exception
|
24
24
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
25
|
+
if exception.is_a?(Concurrent::TimeoutError)
|
26
|
+
# On a busy system this can happen, we just log it as a debug
|
27
|
+
# event instead of an error, Some of the JVM calls can take a long time or block.
|
28
|
+
logger.debug("PeriodicPoller: Timeout exception",
|
29
|
+
:poller => self,
|
30
|
+
:result => result,
|
31
|
+
:polling_timeout => @options[:polling_timeout],
|
32
|
+
:polling_interval => @options[:polling_interval],
|
33
|
+
:exception => exception.class,
|
34
|
+
:executed_at => time)
|
35
|
+
else
|
36
|
+
logger.error("PeriodicPoller: exception",
|
37
|
+
:poller => self,
|
38
|
+
:result => result,
|
39
|
+
:exception => exception.class,
|
40
|
+
:polling_timeout => @options[:polling_timeout],
|
41
|
+
:polling_interval => @options[:polling_interval],
|
42
|
+
:executed_at => time)
|
43
|
+
end
|
30
44
|
end
|
31
45
|
|
32
46
|
def collect
|
@@ -37,7 +51,7 @@ module LogStash module Instrument module PeriodicPoller
|
|
37
51
|
logger.debug("PeriodicPoller: Starting",
|
38
52
|
:polling_interval => @options[:polling_interval],
|
39
53
|
:polling_timeout => @options[:polling_timeout]) if logger.debug?
|
40
|
-
|
54
|
+
|
41
55
|
collect # Collect data right away if possible
|
42
56
|
@task.execute
|
43
57
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require "logstash/java_integration"
|
2
|
+
require "uri"
|
2
3
|
|
3
4
|
module LogStash
|
4
5
|
module Logging
|
@@ -6,6 +7,7 @@ module LogStash
|
|
6
7
|
java_import org.apache.logging.log4j.Level
|
7
8
|
java_import org.apache.logging.log4j.LogManager
|
8
9
|
java_import org.apache.logging.log4j.core.config.Configurator
|
10
|
+
java_import org.apache.logging.log4j.core.config.DefaultConfiguration
|
9
11
|
@@config_mutex = Mutex.new
|
10
12
|
@@logging_context = nil
|
11
13
|
|
@@ -70,7 +72,16 @@ module LogStash
|
|
70
72
|
def self.initialize(config_location)
|
71
73
|
@@config_mutex.synchronize do
|
72
74
|
if @@logging_context.nil?
|
73
|
-
|
75
|
+
file_path = URI(config_location).path
|
76
|
+
if ::File.exists?(file_path)
|
77
|
+
logs_location = java.lang.System.getProperty("ls.logs")
|
78
|
+
puts "Sending Logstash's logs to #{logs_location} which is now configured via log4j2.properties"
|
79
|
+
@@logging_context = Configurator.initialize(nil, config_location)
|
80
|
+
else
|
81
|
+
# fall back to default config
|
82
|
+
puts "Could not find log4j2 configuration at path #{file_path}. Using default config which logs to console"
|
83
|
+
@@logging_context = Configurator.initialize(DefaultConfiguration.new)
|
84
|
+
end
|
74
85
|
end
|
75
86
|
end
|
76
87
|
end
|
data/lib/logstash/pipeline.rb
CHANGED
@@ -257,14 +257,9 @@ module LogStash; class Pipeline
|
|
257
257
|
def filter_batch(batch)
|
258
258
|
batch.each do |event|
|
259
259
|
if event.is_a?(Event)
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
if e.cancelled?
|
264
|
-
batch.cancel(e)
|
265
|
-
else
|
266
|
-
batch.merge(e)
|
267
|
-
end
|
260
|
+
filter_func(event).each do |e|
|
261
|
+
# these are both original and generated events
|
262
|
+
batch.merge(e) unless e.cancelled?
|
268
263
|
end
|
269
264
|
end
|
270
265
|
end
|
@@ -492,9 +487,7 @@ module LogStash; class Pipeline
|
|
492
487
|
def flush_filters_to_batch(batch, options = {})
|
493
488
|
options[:final] = batch.shutdown_signal_received?
|
494
489
|
flush_filters(options) do |event|
|
495
|
-
|
496
|
-
batch.cancel(event)
|
497
|
-
else
|
490
|
+
unless event.cancelled?
|
498
491
|
@logger.debug? and @logger.debug("Pushing flushed events", :event => event)
|
499
492
|
batch.merge(event)
|
500
493
|
end
|
data/lib/logstash/runner.rb
CHANGED
@@ -162,10 +162,11 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
162
162
|
|
163
163
|
begin
|
164
164
|
LogStash::SETTINGS.from_yaml(LogStash::SETTINGS.get("path.settings"))
|
165
|
+
rescue Errno::ENOENT
|
166
|
+
$stderr.puts "WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults"
|
165
167
|
rescue => e
|
166
168
|
# abort unless we're just looking for the help
|
167
|
-
|
168
|
-
$stderr.puts "INFO: Logstash requires a setting file which is typically located in $LS_HOME/config or /etc/logstash. If you installed Logstash through a package and are starting it manually please specify the location to this settings file by passing in \"--path.settings=/path/..\""
|
169
|
+
unless cli_help?(args)
|
169
170
|
$stderr.puts "ERROR: Failed to load settings file from \"path.settings\". Aborting... path.setting=#{LogStash::SETTINGS.get("path.settings")}, exception=#{e.class}, message=>#{e.message}"
|
170
171
|
return 1
|
171
172
|
end
|
@@ -186,16 +187,10 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
186
187
|
java.lang.System.setProperty("ls.log.level", setting("log.level"))
|
187
188
|
unless java.lang.System.getProperty("log4j.configurationFile")
|
188
189
|
log4j_config_location = ::File.join(setting("path.settings"), "log4j2.properties")
|
189
|
-
LogStash::Logging::Logger::initialize(log4j_config_location)
|
190
|
+
LogStash::Logging::Logger::initialize("file://" + log4j_config_location)
|
190
191
|
end
|
191
192
|
# override log level that may have been introduced from a custom log4j config file
|
192
193
|
LogStash::Logging::Logger::configure_logging(setting("log.level"))
|
193
|
-
|
194
|
-
# Adding this here because a ton of LS users install LS via packages and try to manually
|
195
|
-
# start Logstash using bin/logstash. See #5986. I think not logging to console is good for
|
196
|
-
# services, but until LS users re-learn that logs end up in path.logs, we should keep this
|
197
|
-
# message. Otherwise we'll be answering the same question again and again.
|
198
|
-
puts "Sending Logstash logs to #{setting("path.logs")} which is now configured via log4j2.properties."
|
199
194
|
|
200
195
|
if setting("config.debug") && logger.debug?
|
201
196
|
logger.warn("--config.debug was specified, but log.level was not set to \'debug\'! No config info will be logged.")
|
@@ -226,6 +221,8 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
226
221
|
|
227
222
|
return start_shell(setting("interactive"), binding) if setting("interactive")
|
228
223
|
|
224
|
+
@settings.validate_all
|
225
|
+
|
229
226
|
@settings.format_settings.each {|line| logger.debug(line) }
|
230
227
|
|
231
228
|
if setting("config.string").nil? && setting("path.config").nil?
|
@@ -280,7 +277,12 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
280
277
|
show_short_help
|
281
278
|
return 1
|
282
279
|
rescue => e
|
283
|
-
|
280
|
+
# if logger itself is not initialized
|
281
|
+
if LogStash::Logging::Logger.get_logging_context.nil?
|
282
|
+
$stderr.puts "#{I18n.t("oops")} :error => #{e}, :backtrace => #{e.backtrace}"
|
283
|
+
else
|
284
|
+
logger.fatal(I18n.t("oops"), :error => e, :backtrace => e.backtrace)
|
285
|
+
end
|
284
286
|
return 1
|
285
287
|
ensure
|
286
288
|
Stud::untrap("INT", sigint_id) unless sigint_id.nil?
|
@@ -408,5 +410,11 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
408
410
|
nil
|
409
411
|
end
|
410
412
|
end
|
413
|
+
|
414
|
+
# is the user asking for CLI help subcommand?
|
415
|
+
def cli_help?(args)
|
416
|
+
# I know, double negative
|
417
|
+
!(["--help", "-h"] & args).empty?
|
418
|
+
end
|
411
419
|
|
412
420
|
end
|
data/lib/logstash/settings.rb
CHANGED
@@ -95,6 +95,12 @@ module LogStash
|
|
95
95
|
self.merge(flatten_hash(settings))
|
96
96
|
end
|
97
97
|
|
98
|
+
def validate_all
|
99
|
+
@settings.each do |name, setting|
|
100
|
+
setting.validate_value
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
98
104
|
private
|
99
105
|
def read_yaml(path)
|
100
106
|
YAML.safe_load(IO.read(path)) || {}
|
@@ -123,8 +129,9 @@ module LogStash
|
|
123
129
|
@validator_proc = validator_proc
|
124
130
|
@value = nil
|
125
131
|
@value_is_set = false
|
132
|
+
@strict = strict
|
126
133
|
|
127
|
-
validate(default) if strict
|
134
|
+
validate(default) if @strict
|
128
135
|
@default = default
|
129
136
|
end
|
130
137
|
|
@@ -136,8 +143,12 @@ module LogStash
|
|
136
143
|
@value_is_set
|
137
144
|
end
|
138
145
|
|
146
|
+
def strict?
|
147
|
+
@strict
|
148
|
+
end
|
149
|
+
|
139
150
|
def set(value)
|
140
|
-
validate(value)
|
151
|
+
validate(value) if @strict
|
141
152
|
@value = value
|
142
153
|
@value_is_set = true
|
143
154
|
@value
|
@@ -167,12 +178,18 @@ module LogStash
|
|
167
178
|
self.to_hash == other.to_hash
|
168
179
|
end
|
169
180
|
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
181
|
+
def validate_value
|
182
|
+
validate(value)
|
183
|
+
end
|
184
|
+
|
185
|
+
protected
|
186
|
+
def validate(input)
|
187
|
+
if !input.is_a?(@klass)
|
188
|
+
raise ArgumentError.new("Setting \"#{@name}\" must be a #{@klass}. Received: #{input} (#{input.class})")
|
189
|
+
end
|
190
|
+
|
191
|
+
if @validator_proc && !@validator_proc.call(input)
|
192
|
+
raise ArgumentError.new("Failed to validate setting \"#{@name}\" with value: #{input}")
|
176
193
|
end
|
177
194
|
end
|
178
195
|
|
@@ -351,6 +368,13 @@ module LogStash
|
|
351
368
|
end
|
352
369
|
end
|
353
370
|
|
371
|
+
class NullableString < String
|
372
|
+
def validate(value)
|
373
|
+
return if value.nil?
|
374
|
+
super(value)
|
375
|
+
end
|
376
|
+
end
|
377
|
+
|
354
378
|
class ExistingFilePath < Setting
|
355
379
|
def initialize(name, default=nil, strict=true)
|
356
380
|
super(name, ::String, default, strict) do |file_path|
|
@@ -364,7 +388,7 @@ module LogStash
|
|
364
388
|
end
|
365
389
|
|
366
390
|
class WritableDirectory < Setting
|
367
|
-
def initialize(name, default=nil, strict=
|
391
|
+
def initialize(name, default=nil, strict=false)
|
368
392
|
super(name, ::String, default, strict) do |path|
|
369
393
|
if ::File.directory?(path) && ::File.writable?(path)
|
370
394
|
true
|
@@ -378,3 +402,4 @@ module LogStash
|
|
378
402
|
|
379
403
|
SETTINGS = Settings.new
|
380
404
|
end
|
405
|
+
|
@@ -33,17 +33,20 @@ module LogStash::Util
|
|
33
33
|
end
|
34
34
|
|
35
35
|
# tags is an array of string. sprintf syntax can be used.
|
36
|
-
def add_tags(
|
37
|
-
tags
|
38
|
-
|
39
|
-
|
36
|
+
def add_tags(new_tags, event, pluginname)
|
37
|
+
tags = event.get("tags")
|
38
|
+
tags = tags.nil? ? [] : Array(tags)
|
39
|
+
|
40
|
+
new_tags.each do |new_tag|
|
41
|
+
new_tag = event.sprintf(new_tag)
|
42
|
+
self.logger.debug? and self.logger.debug("#{pluginname}: adding tag", "tag" => new_tag)
|
40
43
|
# note below that the tags array field needs to be updated then reassigned to the event.
|
41
44
|
# this is important because a construct like event["tags"] << tag will not work
|
42
45
|
# in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140
|
43
|
-
tags
|
44
|
-
tags << tag
|
45
|
-
event.set("tags", tags)
|
46
|
+
tags << new_tag #unless tags.include?(new_tag)
|
46
47
|
end
|
48
|
+
|
49
|
+
event.set("tags", tags)
|
47
50
|
end
|
48
51
|
|
49
52
|
end # module LogStash::Util::Decorators
|
@@ -148,7 +148,10 @@ module LogStash; module Util
|
|
148
148
|
@shutdown_signal_received = false
|
149
149
|
@flush_signal_received = false
|
150
150
|
@originals = Hash.new
|
151
|
-
|
151
|
+
|
152
|
+
# TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
|
153
|
+
# @cancelled = Hash.new
|
154
|
+
|
152
155
|
@generated = Hash.new
|
153
156
|
@iterating_temp = Hash.new
|
154
157
|
@iterating = false # Atomic Boolean maybe? Although batches are not shared across threads
|
@@ -168,17 +171,22 @@ module LogStash; module Util
|
|
168
171
|
end
|
169
172
|
|
170
173
|
def cancel(event)
|
171
|
-
|
174
|
+
# TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
|
175
|
+
raise("cancel is unsupported")
|
176
|
+
# @cancelled[event] = true
|
172
177
|
end
|
173
178
|
|
174
179
|
def each(&blk)
|
175
180
|
# take care not to cause @originals or @generated to change during iteration
|
176
181
|
@iterating = true
|
182
|
+
|
183
|
+
# below the checks for @cancelled.include?(e) have been replaced by e.cancelled?
|
184
|
+
# TODO: for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
|
177
185
|
@originals.each do |e, _|
|
178
|
-
blk.call(e) unless
|
186
|
+
blk.call(e) unless e.cancelled?
|
179
187
|
end
|
180
188
|
@generated.each do |e, _|
|
181
|
-
blk.call(e) unless
|
189
|
+
blk.call(e) unless e.cancelled?
|
182
190
|
end
|
183
191
|
@iterating = false
|
184
192
|
update_generated
|
@@ -197,7 +205,9 @@ module LogStash; module Util
|
|
197
205
|
end
|
198
206
|
|
199
207
|
def cancelled_size
|
200
|
-
|
208
|
+
# TODO: disabled for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
|
209
|
+
raise("cancelled_size is unsupported ")
|
210
|
+
# @cancelled.size
|
201
211
|
end
|
202
212
|
|
203
213
|
def shutdown_signal_received?
|
data/lib/logstash/version.rb
CHANGED
data/locales/en.yml
CHANGED
@@ -112,8 +112,7 @@ en:
|
|
112
112
|
longer available. %{extra} If you have any questions about this, you
|
113
113
|
are invited to visit https://discuss.elastic.co/c/logstash and ask.
|
114
114
|
file-not-found: |-
|
115
|
-
No config files found: %{path}
|
116
|
-
Can you make sure this path is a logstash config file?
|
115
|
+
No config files found: %{path}. Can you make sure this path is a logstash config file?
|
117
116
|
scheme-not-supported: |-
|
118
117
|
URI scheme not supported: %{path}
|
119
118
|
Either pass a local file path or "file|http://" URI
|
@@ -215,7 +214,7 @@ en:
|
|
215
214
|
Specifying once will show 'informational'
|
216
215
|
logs. Specifying twice will show 'debug'
|
217
216
|
logs. This flag is deprecated. You should use
|
218
|
-
--
|
217
|
+
--log-level=info or --log-level=debug instead.
|
219
218
|
version: |+
|
220
219
|
Emit the version of logstash and its friends,
|
221
220
|
then exit.
|
@@ -253,7 +252,7 @@ en:
|
|
253
252
|
agent: |+
|
254
253
|
Specify an alternate agent plugin name.
|
255
254
|
config_debug: |+
|
256
|
-
Print the compiled config ruby code out as a debug log (you must also have --debug enabled).
|
255
|
+
Print the compiled config ruby code out as a debug log (you must also have --log.level=debug enabled).
|
257
256
|
WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
|
258
257
|
in plaintext passwords appearing in your logs!
|
259
258
|
log_format: |+
|
@@ -264,7 +263,7 @@ en:
|
|
264
263
|
DEPRECATED: use --log.level=debug instead.
|
265
264
|
verbose: |+
|
266
265
|
Set the log level to info.
|
267
|
-
DEPRECATED: use --log.level=
|
266
|
+
DEPRECATED: use --log.level=info instead.
|
268
267
|
quiet: |+
|
269
268
|
Set the log level to info.
|
270
269
|
DEPRECATED: use --log.level=quiet instead.
|
data/logstash-core.gemspec
CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
|
|
17
17
|
gem.require_paths = ["lib", "vendor/jars"]
|
18
18
|
gem.version = LOGSTASH_CORE_VERSION
|
19
19
|
|
20
|
-
gem.add_runtime_dependency "logstash-core-event-java", "5.0.
|
20
|
+
gem.add_runtime_dependency "logstash-core-event-java", "5.0.1"
|
21
21
|
|
22
22
|
gem.add_runtime_dependency "pry", "~> 0.10.1" #(Ruby license)
|
23
23
|
gem.add_runtime_dependency "stud", "~> 0.0.19" #(Apache 2.0 license)
|
@@ -98,6 +98,39 @@ describe LogStash::Filters::NOOP do
|
|
98
98
|
end
|
99
99
|
end
|
100
100
|
|
101
|
+
describe "tags parsing with one tag as string value" do
|
102
|
+
config <<-CONFIG
|
103
|
+
filter {
|
104
|
+
noop {
|
105
|
+
add_tag => ["bar"]
|
106
|
+
}
|
107
|
+
}
|
108
|
+
CONFIG
|
109
|
+
|
110
|
+
sample("type" => "noop") do
|
111
|
+
insist { subject.get("tags") } == ["bar"]
|
112
|
+
end
|
113
|
+
|
114
|
+
sample("type" => "noop", "tags" => "foo") do
|
115
|
+
insist { subject.get("tags") } == ["foo", "bar"]
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
describe "tags parsing with duplicate tags" do
|
120
|
+
config <<-CONFIG
|
121
|
+
filter {
|
122
|
+
noop {
|
123
|
+
add_tag => ["foo"]
|
124
|
+
}
|
125
|
+
}
|
126
|
+
CONFIG
|
127
|
+
|
128
|
+
sample("type" => "noop", "tags" => "foo") do
|
129
|
+
# this is completely weird but seems to be already expected in other specs
|
130
|
+
insist { subject.get("tags") } == ["foo", "foo"]
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
101
134
|
describe "tags parsing with multiple tags" do
|
102
135
|
config <<-CONFIG
|
103
136
|
filter {
|
@@ -133,6 +166,18 @@ describe LogStash::Filters::NOOP do
|
|
133
166
|
}
|
134
167
|
CONFIG
|
135
168
|
|
169
|
+
sample("type" => "noop", "tags" => "foo") do
|
170
|
+
insist { subject.get("tags") } == ["foo"]
|
171
|
+
end
|
172
|
+
|
173
|
+
sample("type" => "noop", "tags" => "t2") do
|
174
|
+
insist { subject.get("tags") } == []
|
175
|
+
end
|
176
|
+
|
177
|
+
sample("type" => "noop", "tags" => ["t2"]) do
|
178
|
+
insist { subject.get("tags") } == []
|
179
|
+
end
|
180
|
+
|
136
181
|
sample("type" => "noop", "tags" => ["t4"]) do
|
137
182
|
insist { subject.get("tags") } == ["t4"]
|
138
183
|
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/instrument/periodic_poller/base"
|
3
|
+
require "logstash/instrument/metric"
|
4
|
+
require "logstash/instrument/collector"
|
5
|
+
|
6
|
+
describe LogStash::Instrument::PeriodicPoller::Base do
|
7
|
+
let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new) }
|
8
|
+
let(:options) { {} }
|
9
|
+
|
10
|
+
subject { described_class.new(metric, options) }
|
11
|
+
|
12
|
+
describe "#update" do
|
13
|
+
it "logs an timeout exception to debug level" do
|
14
|
+
exception = Concurrent::TimeoutError.new
|
15
|
+
expect(subject.logger).to receive(:debug).with(anything, hash_including(:exception => exception.class))
|
16
|
+
subject.update(Time.now, "hola", exception)
|
17
|
+
end
|
18
|
+
|
19
|
+
it "logs any other exception to error level" do
|
20
|
+
exception = Class.new
|
21
|
+
expect(subject.logger).to receive(:error).with(anything, hash_including(:exception => exception.class))
|
22
|
+
subject.update(Time.now, "hola", exception)
|
23
|
+
end
|
24
|
+
|
25
|
+
it "doesnt log anything when no exception is received" do
|
26
|
+
exception = Concurrent::TimeoutError.new
|
27
|
+
expect(subject.logger).not_to receive(:debug).with(anything)
|
28
|
+
expect(subject.logger).not_to receive(:error).with(anything)
|
29
|
+
subject.update(Time.now, "hola", exception)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -100,6 +100,60 @@ describe LogStash::Pipeline do
|
|
100
100
|
pipeline_settings_obj.reset
|
101
101
|
end
|
102
102
|
|
103
|
+
|
104
|
+
describe "event cancellation" do
|
105
|
+
# test harness for https://github.com/elastic/logstash/issues/6055
|
106
|
+
|
107
|
+
let(:output) { DummyOutputWithEventsArray.new }
|
108
|
+
|
109
|
+
before do
|
110
|
+
allow(LogStash::Plugin).to receive(:lookup).with("input", "generator").and_return(LogStash::Inputs::Generator)
|
111
|
+
allow(LogStash::Plugin).to receive(:lookup).with("output", "dummyoutputwitheventsarray").and_return(DummyOutputWithEventsArray)
|
112
|
+
allow(LogStash::Plugin).to receive(:lookup).with("filter", "drop").and_call_original
|
113
|
+
allow(LogStash::Plugin).to receive(:lookup).with("filter", "mutate").and_call_original
|
114
|
+
allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_call_original
|
115
|
+
allow(DummyOutputWithEventsArray).to receive(:new).with(any_args).and_return(output)
|
116
|
+
end
|
117
|
+
|
118
|
+
let(:config) do
|
119
|
+
<<-CONFIG
|
120
|
+
input {
|
121
|
+
generator {
|
122
|
+
lines => ["1", "2", "END"]
|
123
|
+
count => 1
|
124
|
+
}
|
125
|
+
}
|
126
|
+
filter {
|
127
|
+
if [message] == "1" {
|
128
|
+
drop {}
|
129
|
+
}
|
130
|
+
mutate { add_tag => ["notdropped"] }
|
131
|
+
}
|
132
|
+
output { dummyoutputwitheventsarray {} }
|
133
|
+
CONFIG
|
134
|
+
end
|
135
|
+
|
136
|
+
it "should not propage cancelled events from filter to output" do
|
137
|
+
abort_on_exception_state = Thread.abort_on_exception
|
138
|
+
Thread.abort_on_exception = true
|
139
|
+
|
140
|
+
pipeline = LogStash::Pipeline.new(config, pipeline_settings_obj)
|
141
|
+
Thread.new { pipeline.run }
|
142
|
+
sleep 0.1 while !pipeline.ready?
|
143
|
+
wait(3).for do
|
144
|
+
# give us a bit of time to flush the events
|
145
|
+
# puts("*****" + output.events.map{|e| e.message}.to_s)
|
146
|
+
output.events.map{|e| e.get("message")}.include?("END")
|
147
|
+
end.to be_truthy
|
148
|
+
expect(output.events.size).to eq(2)
|
149
|
+
expect(output.events[0].get("tags")).to eq(["notdropped"])
|
150
|
+
expect(output.events[1].get("tags")).to eq(["notdropped"])
|
151
|
+
pipeline.shutdown
|
152
|
+
|
153
|
+
Thread.abort_on_exception = abort_on_exception_state
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
103
157
|
describe "defaulting the pipeline workers based on thread safety" do
|
104
158
|
before(:each) do
|
105
159
|
allow(LogStash::Plugin).to receive(:lookup).with("input", "dummyinput").and_return(DummyInput)
|
@@ -353,8 +353,8 @@ describe LogStash::Runner do
|
|
353
353
|
context "if does not exist" do
|
354
354
|
let(:args) { ["--path.settings", "/tmp/a/a/a/a", "-e", "input {} output {}"] }
|
355
355
|
|
356
|
-
it "should terminate logstash" do
|
357
|
-
expect(subject.run(args)).to eq(
|
356
|
+
it "should not terminate logstash" do
|
357
|
+
expect(subject.run(args)).to eq(nil)
|
358
358
|
end
|
359
359
|
|
360
360
|
context "but if --help is passed" do
|
@@ -63,6 +63,7 @@ describe LogStash::Setting do
|
|
63
63
|
end
|
64
64
|
end
|
65
65
|
end
|
66
|
+
|
66
67
|
describe "#set" do
|
67
68
|
subject { described_class.new("number", Numeric, 1) }
|
68
69
|
it "should change the value of a setting" do
|
@@ -77,12 +78,33 @@ describe LogStash::Setting do
|
|
77
78
|
expect(subject.set?).to eq(true)
|
78
79
|
end
|
79
80
|
end
|
80
|
-
|
81
81
|
context "when the argument's class does not match @klass" do
|
82
82
|
it "should throw an exception" do
|
83
83
|
expect { subject.set("not a number") }.to raise_error
|
84
84
|
end
|
85
85
|
end
|
86
|
+
context "when strict=false" do
|
87
|
+
let(:strict) { false }
|
88
|
+
subject { described_class.new("number", Numeric, 1, strict) }
|
89
|
+
before do
|
90
|
+
expect(subject).not_to receive(:validate)
|
91
|
+
end
|
92
|
+
|
93
|
+
it "should not call #validate" do
|
94
|
+
subject.set(123)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
context "when strict=true" do
|
98
|
+
let(:strict) { true }
|
99
|
+
subject { described_class.new("number", Numeric, 1, strict) }
|
100
|
+
before do
|
101
|
+
expect(subject).to receive(:validate)
|
102
|
+
end
|
103
|
+
|
104
|
+
it "should call #validate" do
|
105
|
+
subject.set(123)
|
106
|
+
end
|
107
|
+
end
|
86
108
|
end
|
87
109
|
|
88
110
|
describe "#reset" do
|
@@ -4,7 +4,7 @@ require "logstash/settings"
|
|
4
4
|
|
5
5
|
describe LogStash::Setting::String do
|
6
6
|
let(:possible_values) { ["a", "b", "c"] }
|
7
|
-
subject { described_class.new("mytext",
|
7
|
+
subject { described_class.new("mytext", possible_values.first, true, possible_values) }
|
8
8
|
describe "#set" do
|
9
9
|
context "when a value is given outside of possible_values" do
|
10
10
|
it "should raise an ArgumentError" do
|
@@ -59,4 +59,31 @@ describe LogStash::Settings do
|
|
59
59
|
expect(subset.get("num.2")).to eq(1000)
|
60
60
|
end
|
61
61
|
end
|
62
|
+
|
63
|
+
describe "#validate_all" do
|
64
|
+
subject { described_class.new }
|
65
|
+
let(:numeric_setting_name) { "example" }
|
66
|
+
let(:numeric_setting) { LogStash::Setting.new(numeric_setting_name, Numeric, 1, false) }
|
67
|
+
|
68
|
+
before do
|
69
|
+
subject.register(numeric_setting)
|
70
|
+
subject.set_value(numeric_setting_name, value)
|
71
|
+
end
|
72
|
+
|
73
|
+
context "when any setting is invalid" do
|
74
|
+
let(:value) { "some string" }
|
75
|
+
|
76
|
+
it "should fail" do
|
77
|
+
expect { subject.validate_all }.to raise_error
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
context "when all settings are valid" do
|
82
|
+
let(:value) { 123 }
|
83
|
+
|
84
|
+
it "should succeed" do
|
85
|
+
expect { subject.validate_all }.not_to raise_error
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
62
89
|
end
|
@@ -94,22 +94,23 @@ describe LogStash::Util::WrappedSynchronousQueue do
|
|
94
94
|
|
95
95
|
it "appends batches to the queue" do
|
96
96
|
batch = write_client.get_new_batch
|
97
|
-
5.times {|i| batch.push("value-#{i}")}
|
97
|
+
5.times {|i| batch.push(LogStash::Event.new({"message" => "value-#{i}"}))}
|
98
98
|
write_client.push_batch(batch)
|
99
99
|
read_batch = read_client.take_batch
|
100
100
|
expect(read_batch.size).to eq(5)
|
101
101
|
i = 0
|
102
102
|
read_batch.each do |data|
|
103
|
-
expect(data).to eq("value-#{i}")
|
104
|
-
read_batch.cancel("value-#{i}") if i > 2
|
105
|
-
|
103
|
+
expect(data.get("message")).to eq("value-#{i}")
|
104
|
+
# read_batch.cancel("value-#{i}") if i > 2 # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
|
105
|
+
data.cancel if i > 2
|
106
|
+
read_batch.merge(LogStash::Event.new({"message" => "generated-#{i}"})) if i > 2
|
106
107
|
i += 1
|
107
108
|
end
|
108
|
-
expect(read_batch.cancelled_size).to eq(2)
|
109
|
+
# expect(read_batch.cancelled_size).to eq(2) # disabled for https://github.com/elastic/logstash/issues/6055
|
109
110
|
i = 0
|
110
111
|
read_batch.each do |data|
|
111
|
-
expect(data).to eq("value-#{i}") if i < 3
|
112
|
-
expect(data).to eq("generated-#{i}") if i > 2
|
112
|
+
expect(data.get("message")).to eq("value-#{i}") if i < 3
|
113
|
+
expect(data.get("message")).to eq("generated-#{i}") if i > 2
|
113
114
|
i += 1
|
114
115
|
end
|
115
116
|
end
|
@@ -25,3 +25,25 @@ class DummyOutput < LogStash::Outputs::Base
|
|
25
25
|
@num_closes = 1
|
26
26
|
end
|
27
27
|
end
|
28
|
+
|
29
|
+
class DummyOutputWithEventsArray < LogStash::Outputs::Base
|
30
|
+
config_name "dummyoutput2"
|
31
|
+
milestone 2
|
32
|
+
|
33
|
+
attr_reader :events
|
34
|
+
|
35
|
+
def initialize(params={})
|
36
|
+
super
|
37
|
+
@events = []
|
38
|
+
end
|
39
|
+
|
40
|
+
def register
|
41
|
+
end
|
42
|
+
|
43
|
+
def receive(event)
|
44
|
+
@events << event
|
45
|
+
end
|
46
|
+
|
47
|
+
def close
|
48
|
+
end
|
49
|
+
end
|
Binary file
|
metadata
CHANGED
@@ -1,21 +1,21 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-core
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.0.
|
4
|
+
version: 5.0.1
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-11-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
15
15
|
requirements:
|
16
16
|
- - '='
|
17
17
|
- !ruby/object:Gem::Version
|
18
|
-
version: 5.0.
|
18
|
+
version: 5.0.1
|
19
19
|
name: logstash-core-event-java
|
20
20
|
prerelease: false
|
21
21
|
type: :runtime
|
@@ -23,7 +23,7 @@ dependencies:
|
|
23
23
|
requirements:
|
24
24
|
- - '='
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 5.0.
|
26
|
+
version: 5.0.1
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
29
29
|
requirements:
|
@@ -343,7 +343,6 @@ files:
|
|
343
343
|
- lib/logstash/instrument/periodic_poller/base.rb
|
344
344
|
- lib/logstash/instrument/periodic_poller/jvm.rb
|
345
345
|
- lib/logstash/instrument/periodic_poller/os.rb
|
346
|
-
- lib/logstash/instrument/periodic_poller/periodic_poller_observer.rb
|
347
346
|
- lib/logstash/instrument/periodic_pollers.rb
|
348
347
|
- lib/logstash/instrument/snapshot.rb
|
349
348
|
- lib/logstash/java_integration.rb
|
@@ -427,6 +426,7 @@ files:
|
|
427
426
|
- spec/logstash/instrument/metric_type/gauge_spec.rb
|
428
427
|
- spec/logstash/instrument/namespaced_metric_spec.rb
|
429
428
|
- spec/logstash/instrument/null_metric_spec.rb
|
429
|
+
- spec/logstash/instrument/periodic_poller/base_spec.rb
|
430
430
|
- spec/logstash/instrument/periodic_poller/jvm_spec.rb
|
431
431
|
- spec/logstash/java_integration_spec.rb
|
432
432
|
- spec/logstash/json_spec.rb
|
@@ -464,7 +464,7 @@ files:
|
|
464
464
|
- vendor/jars/org/apache/logging/log4j/log4j-1.2-api/2.6.2/log4j-1.2-api-2.6.2.jar
|
465
465
|
- vendor/jars/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
|
466
466
|
- vendor/jars/org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar
|
467
|
-
- vendor/jars/org/logstash/logstash-core/5.0.
|
467
|
+
- vendor/jars/org/logstash/logstash-core/5.0.1/logstash-core-5.0.1.jar
|
468
468
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
469
469
|
licenses:
|
470
470
|
- Apache License (2.0)
|
@@ -481,9 +481,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
481
481
|
version: '0'
|
482
482
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
483
483
|
requirements:
|
484
|
-
- - "
|
484
|
+
- - ">="
|
485
485
|
- !ruby/object:Gem::Version
|
486
|
-
version:
|
486
|
+
version: '0'
|
487
487
|
requirements: []
|
488
488
|
rubyforge_project:
|
489
489
|
rubygems_version: 2.4.8
|
@@ -522,6 +522,7 @@ test_files:
|
|
522
522
|
- spec/logstash/instrument/metric_type/gauge_spec.rb
|
523
523
|
- spec/logstash/instrument/namespaced_metric_spec.rb
|
524
524
|
- spec/logstash/instrument/null_metric_spec.rb
|
525
|
+
- spec/logstash/instrument/periodic_poller/base_spec.rb
|
525
526
|
- spec/logstash/instrument/periodic_poller/jvm_spec.rb
|
526
527
|
- spec/logstash/java_integration_spec.rb
|
527
528
|
- spec/logstash/json_spec.rb
|
@@ -1,19 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
module LogStash module Instrument module PeriodicPoller
|
3
|
-
class PeriodicPollerObserver
|
4
|
-
include LogStash::Util::Loggable
|
5
|
-
|
6
|
-
def initialize(poller)
|
7
|
-
@poller = poller
|
8
|
-
end
|
9
|
-
|
10
|
-
def update(time, result, exception)
|
11
|
-
if exception
|
12
|
-
logger.error("PeriodicPoller exception", :poller => @poller,
|
13
|
-
:result => result,
|
14
|
-
:exception => exception,
|
15
|
-
:executed_at => time)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end; end; end
|