logstash-core 6.0.0.beta2-java → 6.0.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/gemspec_jars.rb +5 -5
- data/lib/logstash-core/logstash-core.jar +0 -0
- data/lib/logstash-core/version.rb +8 -4
- data/lib/logstash-core_jars.rb +10 -10
- data/lib/logstash/agent.rb +3 -2
- data/lib/logstash/compiler/lscl.rb +15 -3
- data/lib/logstash/config/config_ast.rb +3 -2
- data/lib/logstash/config/modules_common.rb +1 -0
- data/lib/logstash/config/source/local.rb +2 -1
- data/lib/logstash/instrument/periodic_poller/dlq.rb +8 -3
- data/lib/logstash/instrument/periodic_poller/pq.rb +7 -3
- data/lib/logstash/logging/logger.rb +4 -1
- data/lib/logstash/modules/kibana_client.rb +35 -8
- data/lib/logstash/modules/logstash_config.rb +1 -1
- data/lib/logstash/modules/settings_merger.rb +8 -2
- data/lib/logstash/pipeline.rb +10 -19
- data/lib/logstash/pipeline_action/stop.rb +1 -0
- data/lib/logstash/runner.rb +4 -1
- data/lib/logstash/util/cloud_setting_id.rb +46 -12
- data/lib/logstash/util/modules_setting_array.rb +1 -1
- data/lib/logstash/util/password.rb +2 -4
- data/lib/logstash/util/wrapped_acked_queue.rb +6 -0
- data/lib/logstash/util/wrapped_synchronous_queue.rb +6 -0
- data/lib/logstash/version.rb +8 -10
- data/logstash-core.gemspec +25 -3
- data/spec/logstash/agent/converge_spec.rb +23 -10
- data/spec/logstash/agent_spec.rb +35 -15
- data/spec/logstash/api/modules/node_stats_spec.rb +5 -1
- data/spec/logstash/compiler/compiler_spec.rb +29 -0
- data/spec/logstash/config/source/local_spec.rb +3 -2
- data/spec/logstash/event_spec.rb +57 -0
- data/spec/logstash/modules/kibana_client_spec.rb +60 -0
- data/spec/logstash/modules/logstash_config_spec.rb +7 -1
- data/spec/logstash/modules/scaffold_spec.rb +1 -1
- data/spec/logstash/modules/settings_merger_spec.rb +32 -2
- data/spec/logstash/pipeline_action/create_spec.rb +4 -1
- data/spec/logstash/pipeline_action/reload_spec.rb +4 -1
- data/spec/logstash/pipeline_dlq_commit_spec.rb +3 -1
- data/spec/logstash/pipeline_pq_file_spec.rb +5 -7
- data/spec/logstash/pipeline_spec.rb +26 -38
- data/spec/logstash/runner_spec.rb +1 -5
- data/spec/logstash/settings/modules_spec.rb +13 -2
- data/spec/logstash/settings/writable_directory_spec.rb +13 -10
- data/spec/logstash/timestamp_spec.rb +2 -2
- data/spec/logstash/util/cloud_setting_id_spec.rb +93 -0
- data/spec/support/helpers.rb +1 -1
- data/spec/support/mocks_classes.rb +14 -0
- data/spec/support/shared_contexts.rb +9 -0
- data/versions-gem-copy.yml +23 -0
- metadata +19 -14
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e8c1ff16e405816119bbed2fa145f555e22be2b2bd6db83ba9c072bb589cf58a
|
4
|
+
data.tar.gz: 64a79ecd4f95bdd7118fd5a0a15406eb6e992937fa0f8fac78287d82976b1fdb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8d107250b5c4880960e33dc96dadccd4e54d0b431786d43fbecc5d901e181f6852ee3423a000eaf05eed290d4b061c1344650098a7ffa6b4ff4e1f1e380bde50
|
7
|
+
data.tar.gz: 7e06de56f50e30f957d341ba49fc87f4ea1126344ef48fe9c647232fb5704d1711888fcc17ca5fbabe7a2b6ac248ef0a2b2526ec9484c5a563a52d22ad9ccff5
|
data/gemspec_jars.rb
CHANGED
@@ -5,8 +5,8 @@
|
|
5
5
|
gem.requirements << "jar org.apache.logging.log4j:log4j-slf4j-impl, 2.6.2"
|
6
6
|
gem.requirements << "jar org.apache.logging.log4j:log4j-api, 2.6.2"
|
7
7
|
gem.requirements << "jar org.apache.logging.log4j:log4j-core, 2.6.2"
|
8
|
-
gem.requirements << "jar com.fasterxml.jackson.core:jackson-core, 2.
|
9
|
-
gem.requirements << "jar com.fasterxml.jackson.core:jackson-databind, 2.
|
10
|
-
gem.requirements << "jar com.fasterxml.jackson.core:jackson-annotations, 2.
|
11
|
-
gem.requirements << "jar com.fasterxml.jackson.module:jackson-module-afterburner, 2.
|
12
|
-
gem.requirements << "jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.
|
8
|
+
gem.requirements << "jar com.fasterxml.jackson.core:jackson-core, 2.9.1"
|
9
|
+
gem.requirements << "jar com.fasterxml.jackson.core:jackson-databind, 2.9.1"
|
10
|
+
gem.requirements << "jar com.fasterxml.jackson.core:jackson-annotations, 2.9.1"
|
11
|
+
gem.requirements << "jar com.fasterxml.jackson.module:jackson-module-afterburner, 2.9.1"
|
12
|
+
gem.requirements << "jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.9.1"
|
Binary file
|
@@ -2,7 +2,11 @@
|
|
2
2
|
|
3
3
|
# The version of logstash core gem.
|
4
4
|
#
|
5
|
-
#
|
6
|
-
|
7
|
-
|
8
|
-
|
5
|
+
# sourced from a copy of the master versions.yml file, see logstash-core/logstash-core.gemspec
|
6
|
+
if !defined?(ALL_VERSIONS)
|
7
|
+
require 'yaml'
|
8
|
+
ALL_VERSIONS = YAML.load_file(File.expand_path("../../versions-gem-copy.yml", File.dirname(__FILE__)))
|
9
|
+
end
|
10
|
+
if !defined?(LOGSTASH_CORE_VERSION)
|
11
|
+
LOGSTASH_CORE_VERSION = ALL_VERSIONS.fetch("logstash-core")
|
12
|
+
end
|
data/lib/logstash-core_jars.rb
CHANGED
@@ -2,25 +2,25 @@
|
|
2
2
|
begin
|
3
3
|
require 'jar_dependencies'
|
4
4
|
rescue LoadError
|
5
|
-
require 'com/fasterxml/jackson/core/jackson-databind/2.7.3/jackson-databind-2.7.3.jar'
|
6
5
|
require 'org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar'
|
6
|
+
require 'com/fasterxml/jackson/core/jackson-databind/2.9.1/jackson-databind-2.9.1.jar'
|
7
7
|
require 'org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar'
|
8
8
|
require 'org/slf4j/slf4j-api/1.7.21/slf4j-api-1.7.21.jar'
|
9
|
-
require 'com/fasterxml/jackson/core/jackson-annotations/2.
|
9
|
+
require 'com/fasterxml/jackson/core/jackson-annotations/2.9.1/jackson-annotations-2.9.1.jar'
|
10
10
|
require 'org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar'
|
11
|
-
require 'com/fasterxml/jackson/module/jackson-module-afterburner/2.
|
12
|
-
require 'com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.
|
13
|
-
require 'com/fasterxml/jackson/core/jackson-core/2.
|
11
|
+
require 'com/fasterxml/jackson/module/jackson-module-afterburner/2.9.1/jackson-module-afterburner-2.9.1.jar'
|
12
|
+
require 'com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.9.1/jackson-dataformat-cbor-2.9.1.jar'
|
13
|
+
require 'com/fasterxml/jackson/core/jackson-core/2.9.1/jackson-core-2.9.1.jar'
|
14
14
|
end
|
15
15
|
|
16
16
|
if defined? Jars
|
17
|
-
require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.3' )
|
18
17
|
require_jar( 'org.apache.logging.log4j', 'log4j-core', '2.6.2' )
|
18
|
+
require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.9.1' )
|
19
19
|
require_jar( 'org.apache.logging.log4j', 'log4j-api', '2.6.2' )
|
20
20
|
require_jar( 'org.slf4j', 'slf4j-api', '1.7.21' )
|
21
|
-
require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.
|
21
|
+
require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.9.1' )
|
22
22
|
require_jar( 'org.apache.logging.log4j', 'log4j-slf4j-impl', '2.6.2' )
|
23
|
-
require_jar( 'com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.
|
24
|
-
require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.
|
25
|
-
require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.
|
23
|
+
require_jar( 'com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.9.1' )
|
24
|
+
require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.9.1' )
|
25
|
+
require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.9.1' )
|
26
26
|
end
|
data/lib/logstash/agent.rb
CHANGED
@@ -337,12 +337,13 @@ class LogStash::Agent
|
|
337
337
|
|
338
338
|
unless action_result.successful?
|
339
339
|
logger.error("Failed to execute action", :id => action.pipeline_id,
|
340
|
-
:action_type => action_result.class, :message => action_result.message
|
340
|
+
:action_type => action_result.class, :message => action_result.message,
|
341
|
+
:backtrace => action_result.backtrace)
|
341
342
|
end
|
342
343
|
rescue SystemExit => e
|
343
344
|
converge_result.add(action, e)
|
344
345
|
rescue Exception => e
|
345
|
-
logger.error("Failed to execute action", :action => action, :exception => e.class.name, :message => e.message)
|
346
|
+
logger.error("Failed to execute action", :action => action, :exception => e.class.name, :message => e.message, :backtrace => e.backtrace)
|
346
347
|
converge_result.add(action, e)
|
347
348
|
end
|
348
349
|
end
|
@@ -101,9 +101,21 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
|
|
101
101
|
else
|
102
102
|
[k,v]
|
103
103
|
end
|
104
|
-
}.reduce({}) do |hash,kv|
|
105
|
-
k,v = kv
|
106
|
-
hash[k]
|
104
|
+
}.reduce({}) do |hash, kv|
|
105
|
+
k, v = kv
|
106
|
+
existing = hash[k]
|
107
|
+
if existing.nil?
|
108
|
+
hash[k] = v
|
109
|
+
elsif existing.kind_of?(::Hash)
|
110
|
+
# For legacy reasons, a config can contain multiple `AST::Attribute`s with the same name
|
111
|
+
# and a hash-type value (e.g., "match" in the grok filter), which are merged into a single
|
112
|
+
# hash value; e.g., `{"match" => {"baz" => "bar"}, "match" => {"foo" => "bulb"}}` is
|
113
|
+
# interpreted as `{"match" => {"baz" => "bar", "foo" => "blub"}}`.
|
114
|
+
# (NOTE: this bypasses `AST::Hash`'s ability to detect duplicate keys)
|
115
|
+
hash[k] = existing.merge(v)
|
116
|
+
else
|
117
|
+
hash[k] = existing + v
|
118
|
+
end
|
107
119
|
hash
|
108
120
|
end
|
109
121
|
|
@@ -77,8 +77,9 @@ module LogStash; module Config; module AST
|
|
77
77
|
# of the output/filter function
|
78
78
|
definitions << "define_singleton_method :#{type}_func do |event|"
|
79
79
|
definitions << " targeted_outputs = []" if type == "output"
|
80
|
-
definitions << " events =
|
81
|
-
definitions << " @logger.debug? && @logger.debug(\"#{type} received\", \"event\" => event.to_hash)"
|
80
|
+
definitions << " events = event" if type == "filter"
|
81
|
+
definitions << " @logger.debug? && @logger.debug(\"#{type} received\", \"event\" => event.to_hash)" if type == "output"
|
82
|
+
definitions << " @logger.debug? && events.each { |e| @logger.debug(\"#{type} received\", \"event\" => e.to_hash)}" if type == "filter"
|
82
83
|
|
83
84
|
sections.select { |s| s.plugin_type.text_value == type }.each do |s|
|
84
85
|
definitions << s.compile.split("\n", -1).map { |e| " #{e}" }
|
@@ -65,6 +65,7 @@ module LogStash module Config
|
|
65
65
|
pipeline_id = alt_name
|
66
66
|
module_settings.set("pipeline.id", pipeline_id)
|
67
67
|
LogStash::Modules::SettingsMerger.merge_cloud_settings(module_hash, module_settings)
|
68
|
+
LogStash::Modules::SettingsMerger.merge_kibana_auth!(module_hash)
|
68
69
|
current_module.with_settings(module_hash)
|
69
70
|
config_test = settings.get("config.test_and_exit")
|
70
71
|
modul_setup = settings.get("modules_setup")
|
@@ -71,7 +71,8 @@ module LogStash module Config module Source
|
|
71
71
|
end
|
72
72
|
|
73
73
|
config_string = ::File.read(file)
|
74
|
-
|
74
|
+
config_string.force_encoding("UTF-8")
|
75
|
+
|
75
76
|
if config_string.valid_encoding?
|
76
77
|
part = org.logstash.common.SourceWithMetadata.new("file", file, 0, 0, config_string)
|
77
78
|
config_parts << part
|
@@ -10,10 +10,15 @@ module LogStash module Instrument module PeriodicPoller
|
|
10
10
|
end
|
11
11
|
|
12
12
|
def collect
|
13
|
-
|
14
|
-
unless
|
15
|
-
pipeline
|
13
|
+
pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
|
14
|
+
unless pipelines.nil?
|
15
|
+
pipelines.each {|_, pipeline|
|
16
|
+
unless pipeline.nil?
|
17
|
+
pipeline.collect_dlq_stats
|
18
|
+
end
|
19
|
+
}
|
16
20
|
end
|
17
21
|
end
|
18
22
|
end
|
19
23
|
end end end
|
24
|
+
|
@@ -11,9 +11,13 @@ module LogStash module Instrument module PeriodicPoller
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def collect
|
14
|
-
|
15
|
-
unless
|
16
|
-
pipeline
|
14
|
+
pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
|
15
|
+
unless pipelines.nil?
|
16
|
+
pipelines.each {|_, pipeline|
|
17
|
+
unless pipeline.nil?
|
18
|
+
pipeline.collect_stats
|
19
|
+
end
|
20
|
+
}
|
17
21
|
end
|
18
22
|
end
|
19
23
|
end
|
@@ -73,7 +73,7 @@ module LogStash
|
|
73
73
|
raise ArgumentError, "invalid level[#{level}] for logger[#{path}]"
|
74
74
|
end
|
75
75
|
|
76
|
-
def self.
|
76
|
+
def self.reconfigure(config_location)
|
77
77
|
@@config_mutex.synchronize do
|
78
78
|
config_location_uri = URI.create(config_location)
|
79
79
|
file_path = config_location_uri.path
|
@@ -92,6 +92,9 @@ module LogStash
|
|
92
92
|
end
|
93
93
|
end
|
94
94
|
|
95
|
+
# until dev_utils/rspec/spec_helper is changed, we need to have both methods
|
96
|
+
singleton_class.send(:alias_method, :initialize, :reconfigure)
|
97
|
+
|
95
98
|
def self.get_logging_context
|
96
99
|
return LoggerContext.getContext(false)
|
97
100
|
end
|
@@ -24,9 +24,11 @@ module LogStash module Modules class KibanaClient
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
27
|
-
|
27
|
+
SCHEME_REGEX = /^https?$/
|
28
28
|
|
29
|
-
|
29
|
+
attr_reader :version, :endpoint
|
30
|
+
|
31
|
+
def initialize(settings, client = nil) # allow for test mock injection
|
30
32
|
@settings = settings
|
31
33
|
|
32
34
|
client_options = {
|
@@ -38,8 +40,8 @@ module LogStash module Modules class KibanaClient
|
|
38
40
|
}
|
39
41
|
|
40
42
|
ssl_options = {}
|
41
|
-
|
42
|
-
if
|
43
|
+
ssl_enabled = @settings["var.kibana.ssl.enabled"] == "true"
|
44
|
+
if ssl_enabled
|
43
45
|
ssl_options[:verify] = @settings.fetch("var.kibana.ssl.verification_mode", "strict").to_sym
|
44
46
|
ssl_options[:ca_file] = @settings.fetch("var.kibana.ssl.certificate_authority", nil)
|
45
47
|
ssl_options[:client_cert] = @settings.fetch("var.kibana.ssl.certificate", nil)
|
@@ -48,9 +50,34 @@ module LogStash module Modules class KibanaClient
|
|
48
50
|
|
49
51
|
client_options[:ssl] = ssl_options
|
50
52
|
|
51
|
-
@client = Manticore::Client.new(client_options)
|
52
53
|
@host = @settings.fetch("var.kibana.host", "localhost:5601")
|
53
|
-
|
54
|
+
implicit_scheme, colon_slash_slash, host = @host.partition("://")
|
55
|
+
explicit_scheme = @settings["var.kibana.scheme"]
|
56
|
+
@scheme = "http"
|
57
|
+
if !colon_slash_slash.empty?
|
58
|
+
if !explicit_scheme.nil? && implicit_scheme != explicit_scheme
|
59
|
+
# both are set and not the same - error
|
60
|
+
msg = sprintf("Detected differing Kibana host schemes as sourced from var.kibana.host: '%s' and var.kibana.scheme: '%s'", implicit_scheme, explicit_scheme)
|
61
|
+
raise ArgumentError.new(msg)
|
62
|
+
end
|
63
|
+
@scheme = implicit_scheme
|
64
|
+
@host = host
|
65
|
+
elsif !explicit_scheme.nil?
|
66
|
+
@scheme = explicit_scheme
|
67
|
+
end
|
68
|
+
|
69
|
+
if SCHEME_REGEX.match(@scheme).nil?
|
70
|
+
msg = sprintf("Kibana host scheme given is invalid, given value: '%s' - acceptable values: 'http', 'https'", @scheme)
|
71
|
+
raise ArgumentError.new(msg)
|
72
|
+
end
|
73
|
+
|
74
|
+
if ssl_enabled && @scheme != "https"
|
75
|
+
@scheme = "https"
|
76
|
+
end
|
77
|
+
|
78
|
+
@endpoint = "#{@scheme}://#{@host}"
|
79
|
+
|
80
|
+
@client = client || Manticore::Client.new(client_options)
|
54
81
|
@http_options = {:headers => {'Content-Type' => 'application/json'}}
|
55
82
|
username = @settings["var.kibana.username"]
|
56
83
|
if username
|
@@ -77,7 +104,7 @@ module LogStash module Modules class KibanaClient
|
|
77
104
|
end
|
78
105
|
|
79
106
|
def version_parts
|
80
|
-
@version.split(
|
107
|
+
@version.split(/[.-]/)
|
81
108
|
end
|
82
109
|
|
83
110
|
def host_settings
|
@@ -119,6 +146,6 @@ module LogStash module Modules class KibanaClient
|
|
119
146
|
end
|
120
147
|
|
121
148
|
def full_url(relative)
|
122
|
-
"#{@
|
149
|
+
"#{@endpoint}/#{relative}"
|
123
150
|
end
|
124
151
|
end end end
|
@@ -69,7 +69,7 @@ module LogStash module Modules class LogStashConfig
|
|
69
69
|
password = @settings["var.elasticsearch.password"]
|
70
70
|
lines = ["hosts => #{hosts}", "index => \"#{index}\""]
|
71
71
|
lines.push(user ? "user => \"#{user}\"" : nil)
|
72
|
-
lines.push(password ? "password => \"#{password}\"" : nil)
|
72
|
+
lines.push(password ? "password => \"#{password.value}\"" : nil)
|
73
73
|
lines.push(type_string ? "document_type => #{type_string}" : nil)
|
74
74
|
lines.push("ssl => #{@settings.fetch('var.elasticsearch.ssl.enabled', false)}")
|
75
75
|
if cacert = @settings["var.elasticsearch.ssl.certificate_authority"]
|
@@ -40,9 +40,10 @@ module LogStash module Modules module SettingsMerger
|
|
40
40
|
settings_copy = LogStash::Util.deep_clone(module_settings)
|
41
41
|
end
|
42
42
|
|
43
|
-
module_settings["var.kibana.scheme"] =
|
43
|
+
module_settings["var.kibana.scheme"] = cloud_id.kibana_scheme
|
44
44
|
module_settings["var.kibana.host"] = cloud_id.kibana_host
|
45
|
-
|
45
|
+
# elasticsearch client does not use scheme, it URI parses the host setting
|
46
|
+
module_settings["var.elasticsearch.hosts"] = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}"
|
46
47
|
unless cloud_auth.nil?
|
47
48
|
module_settings["var.elasticsearch.username"] = cloud_auth.username
|
48
49
|
module_settings["var.elasticsearch.password"] = cloud_auth.password
|
@@ -54,6 +55,11 @@ module LogStash module Modules module SettingsMerger
|
|
54
55
|
end
|
55
56
|
end
|
56
57
|
|
58
|
+
def merge_kibana_auth!(module_settings)
|
59
|
+
module_settings["var.kibana.username"] = module_settings["var.elasticsearch.username"] if module_settings["var.kibana.username"].nil?
|
60
|
+
module_settings["var.kibana.password"] = module_settings["var.elasticsearch.password"] if module_settings["var.kibana.password"].nil?
|
61
|
+
end
|
62
|
+
|
57
63
|
def format_module_settings(settings_before, settings_after)
|
58
64
|
output = []
|
59
65
|
output << "-------- Module Settings ---------"
|
data/lib/logstash/pipeline.rb
CHANGED
@@ -498,13 +498,9 @@ module LogStash; class Pipeline < BasePipeline
|
|
498
498
|
end
|
499
499
|
|
500
500
|
def filter_batch(batch)
|
501
|
-
batch.each do |
|
502
|
-
|
503
|
-
|
504
|
-
filter_func(event).each do |e|
|
505
|
-
#these are both original and generated events
|
506
|
-
batch.merge(e) unless e.cancelled?
|
507
|
-
end
|
501
|
+
filter_func(batch.to_a).each do |e|
|
502
|
+
#these are both original and generated events
|
503
|
+
batch.merge(e) unless e.cancelled?
|
508
504
|
end
|
509
505
|
@filter_queue_client.add_filtered_metrics(batch)
|
510
506
|
@events_filtered.increment(batch.size)
|
@@ -584,17 +580,12 @@ module LogStash; class Pipeline < BasePipeline
|
|
584
580
|
end
|
585
581
|
|
586
582
|
# otherwise, report error and restart
|
587
|
-
|
588
|
-
|
589
|
-
|
590
|
-
|
591
|
-
|
592
|
-
|
593
|
-
:stacktrace => e.backtrace.join("\n"))))
|
594
|
-
else
|
595
|
-
@logger.error(I18n.t("logstash.pipeline.worker-error",
|
596
|
-
default_logging_keys(:plugin => plugin.inspect, :error => e.message)))
|
597
|
-
end
|
583
|
+
@logger.error(I18n.t("logstash.pipeline.worker-error-debug",
|
584
|
+
default_logging_keys(
|
585
|
+
:plugin => plugin.inspect,
|
586
|
+
:error => e.message,
|
587
|
+
:exception => e.class,
|
588
|
+
:stacktrace => e.backtrace.join("\n"))))
|
598
589
|
|
599
590
|
# Assuming the failure that caused this exception is transient,
|
600
591
|
# let's sleep for a bit and execute #run again
|
@@ -667,7 +658,7 @@ module LogStash; class Pipeline < BasePipeline
|
|
667
658
|
def filter(event, &block)
|
668
659
|
maybe_setup_out_plugins
|
669
660
|
# filter_func returns all filtered events, including cancelled ones
|
670
|
-
filter_func(event).each {|e| block.call(e)}
|
661
|
+
filter_func([event]).each {|e| block.call(e)}
|
671
662
|
end
|
672
663
|
|
673
664
|
# perform filters flush and yield flushed event to the passed block
|
@@ -14,6 +14,7 @@ module LogStash module PipelineAction
|
|
14
14
|
def execute(agent, pipelines)
|
15
15
|
pipeline = pipelines[pipeline_id]
|
16
16
|
pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
|
17
|
+
pipeline.thread.join
|
17
18
|
pipelines.delete(pipeline_id)
|
18
19
|
# If we reach this part of the code we have succeeded because
|
19
20
|
# the shutdown call will block.
|
data/lib/logstash/runner.rb
CHANGED
@@ -249,7 +249,10 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
249
249
|
java.lang.System.setProperty("ls.log.level", setting("log.level"))
|
250
250
|
unless java.lang.System.getProperty("log4j.configurationFile")
|
251
251
|
log4j_config_location = ::File.join(setting("path.settings"), "log4j2.properties")
|
252
|
-
|
252
|
+
|
253
|
+
# Windows safe way to produce a file: URI.
|
254
|
+
file_schema = "file://" + (LogStash::Environment.windows? ? "/" : "")
|
255
|
+
LogStash::Logging::Logger::reconfigure(URI.join(file_schema + File.absolute_path(log4j_config_location)).to_s)
|
253
256
|
end
|
254
257
|
# override log level that may have been introduced from a custom log4j config file
|
255
258
|
LogStash::Logging::Logger::configure_logging(setting("log.level"))
|
@@ -3,8 +3,26 @@ require "logstash/namespace"
|
|
3
3
|
require "base64"
|
4
4
|
|
5
5
|
module LogStash module Util class CloudSettingId
|
6
|
-
attr_reader :original, :decoded, :label, :elasticsearch_host, :kibana_host
|
7
6
|
|
7
|
+
def self.cloud_id_encode(*args)
|
8
|
+
Base64.urlsafe_encode64(args.join("$"))
|
9
|
+
end
|
10
|
+
DOT_SEPARATOR = "."
|
11
|
+
CLOUD_PORT = ":443"
|
12
|
+
|
13
|
+
attr_reader :original, :decoded, :label, :elasticsearch_host, :elasticsearch_scheme, :kibana_host, :kibana_scheme
|
14
|
+
|
15
|
+
# The constructor is expecting a 'cloud.id', a string in 2 variants.
|
16
|
+
# 1 part example: 'dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy'
|
17
|
+
# 2 part example: 'foobar:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy'
|
18
|
+
# The two part variant has a 'label' prepended with a colon separator. The label is not encoded.
|
19
|
+
# The 1 part (or second section of the 2 part variant) is base64 encoded.
|
20
|
+
# The original string before encoding has three segments separated by a dollar sign.
|
21
|
+
# e.g. 'us-east-1.aws.found.io$notareal$identifier'
|
22
|
+
# The first segment is the cloud base url, e.g. 'us-east-1.aws.found.io'
|
23
|
+
# The second segment is the elasticsearch host identifier, e.g. 'notareal'
|
24
|
+
# The third segment is the kibana host identifier, e.g. 'identifier'
|
25
|
+
# The 'cloud.id' value decoded into the #attr_reader ivars.
|
8
26
|
def initialize(value)
|
9
27
|
return if value.nil?
|
10
28
|
|
@@ -12,27 +30,43 @@ module LogStash module Util class CloudSettingId
|
|
12
30
|
raise ArgumentError.new("Cloud Id must be String. Received: #{value.class}")
|
13
31
|
end
|
14
32
|
@original = value
|
15
|
-
@label,
|
16
|
-
if
|
33
|
+
@label, colon, encoded = @original.partition(":")
|
34
|
+
if encoded.empty?
|
17
35
|
@decoded = Base64.urlsafe_decode64(@label) rescue ""
|
18
36
|
@label = ""
|
19
37
|
else
|
20
|
-
@decoded = Base64.urlsafe_decode64(
|
38
|
+
@decoded = Base64.urlsafe_decode64(encoded) rescue ""
|
21
39
|
end
|
40
|
+
|
41
|
+
@decoded = @decoded.encode(Encoding::UTF_8, :invalid => :replace, :undef => :replace)
|
42
|
+
|
22
43
|
unless @decoded.count("$") == 2
|
23
|
-
raise ArgumentError.new("Cloud Id does not decode. Received: \"#{@
|
44
|
+
raise ArgumentError.new("Cloud Id does not decode. You may need to enable Kibana in the Cloud UI. Received: \"#{@decoded}\".")
|
24
45
|
end
|
25
|
-
|
26
|
-
|
27
|
-
|
46
|
+
|
47
|
+
segments = @decoded.split("$")
|
48
|
+
if segments.any?(&:empty?)
|
49
|
+
raise ArgumentError.new("Cloud Id, after decoding, is invalid. Format: '<segment1>$<segment2>$<segment3>'. Received: \"#{@decoded}\".")
|
50
|
+
end
|
51
|
+
cloud_base = segments.shift
|
52
|
+
cloud_host = "#{DOT_SEPARATOR}#{cloud_base}#{CLOUD_PORT}"
|
53
|
+
|
54
|
+
@elasticsearch_host, @kibana_host = segments
|
55
|
+
if @elasticsearch_host == "undefined"
|
56
|
+
raise ArgumentError.new("Cloud Id, after decoding, elasticsearch segment is 'undefined', literally.")
|
57
|
+
end
|
58
|
+
@elasticsearch_scheme = "https"
|
59
|
+
@elasticsearch_host.concat(cloud_host)
|
60
|
+
|
61
|
+
if @kibana_host == "undefined"
|
62
|
+
raise ArgumentError.new("Cloud Id, after decoding, the kibana segment is 'undefined', literally. You may need to enable Kibana in the Cloud UI.")
|
28
63
|
end
|
29
|
-
|
30
|
-
@
|
31
|
-
@kibana_host = sprintf("%s.%s:443", kb_server, cloud_host)
|
64
|
+
@kibana_scheme = "https"
|
65
|
+
@kibana_host.concat(cloud_host)
|
32
66
|
end
|
33
67
|
|
34
68
|
def to_s
|
35
|
-
@
|
69
|
+
@decoded.to_s
|
36
70
|
end
|
37
71
|
|
38
72
|
def inspect
|