logstash-core 6.0.0.beta2-java → 6.0.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/gemspec_jars.rb +5 -5
- data/lib/logstash-core/logstash-core.jar +0 -0
- data/lib/logstash-core/version.rb +8 -4
- data/lib/logstash-core_jars.rb +10 -10
- data/lib/logstash/agent.rb +3 -2
- data/lib/logstash/compiler/lscl.rb +15 -3
- data/lib/logstash/config/config_ast.rb +3 -2
- data/lib/logstash/config/modules_common.rb +1 -0
- data/lib/logstash/config/source/local.rb +2 -1
- data/lib/logstash/instrument/periodic_poller/dlq.rb +8 -3
- data/lib/logstash/instrument/periodic_poller/pq.rb +7 -3
- data/lib/logstash/logging/logger.rb +4 -1
- data/lib/logstash/modules/kibana_client.rb +35 -8
- data/lib/logstash/modules/logstash_config.rb +1 -1
- data/lib/logstash/modules/settings_merger.rb +8 -2
- data/lib/logstash/pipeline.rb +10 -19
- data/lib/logstash/pipeline_action/stop.rb +1 -0
- data/lib/logstash/runner.rb +4 -1
- data/lib/logstash/util/cloud_setting_id.rb +46 -12
- data/lib/logstash/util/modules_setting_array.rb +1 -1
- data/lib/logstash/util/password.rb +2 -4
- data/lib/logstash/util/wrapped_acked_queue.rb +6 -0
- data/lib/logstash/util/wrapped_synchronous_queue.rb +6 -0
- data/lib/logstash/version.rb +8 -10
- data/logstash-core.gemspec +25 -3
- data/spec/logstash/agent/converge_spec.rb +23 -10
- data/spec/logstash/agent_spec.rb +35 -15
- data/spec/logstash/api/modules/node_stats_spec.rb +5 -1
- data/spec/logstash/compiler/compiler_spec.rb +29 -0
- data/spec/logstash/config/source/local_spec.rb +3 -2
- data/spec/logstash/event_spec.rb +57 -0
- data/spec/logstash/modules/kibana_client_spec.rb +60 -0
- data/spec/logstash/modules/logstash_config_spec.rb +7 -1
- data/spec/logstash/modules/scaffold_spec.rb +1 -1
- data/spec/logstash/modules/settings_merger_spec.rb +32 -2
- data/spec/logstash/pipeline_action/create_spec.rb +4 -1
- data/spec/logstash/pipeline_action/reload_spec.rb +4 -1
- data/spec/logstash/pipeline_dlq_commit_spec.rb +3 -1
- data/spec/logstash/pipeline_pq_file_spec.rb +5 -7
- data/spec/logstash/pipeline_spec.rb +26 -38
- data/spec/logstash/runner_spec.rb +1 -5
- data/spec/logstash/settings/modules_spec.rb +13 -2
- data/spec/logstash/settings/writable_directory_spec.rb +13 -10
- data/spec/logstash/timestamp_spec.rb +2 -2
- data/spec/logstash/util/cloud_setting_id_spec.rb +93 -0
- data/spec/support/helpers.rb +1 -1
- data/spec/support/mocks_classes.rb +14 -0
- data/spec/support/shared_contexts.rb +9 -0
- data/versions-gem-copy.yml +23 -0
- metadata +19 -14
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e8c1ff16e405816119bbed2fa145f555e22be2b2bd6db83ba9c072bb589cf58a
|
|
4
|
+
data.tar.gz: 64a79ecd4f95bdd7118fd5a0a15406eb6e992937fa0f8fac78287d82976b1fdb
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8d107250b5c4880960e33dc96dadccd4e54d0b431786d43fbecc5d901e181f6852ee3423a000eaf05eed290d4b061c1344650098a7ffa6b4ff4e1f1e380bde50
|
|
7
|
+
data.tar.gz: 7e06de56f50e30f957d341ba49fc87f4ea1126344ef48fe9c647232fb5704d1711888fcc17ca5fbabe7a2b6ac248ef0a2b2526ec9484c5a563a52d22ad9ccff5
|
data/gemspec_jars.rb
CHANGED
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
gem.requirements << "jar org.apache.logging.log4j:log4j-slf4j-impl, 2.6.2"
|
|
6
6
|
gem.requirements << "jar org.apache.logging.log4j:log4j-api, 2.6.2"
|
|
7
7
|
gem.requirements << "jar org.apache.logging.log4j:log4j-core, 2.6.2"
|
|
8
|
-
gem.requirements << "jar com.fasterxml.jackson.core:jackson-core, 2.
|
|
9
|
-
gem.requirements << "jar com.fasterxml.jackson.core:jackson-databind, 2.
|
|
10
|
-
gem.requirements << "jar com.fasterxml.jackson.core:jackson-annotations, 2.
|
|
11
|
-
gem.requirements << "jar com.fasterxml.jackson.module:jackson-module-afterburner, 2.
|
|
12
|
-
gem.requirements << "jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.
|
|
8
|
+
gem.requirements << "jar com.fasterxml.jackson.core:jackson-core, 2.9.1"
|
|
9
|
+
gem.requirements << "jar com.fasterxml.jackson.core:jackson-databind, 2.9.1"
|
|
10
|
+
gem.requirements << "jar com.fasterxml.jackson.core:jackson-annotations, 2.9.1"
|
|
11
|
+
gem.requirements << "jar com.fasterxml.jackson.module:jackson-module-afterburner, 2.9.1"
|
|
12
|
+
gem.requirements << "jar com.fasterxml.jackson.dataformat:jackson-dataformat-cbor, 2.9.1"
|
|
Binary file
|
|
@@ -2,7 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
# The version of logstash core gem.
|
|
4
4
|
#
|
|
5
|
-
#
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
# sourced from a copy of the master versions.yml file, see logstash-core/logstash-core.gemspec
|
|
6
|
+
if !defined?(ALL_VERSIONS)
|
|
7
|
+
require 'yaml'
|
|
8
|
+
ALL_VERSIONS = YAML.load_file(File.expand_path("../../versions-gem-copy.yml", File.dirname(__FILE__)))
|
|
9
|
+
end
|
|
10
|
+
if !defined?(LOGSTASH_CORE_VERSION)
|
|
11
|
+
LOGSTASH_CORE_VERSION = ALL_VERSIONS.fetch("logstash-core")
|
|
12
|
+
end
|
data/lib/logstash-core_jars.rb
CHANGED
|
@@ -2,25 +2,25 @@
|
|
|
2
2
|
begin
|
|
3
3
|
require 'jar_dependencies'
|
|
4
4
|
rescue LoadError
|
|
5
|
-
require 'com/fasterxml/jackson/core/jackson-databind/2.7.3/jackson-databind-2.7.3.jar'
|
|
6
5
|
require 'org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar'
|
|
6
|
+
require 'com/fasterxml/jackson/core/jackson-databind/2.9.1/jackson-databind-2.9.1.jar'
|
|
7
7
|
require 'org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar'
|
|
8
8
|
require 'org/slf4j/slf4j-api/1.7.21/slf4j-api-1.7.21.jar'
|
|
9
|
-
require 'com/fasterxml/jackson/core/jackson-annotations/2.
|
|
9
|
+
require 'com/fasterxml/jackson/core/jackson-annotations/2.9.1/jackson-annotations-2.9.1.jar'
|
|
10
10
|
require 'org/apache/logging/log4j/log4j-slf4j-impl/2.6.2/log4j-slf4j-impl-2.6.2.jar'
|
|
11
|
-
require 'com/fasterxml/jackson/module/jackson-module-afterburner/2.
|
|
12
|
-
require 'com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.
|
|
13
|
-
require 'com/fasterxml/jackson/core/jackson-core/2.
|
|
11
|
+
require 'com/fasterxml/jackson/module/jackson-module-afterburner/2.9.1/jackson-module-afterburner-2.9.1.jar'
|
|
12
|
+
require 'com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.9.1/jackson-dataformat-cbor-2.9.1.jar'
|
|
13
|
+
require 'com/fasterxml/jackson/core/jackson-core/2.9.1/jackson-core-2.9.1.jar'
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
if defined? Jars
|
|
17
|
-
require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.7.3' )
|
|
18
17
|
require_jar( 'org.apache.logging.log4j', 'log4j-core', '2.6.2' )
|
|
18
|
+
require_jar( 'com.fasterxml.jackson.core', 'jackson-databind', '2.9.1' )
|
|
19
19
|
require_jar( 'org.apache.logging.log4j', 'log4j-api', '2.6.2' )
|
|
20
20
|
require_jar( 'org.slf4j', 'slf4j-api', '1.7.21' )
|
|
21
|
-
require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.
|
|
21
|
+
require_jar( 'com.fasterxml.jackson.core', 'jackson-annotations', '2.9.1' )
|
|
22
22
|
require_jar( 'org.apache.logging.log4j', 'log4j-slf4j-impl', '2.6.2' )
|
|
23
|
-
require_jar( 'com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.
|
|
24
|
-
require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.
|
|
25
|
-
require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.
|
|
23
|
+
require_jar( 'com.fasterxml.jackson.module', 'jackson-module-afterburner', '2.9.1' )
|
|
24
|
+
require_jar( 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.9.1' )
|
|
25
|
+
require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.9.1' )
|
|
26
26
|
end
|
data/lib/logstash/agent.rb
CHANGED
|
@@ -337,12 +337,13 @@ class LogStash::Agent
|
|
|
337
337
|
|
|
338
338
|
unless action_result.successful?
|
|
339
339
|
logger.error("Failed to execute action", :id => action.pipeline_id,
|
|
340
|
-
:action_type => action_result.class, :message => action_result.message
|
|
340
|
+
:action_type => action_result.class, :message => action_result.message,
|
|
341
|
+
:backtrace => action_result.backtrace)
|
|
341
342
|
end
|
|
342
343
|
rescue SystemExit => e
|
|
343
344
|
converge_result.add(action, e)
|
|
344
345
|
rescue Exception => e
|
|
345
|
-
logger.error("Failed to execute action", :action => action, :exception => e.class.name, :message => e.message)
|
|
346
|
+
logger.error("Failed to execute action", :action => action, :exception => e.class.name, :message => e.message, :backtrace => e.backtrace)
|
|
346
347
|
converge_result.add(action, e)
|
|
347
348
|
end
|
|
348
349
|
end
|
|
@@ -101,9 +101,21 @@ module LogStashCompilerLSCLGrammar; module LogStash; module Compiler; module LSC
|
|
|
101
101
|
else
|
|
102
102
|
[k,v]
|
|
103
103
|
end
|
|
104
|
-
}.reduce({}) do |hash,kv|
|
|
105
|
-
k,v = kv
|
|
106
|
-
hash[k]
|
|
104
|
+
}.reduce({}) do |hash, kv|
|
|
105
|
+
k, v = kv
|
|
106
|
+
existing = hash[k]
|
|
107
|
+
if existing.nil?
|
|
108
|
+
hash[k] = v
|
|
109
|
+
elsif existing.kind_of?(::Hash)
|
|
110
|
+
# For legacy reasons, a config can contain multiple `AST::Attribute`s with the same name
|
|
111
|
+
# and a hash-type value (e.g., "match" in the grok filter), which are merged into a single
|
|
112
|
+
# hash value; e.g., `{"match" => {"baz" => "bar"}, "match" => {"foo" => "bulb"}}` is
|
|
113
|
+
# interpreted as `{"match" => {"baz" => "bar", "foo" => "blub"}}`.
|
|
114
|
+
# (NOTE: this bypasses `AST::Hash`'s ability to detect duplicate keys)
|
|
115
|
+
hash[k] = existing.merge(v)
|
|
116
|
+
else
|
|
117
|
+
hash[k] = existing + v
|
|
118
|
+
end
|
|
107
119
|
hash
|
|
108
120
|
end
|
|
109
121
|
|
|
@@ -77,8 +77,9 @@ module LogStash; module Config; module AST
|
|
|
77
77
|
# of the output/filter function
|
|
78
78
|
definitions << "define_singleton_method :#{type}_func do |event|"
|
|
79
79
|
definitions << " targeted_outputs = []" if type == "output"
|
|
80
|
-
definitions << " events =
|
|
81
|
-
definitions << " @logger.debug? && @logger.debug(\"#{type} received\", \"event\" => event.to_hash)"
|
|
80
|
+
definitions << " events = event" if type == "filter"
|
|
81
|
+
definitions << " @logger.debug? && @logger.debug(\"#{type} received\", \"event\" => event.to_hash)" if type == "output"
|
|
82
|
+
definitions << " @logger.debug? && events.each { |e| @logger.debug(\"#{type} received\", \"event\" => e.to_hash)}" if type == "filter"
|
|
82
83
|
|
|
83
84
|
sections.select { |s| s.plugin_type.text_value == type }.each do |s|
|
|
84
85
|
definitions << s.compile.split("\n", -1).map { |e| " #{e}" }
|
|
@@ -65,6 +65,7 @@ module LogStash module Config
|
|
|
65
65
|
pipeline_id = alt_name
|
|
66
66
|
module_settings.set("pipeline.id", pipeline_id)
|
|
67
67
|
LogStash::Modules::SettingsMerger.merge_cloud_settings(module_hash, module_settings)
|
|
68
|
+
LogStash::Modules::SettingsMerger.merge_kibana_auth!(module_hash)
|
|
68
69
|
current_module.with_settings(module_hash)
|
|
69
70
|
config_test = settings.get("config.test_and_exit")
|
|
70
71
|
modul_setup = settings.get("modules_setup")
|
|
@@ -71,7 +71,8 @@ module LogStash module Config module Source
|
|
|
71
71
|
end
|
|
72
72
|
|
|
73
73
|
config_string = ::File.read(file)
|
|
74
|
-
|
|
74
|
+
config_string.force_encoding("UTF-8")
|
|
75
|
+
|
|
75
76
|
if config_string.valid_encoding?
|
|
76
77
|
part = org.logstash.common.SourceWithMetadata.new("file", file, 0, 0, config_string)
|
|
77
78
|
config_parts << part
|
|
@@ -10,10 +10,15 @@ module LogStash module Instrument module PeriodicPoller
|
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
def collect
|
|
13
|
-
|
|
14
|
-
unless
|
|
15
|
-
pipeline
|
|
13
|
+
pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
|
|
14
|
+
unless pipelines.nil?
|
|
15
|
+
pipelines.each {|_, pipeline|
|
|
16
|
+
unless pipeline.nil?
|
|
17
|
+
pipeline.collect_dlq_stats
|
|
18
|
+
end
|
|
19
|
+
}
|
|
16
20
|
end
|
|
17
21
|
end
|
|
18
22
|
end
|
|
19
23
|
end end end
|
|
24
|
+
|
|
@@ -11,9 +11,13 @@ module LogStash module Instrument module PeriodicPoller
|
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def collect
|
|
14
|
-
|
|
15
|
-
unless
|
|
16
|
-
pipeline
|
|
14
|
+
pipelines = @agent.with_running_user_defined_pipelines {|pipelines| pipelines}
|
|
15
|
+
unless pipelines.nil?
|
|
16
|
+
pipelines.each {|_, pipeline|
|
|
17
|
+
unless pipeline.nil?
|
|
18
|
+
pipeline.collect_stats
|
|
19
|
+
end
|
|
20
|
+
}
|
|
17
21
|
end
|
|
18
22
|
end
|
|
19
23
|
end
|
|
@@ -73,7 +73,7 @@ module LogStash
|
|
|
73
73
|
raise ArgumentError, "invalid level[#{level}] for logger[#{path}]"
|
|
74
74
|
end
|
|
75
75
|
|
|
76
|
-
def self.
|
|
76
|
+
def self.reconfigure(config_location)
|
|
77
77
|
@@config_mutex.synchronize do
|
|
78
78
|
config_location_uri = URI.create(config_location)
|
|
79
79
|
file_path = config_location_uri.path
|
|
@@ -92,6 +92,9 @@ module LogStash
|
|
|
92
92
|
end
|
|
93
93
|
end
|
|
94
94
|
|
|
95
|
+
# until dev_utils/rspec/spec_helper is changed, we need to have both methods
|
|
96
|
+
singleton_class.send(:alias_method, :initialize, :reconfigure)
|
|
97
|
+
|
|
95
98
|
def self.get_logging_context
|
|
96
99
|
return LoggerContext.getContext(false)
|
|
97
100
|
end
|
|
@@ -24,9 +24,11 @@ module LogStash module Modules class KibanaClient
|
|
|
24
24
|
end
|
|
25
25
|
end
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
SCHEME_REGEX = /^https?$/
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
attr_reader :version, :endpoint
|
|
30
|
+
|
|
31
|
+
def initialize(settings, client = nil) # allow for test mock injection
|
|
30
32
|
@settings = settings
|
|
31
33
|
|
|
32
34
|
client_options = {
|
|
@@ -38,8 +40,8 @@ module LogStash module Modules class KibanaClient
|
|
|
38
40
|
}
|
|
39
41
|
|
|
40
42
|
ssl_options = {}
|
|
41
|
-
|
|
42
|
-
if
|
|
43
|
+
ssl_enabled = @settings["var.kibana.ssl.enabled"] == "true"
|
|
44
|
+
if ssl_enabled
|
|
43
45
|
ssl_options[:verify] = @settings.fetch("var.kibana.ssl.verification_mode", "strict").to_sym
|
|
44
46
|
ssl_options[:ca_file] = @settings.fetch("var.kibana.ssl.certificate_authority", nil)
|
|
45
47
|
ssl_options[:client_cert] = @settings.fetch("var.kibana.ssl.certificate", nil)
|
|
@@ -48,9 +50,34 @@ module LogStash module Modules class KibanaClient
|
|
|
48
50
|
|
|
49
51
|
client_options[:ssl] = ssl_options
|
|
50
52
|
|
|
51
|
-
@client = Manticore::Client.new(client_options)
|
|
52
53
|
@host = @settings.fetch("var.kibana.host", "localhost:5601")
|
|
53
|
-
|
|
54
|
+
implicit_scheme, colon_slash_slash, host = @host.partition("://")
|
|
55
|
+
explicit_scheme = @settings["var.kibana.scheme"]
|
|
56
|
+
@scheme = "http"
|
|
57
|
+
if !colon_slash_slash.empty?
|
|
58
|
+
if !explicit_scheme.nil? && implicit_scheme != explicit_scheme
|
|
59
|
+
# both are set and not the same - error
|
|
60
|
+
msg = sprintf("Detected differing Kibana host schemes as sourced from var.kibana.host: '%s' and var.kibana.scheme: '%s'", implicit_scheme, explicit_scheme)
|
|
61
|
+
raise ArgumentError.new(msg)
|
|
62
|
+
end
|
|
63
|
+
@scheme = implicit_scheme
|
|
64
|
+
@host = host
|
|
65
|
+
elsif !explicit_scheme.nil?
|
|
66
|
+
@scheme = explicit_scheme
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
if SCHEME_REGEX.match(@scheme).nil?
|
|
70
|
+
msg = sprintf("Kibana host scheme given is invalid, given value: '%s' - acceptable values: 'http', 'https'", @scheme)
|
|
71
|
+
raise ArgumentError.new(msg)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
if ssl_enabled && @scheme != "https"
|
|
75
|
+
@scheme = "https"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
@endpoint = "#{@scheme}://#{@host}"
|
|
79
|
+
|
|
80
|
+
@client = client || Manticore::Client.new(client_options)
|
|
54
81
|
@http_options = {:headers => {'Content-Type' => 'application/json'}}
|
|
55
82
|
username = @settings["var.kibana.username"]
|
|
56
83
|
if username
|
|
@@ -77,7 +104,7 @@ module LogStash module Modules class KibanaClient
|
|
|
77
104
|
end
|
|
78
105
|
|
|
79
106
|
def version_parts
|
|
80
|
-
@version.split(
|
|
107
|
+
@version.split(/[.-]/)
|
|
81
108
|
end
|
|
82
109
|
|
|
83
110
|
def host_settings
|
|
@@ -119,6 +146,6 @@ module LogStash module Modules class KibanaClient
|
|
|
119
146
|
end
|
|
120
147
|
|
|
121
148
|
def full_url(relative)
|
|
122
|
-
"#{@
|
|
149
|
+
"#{@endpoint}/#{relative}"
|
|
123
150
|
end
|
|
124
151
|
end end end
|
|
@@ -69,7 +69,7 @@ module LogStash module Modules class LogStashConfig
|
|
|
69
69
|
password = @settings["var.elasticsearch.password"]
|
|
70
70
|
lines = ["hosts => #{hosts}", "index => \"#{index}\""]
|
|
71
71
|
lines.push(user ? "user => \"#{user}\"" : nil)
|
|
72
|
-
lines.push(password ? "password => \"#{password}\"" : nil)
|
|
72
|
+
lines.push(password ? "password => \"#{password.value}\"" : nil)
|
|
73
73
|
lines.push(type_string ? "document_type => #{type_string}" : nil)
|
|
74
74
|
lines.push("ssl => #{@settings.fetch('var.elasticsearch.ssl.enabled', false)}")
|
|
75
75
|
if cacert = @settings["var.elasticsearch.ssl.certificate_authority"]
|
|
@@ -40,9 +40,10 @@ module LogStash module Modules module SettingsMerger
|
|
|
40
40
|
settings_copy = LogStash::Util.deep_clone(module_settings)
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
-
module_settings["var.kibana.scheme"] =
|
|
43
|
+
module_settings["var.kibana.scheme"] = cloud_id.kibana_scheme
|
|
44
44
|
module_settings["var.kibana.host"] = cloud_id.kibana_host
|
|
45
|
-
|
|
45
|
+
# elasticsearch client does not use scheme, it URI parses the host setting
|
|
46
|
+
module_settings["var.elasticsearch.hosts"] = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}"
|
|
46
47
|
unless cloud_auth.nil?
|
|
47
48
|
module_settings["var.elasticsearch.username"] = cloud_auth.username
|
|
48
49
|
module_settings["var.elasticsearch.password"] = cloud_auth.password
|
|
@@ -54,6 +55,11 @@ module LogStash module Modules module SettingsMerger
|
|
|
54
55
|
end
|
|
55
56
|
end
|
|
56
57
|
|
|
58
|
+
def merge_kibana_auth!(module_settings)
|
|
59
|
+
module_settings["var.kibana.username"] = module_settings["var.elasticsearch.username"] if module_settings["var.kibana.username"].nil?
|
|
60
|
+
module_settings["var.kibana.password"] = module_settings["var.elasticsearch.password"] if module_settings["var.kibana.password"].nil?
|
|
61
|
+
end
|
|
62
|
+
|
|
57
63
|
def format_module_settings(settings_before, settings_after)
|
|
58
64
|
output = []
|
|
59
65
|
output << "-------- Module Settings ---------"
|
data/lib/logstash/pipeline.rb
CHANGED
|
@@ -498,13 +498,9 @@ module LogStash; class Pipeline < BasePipeline
|
|
|
498
498
|
end
|
|
499
499
|
|
|
500
500
|
def filter_batch(batch)
|
|
501
|
-
batch.each do |
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
filter_func(event).each do |e|
|
|
505
|
-
#these are both original and generated events
|
|
506
|
-
batch.merge(e) unless e.cancelled?
|
|
507
|
-
end
|
|
501
|
+
filter_func(batch.to_a).each do |e|
|
|
502
|
+
#these are both original and generated events
|
|
503
|
+
batch.merge(e) unless e.cancelled?
|
|
508
504
|
end
|
|
509
505
|
@filter_queue_client.add_filtered_metrics(batch)
|
|
510
506
|
@events_filtered.increment(batch.size)
|
|
@@ -584,17 +580,12 @@ module LogStash; class Pipeline < BasePipeline
|
|
|
584
580
|
end
|
|
585
581
|
|
|
586
582
|
# otherwise, report error and restart
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
:stacktrace => e.backtrace.join("\n"))))
|
|
594
|
-
else
|
|
595
|
-
@logger.error(I18n.t("logstash.pipeline.worker-error",
|
|
596
|
-
default_logging_keys(:plugin => plugin.inspect, :error => e.message)))
|
|
597
|
-
end
|
|
583
|
+
@logger.error(I18n.t("logstash.pipeline.worker-error-debug",
|
|
584
|
+
default_logging_keys(
|
|
585
|
+
:plugin => plugin.inspect,
|
|
586
|
+
:error => e.message,
|
|
587
|
+
:exception => e.class,
|
|
588
|
+
:stacktrace => e.backtrace.join("\n"))))
|
|
598
589
|
|
|
599
590
|
# Assuming the failure that caused this exception is transient,
|
|
600
591
|
# let's sleep for a bit and execute #run again
|
|
@@ -667,7 +658,7 @@ module LogStash; class Pipeline < BasePipeline
|
|
|
667
658
|
def filter(event, &block)
|
|
668
659
|
maybe_setup_out_plugins
|
|
669
660
|
# filter_func returns all filtered events, including cancelled ones
|
|
670
|
-
filter_func(event).each {|e| block.call(e)}
|
|
661
|
+
filter_func([event]).each {|e| block.call(e)}
|
|
671
662
|
end
|
|
672
663
|
|
|
673
664
|
# perform filters flush and yield flushed event to the passed block
|
|
@@ -14,6 +14,7 @@ module LogStash module PipelineAction
|
|
|
14
14
|
def execute(agent, pipelines)
|
|
15
15
|
pipeline = pipelines[pipeline_id]
|
|
16
16
|
pipeline.shutdown { LogStash::ShutdownWatcher.start(pipeline) }
|
|
17
|
+
pipeline.thread.join
|
|
17
18
|
pipelines.delete(pipeline_id)
|
|
18
19
|
# If we reach this part of the code we have succeeded because
|
|
19
20
|
# the shutdown call will block.
|
data/lib/logstash/runner.rb
CHANGED
|
@@ -249,7 +249,10 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
|
249
249
|
java.lang.System.setProperty("ls.log.level", setting("log.level"))
|
|
250
250
|
unless java.lang.System.getProperty("log4j.configurationFile")
|
|
251
251
|
log4j_config_location = ::File.join(setting("path.settings"), "log4j2.properties")
|
|
252
|
-
|
|
252
|
+
|
|
253
|
+
# Windows safe way to produce a file: URI.
|
|
254
|
+
file_schema = "file://" + (LogStash::Environment.windows? ? "/" : "")
|
|
255
|
+
LogStash::Logging::Logger::reconfigure(URI.join(file_schema + File.absolute_path(log4j_config_location)).to_s)
|
|
253
256
|
end
|
|
254
257
|
# override log level that may have been introduced from a custom log4j config file
|
|
255
258
|
LogStash::Logging::Logger::configure_logging(setting("log.level"))
|
|
@@ -3,8 +3,26 @@ require "logstash/namespace"
|
|
|
3
3
|
require "base64"
|
|
4
4
|
|
|
5
5
|
module LogStash module Util class CloudSettingId
|
|
6
|
-
attr_reader :original, :decoded, :label, :elasticsearch_host, :kibana_host
|
|
7
6
|
|
|
7
|
+
def self.cloud_id_encode(*args)
|
|
8
|
+
Base64.urlsafe_encode64(args.join("$"))
|
|
9
|
+
end
|
|
10
|
+
DOT_SEPARATOR = "."
|
|
11
|
+
CLOUD_PORT = ":443"
|
|
12
|
+
|
|
13
|
+
attr_reader :original, :decoded, :label, :elasticsearch_host, :elasticsearch_scheme, :kibana_host, :kibana_scheme
|
|
14
|
+
|
|
15
|
+
# The constructor is expecting a 'cloud.id', a string in 2 variants.
|
|
16
|
+
# 1 part example: 'dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy'
|
|
17
|
+
# 2 part example: 'foobar:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy'
|
|
18
|
+
# The two part variant has a 'label' prepended with a colon separator. The label is not encoded.
|
|
19
|
+
# The 1 part (or second section of the 2 part variant) is base64 encoded.
|
|
20
|
+
# The original string before encoding has three segments separated by a dollar sign.
|
|
21
|
+
# e.g. 'us-east-1.aws.found.io$notareal$identifier'
|
|
22
|
+
# The first segment is the cloud base url, e.g. 'us-east-1.aws.found.io'
|
|
23
|
+
# The second segment is the elasticsearch host identifier, e.g. 'notareal'
|
|
24
|
+
# The third segment is the kibana host identifier, e.g. 'identifier'
|
|
25
|
+
# The 'cloud.id' value decoded into the #attr_reader ivars.
|
|
8
26
|
def initialize(value)
|
|
9
27
|
return if value.nil?
|
|
10
28
|
|
|
@@ -12,27 +30,43 @@ module LogStash module Util class CloudSettingId
|
|
|
12
30
|
raise ArgumentError.new("Cloud Id must be String. Received: #{value.class}")
|
|
13
31
|
end
|
|
14
32
|
@original = value
|
|
15
|
-
@label,
|
|
16
|
-
if
|
|
33
|
+
@label, colon, encoded = @original.partition(":")
|
|
34
|
+
if encoded.empty?
|
|
17
35
|
@decoded = Base64.urlsafe_decode64(@label) rescue ""
|
|
18
36
|
@label = ""
|
|
19
37
|
else
|
|
20
|
-
@decoded = Base64.urlsafe_decode64(
|
|
38
|
+
@decoded = Base64.urlsafe_decode64(encoded) rescue ""
|
|
21
39
|
end
|
|
40
|
+
|
|
41
|
+
@decoded = @decoded.encode(Encoding::UTF_8, :invalid => :replace, :undef => :replace)
|
|
42
|
+
|
|
22
43
|
unless @decoded.count("$") == 2
|
|
23
|
-
raise ArgumentError.new("Cloud Id does not decode. Received: \"#{@
|
|
44
|
+
raise ArgumentError.new("Cloud Id does not decode. You may need to enable Kibana in the Cloud UI. Received: \"#{@decoded}\".")
|
|
24
45
|
end
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
46
|
+
|
|
47
|
+
segments = @decoded.split("$")
|
|
48
|
+
if segments.any?(&:empty?)
|
|
49
|
+
raise ArgumentError.new("Cloud Id, after decoding, is invalid. Format: '<segment1>$<segment2>$<segment3>'. Received: \"#{@decoded}\".")
|
|
50
|
+
end
|
|
51
|
+
cloud_base = segments.shift
|
|
52
|
+
cloud_host = "#{DOT_SEPARATOR}#{cloud_base}#{CLOUD_PORT}"
|
|
53
|
+
|
|
54
|
+
@elasticsearch_host, @kibana_host = segments
|
|
55
|
+
if @elasticsearch_host == "undefined"
|
|
56
|
+
raise ArgumentError.new("Cloud Id, after decoding, elasticsearch segment is 'undefined', literally.")
|
|
57
|
+
end
|
|
58
|
+
@elasticsearch_scheme = "https"
|
|
59
|
+
@elasticsearch_host.concat(cloud_host)
|
|
60
|
+
|
|
61
|
+
if @kibana_host == "undefined"
|
|
62
|
+
raise ArgumentError.new("Cloud Id, after decoding, the kibana segment is 'undefined', literally. You may need to enable Kibana in the Cloud UI.")
|
|
28
63
|
end
|
|
29
|
-
|
|
30
|
-
@
|
|
31
|
-
@kibana_host = sprintf("%s.%s:443", kb_server, cloud_host)
|
|
64
|
+
@kibana_scheme = "https"
|
|
65
|
+
@kibana_host.concat(cloud_host)
|
|
32
66
|
end
|
|
33
67
|
|
|
34
68
|
def to_s
|
|
35
|
-
@
|
|
69
|
+
@decoded.to_s
|
|
36
70
|
end
|
|
37
71
|
|
|
38
72
|
def inspect
|