logstash-core 5.0.0.alpha3.snapshot8-java → 5.0.0.alpha4.snapshot1-java
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of logstash-core might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/lib/logstash-core/version.rb +1 -1
- data/lib/logstash/agent.rb +2 -1
- data/lib/logstash/api/modules/base.rb +6 -11
- data/lib/logstash/api/modules/node.rb +2 -0
- data/lib/logstash/api/rack_app.rb +77 -1
- data/lib/logstash/api/service.rb +1 -1
- data/lib/logstash/environment.rb +1 -0
- data/lib/logstash/runner.rb +9 -1
- data/lib/logstash/version.rb +1 -1
- data/lib/logstash/webserver.rb +5 -4
- data/logstash-core.gemspec +1 -1
- data/spec/api/lib/rack_app_spec.rb +88 -0
- data/spec/api/spec_helper.rb +6 -3
- data/spec/logstash/agent_spec.rb +1 -1
- metadata +6 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 34ea2c9c79b6a5828714ad66dc7ce558d7952280
|
4
|
+
data.tar.gz: 76dcab330cb125653bfe94ef514f82f80a08a974
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7274fedd8a22a9518fec19299c53295243832c17a376c85f59c2ea6dfe85c139d2161b0f2d5e6854e348824c38b3427fa60ea4d0e7281a6232bfee628848e45f
|
7
|
+
data.tar.gz: f1432ba6f7d34a38c3fbf57a7ddc6f02fefc6916f1349775a3254f75d720735101ad24da81971f81edbf42cd091c783959b2e46e2e2502162e0c2f10d39742ec
|
data/lib/logstash/agent.rb
CHANGED
@@ -38,6 +38,7 @@ class LogStash::Agent
|
|
38
38
|
@node_name = setting("node.name")
|
39
39
|
@http_host = setting("http.host")
|
40
40
|
@http_port = setting("http.port")
|
41
|
+
@http_environment = setting("http.environment")
|
41
42
|
|
42
43
|
@config_loader = LogStash::Config::Loader.new(@logger)
|
43
44
|
@reload_interval = setting("config.reload.interval")
|
@@ -129,7 +130,7 @@ class LogStash::Agent
|
|
129
130
|
|
130
131
|
private
|
131
132
|
def start_webserver
|
132
|
-
options = {:http_host => @http_host, :http_port => @http_port }
|
133
|
+
options = {:http_host => @http_host, :http_port => @http_port, :http_environment => @http_environment }
|
133
134
|
@webserver = LogStash::WebServer.new(@logger, options)
|
134
135
|
Thread.new(@webserver) do |webserver|
|
135
136
|
LogStash::Util.set_thread_name("Api Webserver")
|
@@ -7,15 +7,16 @@ module LogStash
|
|
7
7
|
class Base < ::Sinatra::Base
|
8
8
|
helpers AppHelpers
|
9
9
|
|
10
|
+
# These options never change
|
11
|
+
# Sinatra isn't good at letting you change internal settings at runtime
|
12
|
+
# which is a requirement. We always propagate errors up and catch them
|
13
|
+
# in a custom rack handler in the RackApp class
|
10
14
|
set :environment, :production
|
15
|
+
set :raise_errors, true
|
16
|
+
set :show_exceptions, false
|
11
17
|
|
12
18
|
attr_reader :factory
|
13
19
|
|
14
|
-
if settings.environment != :production
|
15
|
-
set :raise_errors, true
|
16
|
-
set :show_exceptions, :after_handler
|
17
|
-
end
|
18
|
-
|
19
20
|
include LogStash::Util::Loggable
|
20
21
|
|
21
22
|
helpers AppHelpers
|
@@ -31,12 +32,6 @@ module LogStash
|
|
31
32
|
text = as == :string ? "" : {}
|
32
33
|
respond_with(text, :as => as)
|
33
34
|
end
|
34
|
-
|
35
|
-
error do
|
36
|
-
e = env['sinatra.error']
|
37
|
-
logger.error(e.message, :url => request.url, :ip => request.ip, :params => request.params, :class => e.class.name, :backtrace => e.backtrace)
|
38
|
-
end
|
39
|
-
|
40
35
|
end
|
41
36
|
end
|
42
37
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require "sinatra"
|
2
|
+
require "rack"
|
1
3
|
require "logstash/api/modules/base"
|
2
4
|
require "logstash/api/modules/node"
|
3
5
|
require "logstash/api/modules/node_stats"
|
@@ -8,9 +10,83 @@ require "logstash/api/modules/stats"
|
|
8
10
|
module LogStash
|
9
11
|
module Api
|
10
12
|
module RackApp
|
11
|
-
|
13
|
+
# Cabin is not compatible with CommonLogger, and this gives us more control anyway
|
14
|
+
METADATA_FIELDS = [:request_method, :path_info, :query_string, :http_version, :http_accept].freeze
|
15
|
+
def self.log_metadata(status, env)
|
16
|
+
METADATA_FIELDS.reduce({:status => status}) do |acc, field|
|
17
|
+
acc[field] = env[field.to_s.upcase]
|
18
|
+
acc
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
class ApiLogger
|
23
|
+
LOG_MESSAGE = "API HTTP Request".freeze
|
24
|
+
|
25
|
+
def initialize(app, logger)
|
26
|
+
@app = app
|
27
|
+
@logger = logger
|
28
|
+
end
|
29
|
+
|
30
|
+
def call(env)
|
31
|
+
res = @app.call(env)
|
32
|
+
status, headers, body = res
|
33
|
+
|
34
|
+
if fatal_error?(status)
|
35
|
+
@logger.warn? && @logger.warn(LOG_MESSAGE, RackApp.log_metadata(status, env))
|
36
|
+
else
|
37
|
+
@logger.info? && @logger.info(LOG_MESSAGE, RackApp.log_metadata(status, env))
|
38
|
+
end
|
39
|
+
|
40
|
+
res
|
41
|
+
end
|
42
|
+
|
43
|
+
def fatal_error?(status)
|
44
|
+
status >= 500 && status < 600
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
class ApiErrorHandler
|
49
|
+
LOG_MESSAGE = "Internal API server error".freeze
|
50
|
+
|
51
|
+
def initialize(app, logger)
|
52
|
+
@app = app
|
53
|
+
@logger = logger
|
54
|
+
end
|
55
|
+
|
56
|
+
def call(env)
|
57
|
+
@app.call(env)
|
58
|
+
rescue => e
|
59
|
+
body = RackApp.log_metadata(500, env).
|
60
|
+
merge({
|
61
|
+
:error => "Unexpected Internal Error",
|
62
|
+
:class => e.class.name,
|
63
|
+
:message => e.message,
|
64
|
+
:backtrace => e.backtrace
|
65
|
+
})
|
66
|
+
|
67
|
+
@logger.error(LOG_MESSAGE, body)
|
68
|
+
|
69
|
+
[500,
|
70
|
+
{'Content-Type' => 'application/json'},
|
71
|
+
[LogStash::Json.dump(body)]
|
72
|
+
]
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.app(logger, environment)
|
12
77
|
namespaces = rack_namespaces
|
13
78
|
Rack::Builder.new do
|
79
|
+
# Custom logger object. Rack CommonLogger does not work with cabin
|
80
|
+
use ApiLogger, logger
|
81
|
+
|
82
|
+
# In test env we want errors to propogate up the chain
|
83
|
+
# so we get easy to understand test failures.
|
84
|
+
# In production / dev we don't want a bad API endpoint
|
85
|
+
# to crash the process
|
86
|
+
if environment != "test"
|
87
|
+
use ApiErrorHandler, logger
|
88
|
+
end
|
89
|
+
|
14
90
|
run LogStash::Api::Modules::Root
|
15
91
|
namespaces.each_pair do |namespace, app|
|
16
92
|
map(namespace) do
|
data/lib/logstash/api/service.rb
CHANGED
data/lib/logstash/environment.rb
CHANGED
@@ -29,6 +29,7 @@ module LogStash
|
|
29
29
|
Setting::String.new("log.format", "plain", true, ["json", "plain"]),
|
30
30
|
Setting::String.new("http.host", "127.0.0.1"),
|
31
31
|
Setting::Port.new("http.port", 9600),
|
32
|
+
Setting::String.new("http.environment", "production"),
|
32
33
|
].each {|setting| SETTINGS.register(setting) }
|
33
34
|
|
34
35
|
module Environment
|
data/lib/logstash/runner.rb
CHANGED
@@ -138,7 +138,15 @@ class LogStash::Runner < Clamp::StrictCommand
|
|
138
138
|
|
139
139
|
@settings.set("path.settings", settings_path) if settings_path
|
140
140
|
|
141
|
-
|
141
|
+
begin
|
142
|
+
LogStash::SETTINGS.from_yaml(LogStash::SETTINGS.get("path.settings"))
|
143
|
+
rescue => e
|
144
|
+
@logger.subscribe(STDOUT)
|
145
|
+
@logger.warn("Logstash has a new settings file which defines start up time settings. This file is typically located in $LS_HOME/config or /etc/logstash. If you installed Logstash through a package and are starting it manually please specify the location to this settings file by passing in \"--path.settings=/path/..\" in the command line options")
|
146
|
+
@logger.fatal("Failed to load settings file from \"path.settings\". Aborting...", "path.settings" => LogStash::SETTINGS.get("path.settings"), "exception" => e.class, "message" => e.message)
|
147
|
+
exit(-1)
|
148
|
+
end
|
149
|
+
|
142
150
|
super(*[args])
|
143
151
|
end
|
144
152
|
|
data/lib/logstash/version.rb
CHANGED
data/lib/logstash/webserver.rb
CHANGED
@@ -1,25 +1,25 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "puma"
|
3
3
|
require "puma/server"
|
4
|
-
require "sinatra"
|
5
|
-
require "rack"
|
6
4
|
require "logstash/api/rack_app"
|
7
5
|
|
8
6
|
module LogStash
|
9
7
|
class WebServer
|
10
8
|
extend Forwardable
|
11
9
|
|
12
|
-
attr_reader :logger, :status, :config, :options, :cli_options, :runner, :binder, :events, :http_host, :http_port
|
10
|
+
attr_reader :logger, :status, :config, :options, :cli_options, :runner, :binder, :events, :http_host, :http_port, :http_environment
|
13
11
|
|
14
12
|
def_delegator :@runner, :stats
|
15
13
|
|
16
14
|
DEFAULT_HOST = "127.0.0.1".freeze
|
17
15
|
DEFAULT_PORT = 9600.freeze
|
16
|
+
DEFAULT_ENVIRONMENT = 'production'.freeze
|
18
17
|
|
19
18
|
def initialize(logger, options={})
|
20
19
|
@logger = logger
|
21
20
|
@http_host = options[:http_host] || DEFAULT_HOST
|
22
21
|
@http_port = options[:http_port] || DEFAULT_PORT
|
22
|
+
@http_environment = options[:http_environment] || DEFAULT_ENVIRONMENT
|
23
23
|
@options = {}
|
24
24
|
@cli_options = options.merge({ :rackup => ::File.join(::File.dirname(__FILE__), "api", "init.ru"),
|
25
25
|
:binds => ["tcp://#{http_host}:#{http_port}"],
|
@@ -37,7 +37,8 @@ module LogStash
|
|
37
37
|
|
38
38
|
stop # Just in case
|
39
39
|
|
40
|
-
|
40
|
+
app = LogStash::Api::RackApp.app(logger, http_environment)
|
41
|
+
@server = ::Puma::Server.new(app)
|
41
42
|
@server.add_tcp_listener(http_host, http_port)
|
42
43
|
|
43
44
|
@server.run.join
|
data/logstash-core.gemspec
CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
|
|
17
17
|
gem.require_paths = ["lib"]
|
18
18
|
gem.version = LOGSTASH_CORE_VERSION.gsub(/-/, '.')
|
19
19
|
|
20
|
-
gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.
|
20
|
+
gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha4.snapshot1"
|
21
21
|
|
22
22
|
gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
|
23
23
|
gem.add_runtime_dependency "pry", "~> 0.10.1" #(Ruby license)
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require "logstash/api/rack_app"
|
2
|
+
require "rack/test"
|
3
|
+
|
4
|
+
describe LogStash::Api::RackApp do
|
5
|
+
include Rack::Test::Methods
|
6
|
+
|
7
|
+
class DummyApp
|
8
|
+
class RaisedError < StandardError; end
|
9
|
+
|
10
|
+
def call(env)
|
11
|
+
case env["PATH_INFO"]
|
12
|
+
when "/good-page"
|
13
|
+
[200, {}, ["200 OK"]]
|
14
|
+
when "/service-unavailable"
|
15
|
+
[503, {}, ["503 service unavailable"]]
|
16
|
+
when "/raise-error"
|
17
|
+
raise RaisedError, "Error raised"
|
18
|
+
else
|
19
|
+
[404, {}, ["404 Page not found"]]
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
let(:logger) { Cabin::Channel.get }
|
25
|
+
|
26
|
+
describe LogStash::Api::RackApp::ApiErrorHandler do
|
27
|
+
let(:app) do
|
28
|
+
# Scoping in rack builder is weird, these need to be locals
|
29
|
+
rack_class = described_class
|
30
|
+
rack_logger = logger
|
31
|
+
Rack::Builder.new do
|
32
|
+
use rack_class, rack_logger
|
33
|
+
run DummyApp.new
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
it "should let good requests through as normal" do
|
38
|
+
get "/good-page"
|
39
|
+
expect(last_response).to be_ok
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should let through 5xx codes" do
|
43
|
+
get "/service-unavailable"
|
44
|
+
expect(last_response.status).to eql(503)
|
45
|
+
end
|
46
|
+
|
47
|
+
describe "raised exceptions" do
|
48
|
+
before do
|
49
|
+
allow(logger).to receive(:error).with(any_args)
|
50
|
+
get "/raise-error"
|
51
|
+
end
|
52
|
+
|
53
|
+
it "should return a 500 error" do
|
54
|
+
expect(last_response.status).to eql(500)
|
55
|
+
end
|
56
|
+
|
57
|
+
it "should return valid JSON" do
|
58
|
+
expect { LogStash::Json.load(last_response.body) }.not_to raise_error
|
59
|
+
end
|
60
|
+
|
61
|
+
it "should log the error" do
|
62
|
+
expect(logger).to have_received(:error).with(LogStash::Api::RackApp::ApiErrorHandler::LOG_MESSAGE, anything).once
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
describe LogStash::Api::RackApp::ApiLogger do
|
68
|
+
let(:app) do
|
69
|
+
# Scoping in rack builder is weird, these need to be locals
|
70
|
+
rack_class = described_class
|
71
|
+
rack_logger = logger
|
72
|
+
Rack::Builder.new do
|
73
|
+
use rack_class, rack_logger
|
74
|
+
run DummyApp.new
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
it "should log good requests as info" do
|
79
|
+
expect(logger).to receive(:info).with(LogStash::Api::RackApp::ApiLogger::LOG_MESSAGE, anything).once
|
80
|
+
get "/good-page"
|
81
|
+
end
|
82
|
+
|
83
|
+
it "should log 5xx requests as warnings" do
|
84
|
+
expect(logger).to receive(:warn).with(LogStash::Api::RackApp::ApiLogger::LOG_MESSAGE, anything).once
|
85
|
+
get "/service-unavailable"
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
data/spec/api/spec_helper.rb
CHANGED
@@ -46,6 +46,7 @@ class LogStashRunner
|
|
46
46
|
"log.level" => "debug",
|
47
47
|
"node.name" => "test_agent",
|
48
48
|
"http.port" => rand(9600..9700),
|
49
|
+
"http.environment" => "test",
|
49
50
|
"config.string" => @config_str,
|
50
51
|
"pipeline.batch.size" => 1,
|
51
52
|
"pipeline.workers" => 1
|
@@ -60,7 +61,8 @@ class LogStashRunner
|
|
60
61
|
@runner = Thread.new(agent) do |_agent|
|
61
62
|
_agent.execute
|
62
63
|
end
|
63
|
-
|
64
|
+
|
65
|
+
wait_until_ready
|
64
66
|
end
|
65
67
|
|
66
68
|
def stop
|
@@ -71,8 +73,9 @@ class LogStashRunner
|
|
71
73
|
|
72
74
|
private
|
73
75
|
|
74
|
-
def
|
75
|
-
|
76
|
+
def wait_until_ready
|
77
|
+
# Wait until the service and pipeline have started
|
78
|
+
while !(LogStash::Api::Service.instance.started? && agent.pipelines["main"].running?) do
|
76
79
|
sleep 0.5
|
77
80
|
end
|
78
81
|
end
|
data/spec/logstash/agent_spec.rb
CHANGED
@@ -373,7 +373,7 @@ describe LogStash::Agent do
|
|
373
373
|
@t.join
|
374
374
|
end
|
375
375
|
|
376
|
-
|
376
|
+
it "resets the metric collector" do
|
377
377
|
# We know that the store has more events coming in.
|
378
378
|
sleep(0.01) while dummy_output.events.size < new_config_generator_counter
|
379
379
|
snapshot = LogStash::Instrument::Collector.instance.snapshot_metric
|
metadata
CHANGED
@@ -1,21 +1,21 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-core
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.0.0.
|
4
|
+
version: 5.0.0.alpha4.snapshot1
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-06-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
15
15
|
requirements:
|
16
16
|
- - '='
|
17
17
|
- !ruby/object:Gem::Version
|
18
|
-
version: 5.0.0.
|
18
|
+
version: 5.0.0.alpha4.snapshot1
|
19
19
|
name: logstash-core-event-java
|
20
20
|
prerelease: false
|
21
21
|
type: :runtime
|
@@ -23,7 +23,7 @@ dependencies:
|
|
23
23
|
requirements:
|
24
24
|
- - '='
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 5.0.0.
|
26
|
+
version: 5.0.0.alpha4.snapshot1
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
29
29
|
requirements:
|
@@ -388,6 +388,7 @@ files:
|
|
388
388
|
- spec/api/lib/api/root_spec.rb
|
389
389
|
- spec/api/lib/api/support/resource_dsl_methods.rb
|
390
390
|
- spec/api/lib/commands/stats.rb
|
391
|
+
- spec/api/lib/rack_app_spec.rb
|
391
392
|
- spec/api/spec_helper.rb
|
392
393
|
- spec/conditionals_spec.rb
|
393
394
|
- spec/logstash/agent_spec.rb
|
@@ -464,6 +465,7 @@ test_files:
|
|
464
465
|
- spec/api/lib/api/root_spec.rb
|
465
466
|
- spec/api/lib/api/support/resource_dsl_methods.rb
|
466
467
|
- spec/api/lib/commands/stats.rb
|
468
|
+
- spec/api/lib/rack_app_spec.rb
|
467
469
|
- spec/api/spec_helper.rb
|
468
470
|
- spec/conditionals_spec.rb
|
469
471
|
- spec/logstash/agent_spec.rb
|