logstash-core 5.0.0.alpha2.snapshot2-java → 5.0.0.alpha3-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of logstash-core might be problematic. Click here for more details.

Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/lib/logstash-core/version.rb +1 -1
  3. data/lib/logstash/agent.rb +48 -31
  4. data/lib/logstash/api/{lib/helpers/app_helpers.rb → app_helpers.rb} +0 -0
  5. data/lib/logstash/api/command_factory.rb +34 -0
  6. data/lib/logstash/api/commands/base.rb +25 -0
  7. data/lib/logstash/api/commands/stats.rb +105 -0
  8. data/lib/logstash/api/commands/system/basicinfo_command.rb +23 -0
  9. data/lib/logstash/api/commands/system/plugins_command.rb +35 -0
  10. data/lib/logstash/api/modules/base.rb +43 -0
  11. data/lib/logstash/api/modules/node.rb +24 -0
  12. data/lib/logstash/api/modules/node_stats.rb +59 -0
  13. data/lib/logstash/api/modules/plugins.rb +15 -0
  14. data/lib/logstash/api/modules/root.rb +15 -0
  15. data/lib/logstash/api/modules/stats.rb +63 -0
  16. data/lib/logstash/api/rack_app.rb +33 -0
  17. data/lib/logstash/api/service.rb +73 -0
  18. data/lib/logstash/config/config_ast.rb +23 -18
  19. data/lib/logstash/config/loader.rb +4 -4
  20. data/lib/logstash/config/mixin.rb +10 -21
  21. data/lib/logstash/environment.rb +29 -0
  22. data/lib/logstash/filters/base.rb +2 -2
  23. data/lib/logstash/inputs/base.rb +2 -2
  24. data/lib/logstash/instrument/collector.rb +1 -1
  25. data/lib/logstash/instrument/metric_store.rb +11 -1
  26. data/lib/logstash/instrument/periodic_poller/base.rb +2 -0
  27. data/lib/logstash/instrument/periodic_poller/jvm.rb +47 -2
  28. data/lib/logstash/logging/json.rb +21 -0
  29. data/lib/logstash/output_delegator.rb +2 -2
  30. data/lib/logstash/patches/clamp.rb +69 -0
  31. data/lib/logstash/pipeline.rb +36 -69
  32. data/lib/logstash/plugin.rb +1 -1
  33. data/lib/logstash/runner.rb +171 -146
  34. data/lib/logstash/settings.rb +267 -0
  35. data/lib/logstash/util/decorators.rb +6 -6
  36. data/lib/logstash/util/java_version.rb +1 -10
  37. data/lib/logstash/util/thread_dump.rb +55 -0
  38. data/lib/logstash/util/worker_threads_default_printer.rb +2 -2
  39. data/lib/logstash/version.rb +1 -1
  40. data/lib/logstash/webserver.rb +15 -49
  41. data/locales/en.yml +22 -25
  42. data/logstash-core.gemspec +3 -3
  43. data/spec/api/lib/api/node_spec.rb +2 -2
  44. data/spec/api/lib/api/node_stats_spec.rb +32 -57
  45. data/spec/api/lib/api/plugins_spec.rb +3 -3
  46. data/spec/api/lib/api/root_spec.rb +2 -2
  47. data/spec/api/lib/api/support/resource_dsl_methods.rb +47 -0
  48. data/spec/api/lib/commands/stats.rb +47 -0
  49. data/spec/api/spec_helper.rb +21 -21
  50. data/spec/conditionals_spec.rb +113 -113
  51. data/spec/logstash/agent_spec.rb +85 -68
  52. data/spec/logstash/config/config_ast_spec.rb +4 -2
  53. data/spec/logstash/config/mixin_spec.rb +33 -7
  54. data/spec/logstash/filters/base_spec.rb +16 -16
  55. data/spec/logstash/inputs/base_spec.rb +8 -8
  56. data/spec/logstash/instrument/periodic_poller/jvm_spec.rb +45 -0
  57. data/spec/logstash/output_delegator_spec.rb +2 -0
  58. data/spec/logstash/pipeline_reporter_spec.rb +5 -1
  59. data/spec/logstash/pipeline_spec.rb +75 -30
  60. data/spec/logstash/plugin_spec.rb +2 -2
  61. data/spec/logstash/runner_spec.rb +112 -25
  62. data/spec/logstash/setting_spec.rb +130 -0
  63. data/spec/logstash/settings_spec.rb +62 -0
  64. metadata +37 -43
  65. data/lib/logstash/api/init.ru +0 -31
  66. data/lib/logstash/api/lib/app.rb +0 -40
  67. data/lib/logstash/api/lib/app/command.rb +0 -29
  68. data/lib/logstash/api/lib/app/command_factory.rb +0 -29
  69. data/lib/logstash/api/lib/app/commands/stats/events_command.rb +0 -13
  70. data/lib/logstash/api/lib/app/commands/stats/hotthreads_command.rb +0 -120
  71. data/lib/logstash/api/lib/app/commands/stats/memory_command.rb +0 -25
  72. data/lib/logstash/api/lib/app/commands/system/basicinfo_command.rb +0 -15
  73. data/lib/logstash/api/lib/app/commands/system/plugins_command.rb +0 -28
  74. data/lib/logstash/api/lib/app/modules/node.rb +0 -25
  75. data/lib/logstash/api/lib/app/modules/node_stats.rb +0 -51
  76. data/lib/logstash/api/lib/app/modules/plugins.rb +0 -15
  77. data/lib/logstash/api/lib/app/modules/stats.rb +0 -21
  78. data/lib/logstash/api/lib/app/root.rb +0 -13
  79. data/lib/logstash/api/lib/app/service.rb +0 -61
  80. data/lib/logstash/api/lib/app/stats.rb +0 -56
  81. data/lib/logstash/util/defaults_printer.rb +0 -31
  82. data/spec/api/lib/api/stats_spec.rb +0 -19
  83. data/spec/api/lib/commands/events_spec.rb +0 -17
  84. data/spec/api/lib/commands/jvm_spec.rb +0 -45
  85. data/spec/logstash/util/defaults_printer_spec.rb +0 -50
  86. data/spec/logstash/util/worker_threads_default_printer_spec.rb +0 -45
@@ -1,13 +0,0 @@
1
- # encoding: utf-8
2
- require "app"
3
-
4
- module LogStash::Api
5
- class Root < BaseApp
6
-
7
- get "/" do
8
- command = factory.build(:system_basic_info)
9
- respond_with command.run
10
- end
11
-
12
- end
13
- end
@@ -1,61 +0,0 @@
1
- # encoding: utf-8
2
- require "logstash/instrument/collector"
3
- require "logstash/util/loggable"
4
-
5
- class LogStash::Api::Service
6
-
7
- include Singleton
8
- include LogStash::Util::Loggable
9
-
10
- def initialize
11
- @snapshot_rotation_mutex = Mutex.new
12
- @snapshot = nil
13
- logger.debug("[api-service] start") if logger.debug?
14
- LogStash::Instrument::Collector.instance.add_observer(self)
15
- end
16
-
17
- def stop
18
- logger.debug("[api-service] stop") if logger.debug?
19
- LogStash::Instrument::Collector.instance.delete_observer(self)
20
- end
21
-
22
- def agent
23
- LogStash::Instrument::Collector.instance.agent
24
- end
25
-
26
- def started?
27
- !@snapshot.nil? && has_counters?
28
- end
29
-
30
- def update(snapshot)
31
- logger.debug("[api-service] snapshot received", :snapshot => snapshot) if logger.debug?
32
- if @snapshot_rotation_mutex.try_lock
33
- @snapshot = snapshot
34
- @snapshot_rotation_mutex.unlock
35
- end
36
- end
37
-
38
- def get(key)
39
- metric_store = @snapshot.metric_store
40
- if key == :jvm_memory_stats
41
- data = metric_store.get_with_path("jvm/memory")[:jvm][:memory]
42
- else
43
- data = metric_store.get_with_path("stats/events")
44
- end
45
- LogStash::Json.dump(data)
46
- end
47
-
48
- private
49
-
50
- def has_counters?
51
- (["LogStash::Instrument::MetricType::Counter", "LogStash::Instrument::MetricType::Gauge"] - metric_types).empty?
52
- end
53
-
54
- def metric_types
55
- types = []
56
- @snapshot_rotation_mutex.synchronize do
57
- types = @snapshot.metric_store.all.map { |t| t.class.to_s }
58
- end
59
- return types
60
- end
61
- end
@@ -1,56 +0,0 @@
1
- # encoding: utf-8
2
- require "app"
3
- require "app/stats/events_command"
4
- require "app/stats/hotthreads_command"
5
-
6
- module LogStash::Api
7
- class Stats < BaseApp
8
-
9
- helpers AppHelpers
10
-
11
-
12
- # Global _stats resource where all information is
13
- # retrieved and show
14
- get "/" do
15
- events_command = factory.build(:events_command)
16
- memory_command = factory.build(:memory_command)
17
- payload = {
18
- :events => events_command.run,
19
- :jvm => { :memory => memory_command.run }
20
- }
21
- respond_with payload
22
- end
23
-
24
- # Show all events stats information
25
- # (for ingested, emitted, dropped)
26
- # - #events since startup
27
- # - #data (bytes) since startup
28
- # - events/s
29
- # - bytes/s
30
- # - dropped events/s
31
- # - events in the pipeline
32
- get "/events" do
33
- command = factory.build(:events_command)
34
- respond_with({ :events => command.run })
35
- end
36
-
37
- # return hot threads information
38
- get "/jvm/hot_threads" do
39
- top_threads_count = params["threads"] || 3
40
- ignore_idle_threads = params["ignore_idle_threads"] || true
41
- options = {
42
- :threads => top_threads_count.to_i,
43
- :ignore_idle_threads => as_boolean(ignore_idle_threads)
44
- }
45
- command = factory.build(:hot_threads_command)
46
- respond_with(command.run(options), :string)
47
- end
48
-
49
- # return hot threads information
50
- get "/jvm/memory" do
51
- command = factory.build(:memory_command)
52
- respond_with({ :memory => command.run })
53
- end
54
-
55
- end
56
- end
@@ -1,31 +0,0 @@
1
- # encoding: utf-8
2
- require "logstash/namespace"
3
- require "logstash/util"
4
- require "logstash/util/worker_threads_default_printer"
5
-
6
-
7
- # This class exists to format the settings for defaults used
8
- module LogStash module Util class DefaultsPrinter
9
- def self.print(settings)
10
- new(settings).print
11
- end
12
-
13
- def initialize(settings)
14
- @settings = settings
15
- @printers = [workers]
16
- end
17
-
18
- def print
19
- collector = []
20
- @printers.each do |printer|
21
- printer.visit(collector)
22
- end
23
- "Settings: " + collector.join(', ')
24
- end
25
-
26
- private
27
-
28
- def workers
29
- WorkerThreadsDefaultPrinter.new(@settings)
30
- end
31
- end end end
@@ -1,19 +0,0 @@
1
- # encoding: utf-8
2
- require_relative "../../spec_helper"
3
- require "sinatra"
4
- require "app/modules/stats"
5
-
6
- describe LogStash::Api::Stats do
7
-
8
- include Rack::Test::Methods
9
-
10
- def app()
11
- described_class
12
- end
13
-
14
- it "respond to the jvm resource" do
15
- do_request { get "/jvm" }
16
- expect(last_response).to be_ok
17
- end
18
-
19
- end
@@ -1,17 +0,0 @@
1
- # encoding: utf-8
2
- require_relative "../../spec_helper"
3
- require "app/commands/stats/events_command"
4
-
5
- describe LogStash::Api::StatsEventsCommand do
6
-
7
- context "#schema" do
8
-
9
- let(:report) do
10
- do_request { subject.run }
11
- end
12
-
13
- it "return events information" do
14
- expect(report).to include("in", "filtered", "out")
15
- end
16
- end
17
- end
@@ -1,45 +0,0 @@
1
- # encoding: utf-8
2
- require_relative "../../spec_helper"
3
- require "app/commands/stats/hotthreads_command"
4
- require "app/commands/stats/memory_command"
5
-
6
- describe "JVM stats" do
7
-
8
- describe LogStash::Api::HotThreadsCommand do
9
-
10
- let(:report) do
11
- do_request { subject.run }
12
- end
13
-
14
- context "#schema" do
15
- it "return hot threads information" do
16
- report = do_request { subject.run }
17
- expect(report.to_s).not_to be_empty
18
- end
19
-
20
- end
21
- end
22
-
23
- describe LogStash::Api::JvmMemoryCommand do
24
-
25
- context "#schema" do
26
-
27
- let(:report) do
28
- do_request { subject.run }
29
- end
30
-
31
- it "return hot threads information" do
32
- expect(report).not_to be_empty
33
- end
34
-
35
- it "return heap information" do
36
- expect(report.keys).to include(:heap_used_in_bytes)
37
- end
38
-
39
- it "return non heap information" do
40
- expect(report.keys).to include(:non_heap_used_in_bytes)
41
- end
42
-
43
- end
44
- end
45
- end
@@ -1,50 +0,0 @@
1
- # encoding: utf-8
2
- require "spec_helper"
3
- require "logstash/util/defaults_printer"
4
-
5
- describe LogStash::Util::DefaultsPrinter do
6
- shared_examples "a defaults printer" do
7
- it 'the .print method returns a defaults description' do
8
- expect(actual_block.call).to eq(expected)
9
- end
10
- end
11
-
12
- let(:workers) { 1 }
13
- let(:expected) { "Settings: User set pipeline workers: #{workers}" }
14
- let(:settings) { {} }
15
-
16
- describe 'class methods API' do
17
- let(:actual_block) do
18
- -> {described_class.print(settings)}
19
- end
20
-
21
- context 'when the settings hash is empty' do
22
- let(:expected) { "Settings: " }
23
- it_behaves_like "a defaults printer"
24
- end
25
-
26
- context 'when the settings hash has content' do
27
- let(:worker_queue) { 42 }
28
- let(:settings) { {:pipeline_workers => workers} }
29
- it_behaves_like "a defaults printer"
30
- end
31
- end
32
-
33
- describe 'instance method API' do
34
- let(:actual_block) do
35
- -> {described_class.new(settings).print}
36
- end
37
-
38
- context 'when the settings hash is empty' do
39
- let(:expected) { "Settings: " }
40
- it_behaves_like "a defaults printer"
41
- end
42
-
43
- context 'when the settings hash has content' do
44
- let(:workers) { 13 }
45
- let(:settings) { {:pipeline_workers => workers} }
46
-
47
- it_behaves_like "a defaults printer"
48
- end
49
- end
50
- end
@@ -1,45 +0,0 @@
1
- # encoding: utf-8
2
- require "spec_helper"
3
- require "logstash/util/worker_threads_default_printer"
4
-
5
- describe LogStash::Util::WorkerThreadsDefaultPrinter do
6
- let(:settings) { {} }
7
- let(:collector) { [] }
8
-
9
- subject { described_class.new(settings) }
10
-
11
- before { subject.visit(collector) }
12
-
13
- describe "the #visit method" do
14
- context 'when the settings hash is empty' do
15
- it 'adds nothing to the collector' do
16
- subject.visit(collector)
17
- expect(collector).to eq([])
18
- end
19
- end
20
-
21
- context 'when the settings hash has both user and default content' do
22
- let(:settings) { {:pipeline_workers => 42, :default_pipeline_workers => 5} }
23
-
24
- it 'adds two strings' do
25
- expect(collector).to eq(["User set pipeline workers: 42", "Default pipeline workers: 5"])
26
- end
27
- end
28
-
29
- context 'when the settings hash has only user content' do
30
- let(:settings) { {:pipeline_workers => 42} }
31
-
32
- it 'adds a string with user set pipeline workers' do
33
- expect(collector.first).to eq("User set pipeline workers: 42")
34
- end
35
- end
36
-
37
- context 'when the settings hash has only default content' do
38
- let(:settings) { {:default_pipeline_workers => 5} }
39
-
40
- it 'adds a string with default pipeline workers' do
41
- expect(collector.first).to eq("Default pipeline workers: 5")
42
- end
43
- end
44
- end
45
- end