logstash-core 5.0.0.alpha4.snapshot2-java → 5.0.0.alpha4.snapshot3-java

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of logstash-core might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: bd0ebd412460fd45c6abdef1ae4a21b75f24b6c5
4
- data.tar.gz: 4addbb53a17c98501b27f8f8fa33f8475a44bca4
3
+ metadata.gz: c90549a9a03e8d32ee0eb8146780ad53609abc84
4
+ data.tar.gz: 14e872af82cfa7cb8dbfc1b5bbce6e8fcd4e8031
5
5
  SHA512:
6
- metadata.gz: b602623cc49f59c90e0af86c1d43e90e2b611aef12e44d5590a8536c3efd0cf06deab5644f49477fdcc2a4d7615340bef5e4fcac613ddfff9e6e792594511958
7
- data.tar.gz: 7dcc49def008f07dae78a07e78c6ba8fa7e2942e30341637aa873068bf71cbc3ccb82557299b915e79934f523eb469d98a4dfaaeb39abcffe715efe66057bfe3
6
+ metadata.gz: c666d73e6f87dd8de677888232066d807a2878105a40c023ad0a9d9f221eb9fea16ddf8694a3cd692c816ae92537c23a034e54162c71505990b53ba7a710b7d8
7
+ data.tar.gz: 45d360a8b9896e2ff744d0e84efce8126c33d5c4c6ce1d71a8fdbf89a61ee6940a2a80843cb7613d3eaa8baab677bf7c717009ac93a7c55b559678c840cc37f1
@@ -5,4 +5,4 @@
5
5
  # Note to authors: this should not include dashes because 'gem' barfs if
6
6
  # you include a dash in the version string.
7
7
 
8
- LOGSTASH_CORE_VERSION = "5.0.0-alpha4.snapshot2"
8
+ LOGSTASH_CORE_VERSION = "5.0.0-alpha4.snapshot3"
@@ -33,6 +33,11 @@ module LogStash
33
33
  )
34
34
  end
35
35
 
36
+ def pipeline
37
+ stats = service.get_shallow(:stats, :pipelines)
38
+ PluginsStats.report(stats)
39
+ end
40
+
36
41
  def memory
37
42
  memory = service.get_shallow(:jvm, :memory)
38
43
  {
@@ -49,7 +54,98 @@ module LogStash
49
54
  acc
50
55
  end
51
56
  }
52
- end
57
+ end
58
+
59
+ def hot_threads(options={})
60
+ HotThreadsReport.new(self, options)
61
+ end
62
+
63
+ class HotThreadsReport
64
+ HOT_THREADS_STACK_TRACES_SIZE_DEFAULT = 10.freeze
65
+
66
+ def initialize(cmd, options)
67
+ @cmd = cmd
68
+ filter = { :stacktrace_size => options.fetch(:stacktrace_size, HOT_THREADS_STACK_TRACES_SIZE_DEFAULT) }
69
+ jr_dump = JRMonitor.threads.generate(filter)
70
+ @thread_dump = ::LogStash::Util::ThreadDump.new(options.merge(:dump => jr_dump))
71
+ end
72
+
73
+ def to_s
74
+ hash = to_hash
75
+ report = "#{I18n.t("logstash.web_api.hot_threads.title", :hostname => hash[:hostname], :time => hash[:time], :top_count => @thread_dump.top_count )} \n"
76
+ report << '=' * 80
77
+ report << "\n"
78
+ hash[:threads].each do |thread|
79
+ thread_report = ""
80
+ thread_report = "#{I18n.t("logstash.web_api.
81
+ hot_threads.thread_title", :percent_of_cpu_time => thread[:percent_of_cpu_time], :thread_state => thread[:state], :thread_name => thread[:name])} \n"
82
+ thread_report = "#{thread[:percent_of_cpu_time]} % of of cpu usage by #{thread[:state]} thread named '#{thread[:name]}'\n"
83
+ thread_report << "#{thread[:path]}\n" if thread[:path]
84
+ thread[:traces].each do |trace|
85
+ thread_report << "\t#{trace}\n"
86
+ end
87
+ report << thread_report
88
+ report << '-' * 80
89
+ report << "\n"
90
+ end
91
+ report
92
+ end
93
+
94
+ def to_hash
95
+ hash = { :hostname => @cmd.hostname, :time => Time.now.iso8601, :busiest_threads => @thread_dump.top_count, :threads => [] }
96
+ @thread_dump.each do |thread_name, _hash|
97
+ thread_name, thread_path = _hash["thread.name"].split(": ")
98
+ thread = { :name => thread_name,
99
+ :percent_of_cpu_time => cpu_time_as_percent(_hash),
100
+ :state => _hash["thread.state"]
101
+ }
102
+ thread[:path] = thread_path if thread_path
103
+ traces = []
104
+ _hash["thread.stacktrace"].each do |trace|
105
+ traces << trace
106
+ end
107
+ thread[:traces] = traces unless traces.empty?
108
+ hash[:threads] << thread
109
+ end
110
+ hash
111
+ end
112
+
113
+ def cpu_time_as_percent(hash)
114
+ (((cpu_time(hash) / @cmd.uptime * 1.0)*10000).to_i)/100.0
115
+ end
116
+
117
+ def cpu_time(hash)
118
+ hash["cpu.time"] / 1000000.0
119
+ end
120
+ end # class HotThreadsReport
121
+
122
+ module PluginsStats
123
+ module_function
124
+
125
+ def plugin_stats(stats, plugin_type)
126
+ # Turn the `plugins` stats hash into an array of [ {}, {}, ... ]
127
+ # This is to produce an array of data points, one point for each
128
+ # plugin instance.
129
+ return [] unless stats[:plugins].include?(plugin_type)
130
+ stats[:plugins][plugin_type].collect do |id, data|
131
+ { :id => id }.merge(data)
132
+ end
133
+ end
134
+
135
+ def report(stats)
136
+ # Only one pipeline right now.
137
+ stats = stats[:main]
138
+
139
+ {
140
+ :events => stats[:events],
141
+ :pipeline => {
142
+ :inputs => plugin_stats(stats, :inputs),
143
+ :filters => plugin_stats(stats, :filters),
144
+ :outputs => plugin_stats(stats, :outputs)
145
+ }
146
+ }
147
+ end
148
+ end # module PluginsStats
53
149
  end
54
150
  end
55
151
  end
@@ -50,6 +50,10 @@ module LogStash
50
50
  respond_with :mem => mem_payload
51
51
  end
52
52
 
53
+ get "/pipeline" do
54
+ respond_with :pipeline => pipeline_payload
55
+ end
56
+
53
57
  private
54
58
 
55
59
  def events_payload
@@ -67,6 +71,10 @@ module LogStash
67
71
  def mem_payload
68
72
  @stats.memory
69
73
  end
74
+
75
+ def pipeline_payload
76
+ @stats.pipeline
77
+ end
70
78
  end
71
79
  end
72
80
  end
@@ -430,7 +430,7 @@ module LogStash::Config::Mixin
430
430
  result = nil
431
431
 
432
432
  if validator.nil?
433
- return true
433
+ return true, value
434
434
  elsif validator.is_a?(Array)
435
435
  value = [*value]
436
436
  if value.size > 1
@@ -2,6 +2,7 @@
2
2
  require "logstash/errors"
3
3
  require "logstash/config/cpu_core_strategy"
4
4
  require "logstash/settings"
5
+ require "socket"
5
6
 
6
7
  module LogStash
7
8
 
@@ -25,6 +25,7 @@ module LogStash
25
25
  @filter.metric = metric
26
26
 
27
27
  @metric_events = namespaced_metric.namespace(:events)
28
+ namespaced_metric.gauge(:name, config_name)
28
29
 
29
30
  # Not all the filters will do bufferings
30
31
  define_flush_method if @filter.respond_to?(:flush)
@@ -37,7 +38,9 @@ module LogStash
37
38
  def multi_filter(events)
38
39
  @metric_events.increment(:in, events.size)
39
40
 
41
+ clock = @metric_events.time(:duration_in_millis)
40
42
  new_events = @filter.multi_filter(events)
43
+ clock.stop
41
44
 
42
45
  # There is no garantee in the context of filter
43
46
  # that EVENTS_INT == EVENTS_OUT, see the aggregates and
@@ -46,7 +46,7 @@ module LogStash module Instrument
46
46
  end
47
47
 
48
48
  def report_time(namespace, key, duration)
49
- collector.push(namespace, key, :mean, :increment, duration)
49
+ collector.push(namespace, key, :counter, :increment, duration)
50
50
  end
51
51
 
52
52
  # This method return a metric instance tied to a specific namespace
@@ -81,7 +81,7 @@ module LogStash module Instrument
81
81
  #
82
82
  # @see LogStash::Instrument::Metric#time
83
83
  class TimedExecution
84
- MILLISECONDS = 1_000_000.0.freeze
84
+ MILLISECONDS = 1_000.0.freeze
85
85
 
86
86
  def initialize(metric, namespace, key)
87
87
  @metric = metric
@@ -45,7 +45,7 @@ module LogStash module Instrument
45
45
  end
46
46
 
47
47
  def namespace(name)
48
- NamespacedMetric.new(metric, namespace_name.concat(Array(name)))
48
+ NamespacedMetric.new(metric, namespace_name + Array(name))
49
49
  end
50
50
 
51
51
  private
@@ -28,6 +28,7 @@ module LogStash class OutputDelegator
28
28
  # Scope the metrics to the plugin
29
29
  namespaced_metric = metric.namespace(output.plugin_unique_name.to_sym)
30
30
  @metric_events = namespaced_metric.namespace(:events)
31
+ namespaced_metric.gauge(:name, config_name)
31
32
 
32
33
  @events_received = Concurrent::AtomicFixnum.new(0)
33
34
  end
@@ -129,7 +130,9 @@ module LogStash class OutputDelegator
129
130
  @events_received.increment(events.length)
130
131
  @metric_events.increment(:in, events.length)
131
132
 
133
+ clock = @metric_events.time(:duration_in_millis)
132
134
  @threadsafe_worker.multi_receive(events)
135
+ clock.stop
133
136
  @metric_events.increment(:out, events.length)
134
137
  end
135
138
 
@@ -139,7 +142,9 @@ module LogStash class OutputDelegator
139
142
 
140
143
  worker = @worker_queue.pop
141
144
  begin
145
+ clock = @metric_events.time(:duration_in_millis)
142
146
  worker.multi_receive(events)
147
+ clock.stop
143
148
  @metric_events.increment(:out, events.length)
144
149
  ensure
145
150
  @worker_queue.push(worker)
@@ -11,4 +11,4 @@
11
11
  # eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
12
12
  # fixed.
13
13
 
14
- LOGSTASH_VERSION = "5.0.0-alpha4.snapshot2"
14
+ LOGSTASH_VERSION = "5.0.0-alpha4.snapshot3"
@@ -43,6 +43,9 @@ module LogStash
43
43
  @server.add_tcp_listener(http_host, http_port)
44
44
 
45
45
  @server.run.join
46
+ rescue Errno::EADDRINUSE
47
+ message = "Logstash tried to bind to port #{@http_port}, but the port is already in use. You can specify a new port by launching logtash with the --http-port option."
48
+ raise Errno::EADDRINUSE.new(message)
46
49
  end
47
50
 
48
51
  def log(str)
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
17
17
  gem.require_paths = ["lib"]
18
18
  gem.version = LOGSTASH_CORE_VERSION.gsub(/-/, '.')
19
19
 
20
- gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha4.snapshot2"
20
+ gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha4.snapshot3"
21
21
 
22
22
  gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
23
23
  gem.add_runtime_dependency "pry", "~> 0.10.1" #(Ruby license)
@@ -67,15 +67,22 @@ describe LogStash::FilterDelegator do
67
67
  end
68
68
 
69
69
  context "when the filter buffer events" do
70
- it "doesn't increment out" do
70
+ before do
71
+ allow(metric).to receive(:increment).with(anything, anything)
72
+ end
73
+
74
+ it "has incremented :in" do
71
75
  expect(metric).to receive(:increment).with(:in, events.size)
72
- expect(metric).not_to receive(:increment)
76
+ subject.multi_filter(events)
77
+ end
73
78
 
79
+ it "has not incremented :out" do
80
+ expect(metric).not_to receive(:increment).with(:out, anything)
74
81
  subject.multi_filter(events)
75
82
  end
76
83
  end
77
84
 
78
- context "when the fitler create more events" do
85
+ context "when the filter create more events" do
79
86
  let(:plugin_klass) do
80
87
  Class.new(LogStash::Filters::Base) do
81
88
  config_name "super_plugin"
@@ -91,6 +98,10 @@ describe LogStash::FilterDelegator do
91
98
  end
92
99
  end
93
100
 
101
+ before do
102
+ allow(metric).to receive(:increment).with(anything, anything)
103
+ end
104
+
94
105
  it "increments the in/out of the metric" do
95
106
  expect(metric).to receive(:increment).with(:in, events.size)
96
107
  expect(metric).to receive(:increment).with(:out, events.size * 2)
@@ -112,6 +123,10 @@ describe LogStash::FilterDelegator do
112
123
  end
113
124
  end
114
125
 
126
+ before do
127
+ allow(metric).to receive(:increment).with(anything, anything)
128
+ end
129
+
115
130
  it "doesnt define a flush method" do
116
131
  expect(subject.respond_to?(:flush)).to be_falsey
117
132
  end
@@ -67,15 +67,15 @@ describe LogStash::Instrument::Metric do
67
67
 
68
68
  context "#time" do
69
69
  let(:sleep_time) { 2 }
70
- let(:sleep_time_ms) { sleep_time * 1_000_000 }
70
+ let(:sleep_time_ms) { sleep_time * 1_000 }
71
71
 
72
72
  it "records the duration" do
73
73
  subject.time(:root, :duration_ms) { sleep(sleep_time) }
74
74
 
75
- expect(collector.last).to be_within(sleep_time_ms).of(sleep_time_ms + 5000)
75
+ expect(collector.last).to be_within(sleep_time_ms).of(sleep_time_ms + 5)
76
76
  expect(collector[0]).to match(:root)
77
77
  expect(collector[1]).to be(:duration_ms)
78
- expect(collector[2]).to be(:mean)
78
+ expect(collector[2]).to be(:counter)
79
79
  end
80
80
 
81
81
  it "returns the value of the executed block" do
@@ -90,7 +90,7 @@ describe LogStash::Instrument::Metric do
90
90
  expect(collector.last).to be_within(sleep_time_ms).of(sleep_time_ms + 0.1)
91
91
  expect(collector[0]).to match(:root)
92
92
  expect(collector[1]).to be(:duration_ms)
93
- expect(collector[2]).to be(:mean)
93
+ expect(collector[2]).to be(:counter)
94
94
  end
95
95
  end
96
96
 
@@ -22,4 +22,11 @@ describe LogStash::Instrument::NamespacedMetric do
22
22
  it "returns the value of the block" do
23
23
  expect(subject.time(:duration_ms) { "hello" }).to eq("hello")
24
24
  end
25
+
26
+ it "its doesnt change the original `namespace` when creating a subnamespace" do
27
+ new_namespace = subject.namespace(:wally)
28
+
29
+ expect(subject.namespace_name).to eq([namespace])
30
+ expect(new_namespace.namespace_name).to eq([:stats, :wally])
31
+ end
25
32
  end
@@ -19,6 +19,7 @@ describe LogStash::OutputDelegator do
19
19
  allow(out_klass).to receive(:threadsafe?).and_return(false)
20
20
  allow(out_klass).to receive(:workers_not_supported?).and_return(false)
21
21
  allow(out_klass).to receive(:name).and_return("example")
22
+ allow(out_klass).to receive(:config_name)
22
23
  allow(out_inst).to receive(:register)
23
24
  allow(out_inst).to receive(:multi_receive)
24
25
  allow(out_inst).to receive(:metric=).with(any_args)
metadata CHANGED
@@ -1,21 +1,21 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-core
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.0.0.alpha4.snapshot2
4
+ version: 5.0.0.alpha4.snapshot3
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-15 00:00:00.000000000 Z
11
+ date: 2016-06-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
16
  - - '='
17
17
  - !ruby/object:Gem::Version
18
- version: 5.0.0.alpha4.snapshot2
18
+ version: 5.0.0.alpha4.snapshot3
19
19
  name: logstash-core-event-java
20
20
  prerelease: false
21
21
  type: :runtime
@@ -23,7 +23,7 @@ dependencies:
23
23
  requirements:
24
24
  - - '='
25
25
  - !ruby/object:Gem::Version
26
- version: 5.0.0.alpha4.snapshot2
26
+ version: 5.0.0.alpha4.snapshot3
27
27
  - !ruby/object:Gem::Dependency
28
28
  requirement: !ruby/object:Gem::Requirement
29
29
  requirements: