logstash-core 5.4.3-java → 5.5.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/lib/logstash-core/logstash-core.jar +0 -0
  3. data/lib/logstash-core/version.rb +1 -1
  4. data/lib/logstash/api/commands/hot_threads_reporter.rb +2 -2
  5. data/lib/logstash/api/commands/node.rb +0 -1
  6. data/lib/logstash/api/commands/stats.rb +0 -1
  7. data/lib/logstash/config/mixin.rb +5 -43
  8. data/lib/logstash/config/modules_common.rb +71 -0
  9. data/lib/logstash/elasticsearch_client.rb +120 -0
  10. data/lib/logstash/environment.rb +14 -3
  11. data/lib/logstash/errors.rb +1 -0
  12. data/lib/logstash/execution_context.rb +11 -3
  13. data/lib/logstash/inputs/base.rb +2 -0
  14. data/lib/logstash/instrument/global_metrics.rb +13 -0
  15. data/lib/logstash/instrument/metric_type/mean.rb +5 -0
  16. data/lib/logstash/instrument/periodic_poller/jvm.rb +5 -5
  17. data/lib/logstash/logging/logger.rb +26 -1
  18. data/lib/logstash/modules/cli_parser.rb +74 -0
  19. data/lib/logstash/modules/elasticsearch_config.rb +22 -0
  20. data/lib/logstash/modules/elasticsearch_resource.rb +10 -0
  21. data/lib/logstash/modules/file_reader.rb +36 -0
  22. data/lib/logstash/modules/importer.rb +37 -0
  23. data/lib/logstash/modules/kibana_base_resource.rb +10 -0
  24. data/lib/logstash/modules/kibana_config.rb +104 -0
  25. data/lib/logstash/modules/kibana_resource.rb +10 -0
  26. data/lib/logstash/modules/logstash_config.rb +48 -0
  27. data/lib/logstash/modules/resource_base.rb +37 -0
  28. data/lib/logstash/modules/scaffold.rb +44 -0
  29. data/lib/logstash/namespace.rb +1 -0
  30. data/lib/logstash/outputs/base.rb +2 -0
  31. data/lib/logstash/pipeline.rb +18 -4
  32. data/lib/logstash/plugin.rb +1 -0
  33. data/lib/logstash/plugins/registry.rb +5 -0
  34. data/lib/logstash/runner.rb +42 -2
  35. data/lib/logstash/settings.rb +7 -1
  36. data/lib/logstash/timestamp.rb +4 -0
  37. data/lib/logstash/util/dead_letter_queue_manager.rb +61 -0
  38. data/lib/logstash/util/safe_uri.rb +130 -11
  39. data/lib/logstash/util/thread_dump.rb +3 -1
  40. data/lib/logstash/util/wrapped_acked_queue.rb +24 -6
  41. data/lib/logstash/util/wrapped_synchronous_queue.rb +19 -5
  42. data/lib/logstash/version.rb +1 -1
  43. data/locales/en.yml +46 -0
  44. data/logstash-core.gemspec +7 -2
  45. data/spec/{api/lib/commands/stats.rb → logstash/api/commands/stats_spec.rb} +7 -2
  46. data/spec/{api/lib → logstash/api}/errors_spec.rb +1 -1
  47. data/spec/{api/lib/api → logstash/api/modules}/logging_spec.rb +1 -10
  48. data/spec/{api/lib/api → logstash/api/modules}/node_plugins_spec.rb +2 -3
  49. data/spec/{api/lib/api → logstash/api/modules}/node_spec.rb +6 -7
  50. data/spec/{api/lib/api → logstash/api/modules}/node_stats_spec.rb +2 -2
  51. data/spec/{api/lib/api → logstash/api/modules}/plugins_spec.rb +4 -3
  52. data/spec/{api/lib/api → logstash/api/modules}/root_spec.rb +3 -3
  53. data/spec/{api/lib → logstash/api}/rack_app_spec.rb +0 -0
  54. data/spec/logstash/config/mixin_spec.rb +2 -2
  55. data/spec/logstash/execution_context_spec.rb +20 -1
  56. data/spec/logstash/filter_delegator_spec.rb +2 -1
  57. data/spec/logstash/inputs/base_spec.rb +1 -1
  58. data/spec/logstash/output_delegator_spec.rb +2 -1
  59. data/spec/logstash/outputs/base_spec.rb +1 -1
  60. data/spec/logstash/pipeline_dlq_commit_spec.rb +107 -0
  61. data/spec/logstash/pipeline_pq_file_spec.rb +1 -1
  62. data/spec/logstash/plugin_spec.rb +1 -1
  63. data/spec/logstash/plugins/registry_spec.rb +22 -5
  64. data/spec/logstash/runner_spec.rb +122 -19
  65. data/spec/logstash/settings_spec.rb +91 -0
  66. data/spec/logstash/timestamp_spec.rb +6 -0
  67. data/spec/support/helpers.rb +80 -1
  68. data/spec/support/matchers.rb +13 -0
  69. data/spec/support/shared_contexts.rb +38 -0
  70. data/spec/support/shared_examples.rb +1 -1
  71. metadata +95 -40
  72. data/spec/api/lib/api/support/resource_dsl_methods.rb +0 -87
  73. data/spec/api/spec_helper.rb +0 -111
@@ -1,4 +1,6 @@
1
1
  # encoding: utf-8
2
+ java_import 'org.logstash.instrument.reports.ThreadsReport'
3
+
2
4
  module LogStash
3
5
  module Util
4
6
  class ThreadDump
@@ -10,7 +12,7 @@ module LogStash
10
12
 
11
13
  def initialize(options={})
12
14
  @options = options
13
- @dump = options.fetch(:dump, JRMonitor.threads.generate({}))
15
+ @dump = options.fetch(:dump, ThreadsReport.generate({}))
14
16
  @top_count = options.fetch(:threads, THREADS_COUNT_DEFAULT)
15
17
  @ignore = options.fetch(:ignore_idle_threads, IGNORE_IDLE_THREADS_DEFAULT)
16
18
  end
@@ -127,7 +127,12 @@ module LogStash; module Util
127
127
  end
128
128
 
129
129
  def empty?
130
- @mutex.synchronize { @queue.is_fully_acked? }
130
+ @mutex.lock
131
+ begin
132
+ @queue.is_fully_acked?
133
+ ensure
134
+ @mutex.unlock
135
+ end
131
136
  end
132
137
 
133
138
  def set_batch_dimensions(batch_size, wait_for)
@@ -152,8 +157,11 @@ module LogStash; module Util
152
157
  end
153
158
 
154
159
  def inflight_batches
155
- @mutex.synchronize do
160
+ @mutex.lock
161
+ begin
156
162
  yield(@inflight_batches)
163
+ ensure
164
+ @mutex.unlock
157
165
  end
158
166
  end
159
167
 
@@ -173,16 +181,24 @@ module LogStash; module Util
173
181
  end
174
182
 
175
183
  batch = new_batch
176
- @mutex.synchronize { batch.read_next }
184
+ @mutex.lock
185
+ begin
186
+ batch.read_next
187
+ ensure
188
+ @mutex.unlock
189
+ end
177
190
  start_metrics(batch)
178
191
  batch
179
192
  end
180
193
 
181
194
  def start_metrics(batch)
182
- @mutex.synchronize do
195
+ @mutex.lock
196
+ begin
183
197
  # there seems to be concurrency issues with metrics, keep it in the mutex
184
198
  set_current_thread_inflight_batch(batch)
185
199
  start_clock
200
+ ensure
201
+ @mutex.unlock
186
202
  end
187
203
  end
188
204
 
@@ -191,12 +207,14 @@ module LogStash; module Util
191
207
  end
192
208
 
193
209
  def close_batch(batch)
194
- @mutex.synchronize do
210
+ @mutex.lock
211
+ begin
195
212
  batch.close
196
-
197
213
  # there seems to be concurrency issues with metrics, keep it in the mutex
198
214
  @inflight_batches.delete(Thread.current)
199
215
  stop_clock(batch)
216
+ ensure
217
+ @mutex.unlock
200
218
  end
201
219
  end
202
220
 
@@ -98,8 +98,11 @@ module LogStash; module Util
98
98
  end
99
99
 
100
100
  def inflight_batches
101
- @mutex.synchronize do
101
+ @mutex.lock
102
+ begin
102
103
  yield(@inflight_batches)
104
+ ensure
105
+ @mutex.unlock
103
106
  end
104
107
  end
105
108
 
@@ -115,16 +118,24 @@ module LogStash; module Util
115
118
 
116
119
  def read_batch
117
120
  batch = new_batch
118
- @mutex.synchronize { batch.read_next }
121
+ @mutex.lock
122
+ begin
123
+ batch.read_next
124
+ ensure
125
+ @mutex.unlock
126
+ end
119
127
  start_metrics(batch)
120
128
  batch
121
129
  end
122
130
 
123
131
  def start_metrics(batch)
124
- @mutex.synchronize do
125
- # there seems to be concurrency issues with metrics, keep it in the mutex
132
+ @mutex.lock
133
+ # there seems to be concurrency issues with metrics, keep it in the mutex
134
+ begin
126
135
  set_current_thread_inflight_batch(batch)
127
136
  start_clock
137
+ ensure
138
+ @mutex.unlock
128
139
  end
129
140
  end
130
141
 
@@ -133,10 +144,13 @@ module LogStash; module Util
133
144
  end
134
145
 
135
146
  def close_batch(batch)
136
- @mutex.synchronize do
147
+ @mutex.lock
148
+ begin
137
149
  # there seems to be concurrency issues with metrics, keep it in the mutex
138
150
  @inflight_batches.delete(Thread.current)
139
151
  stop_clock(batch)
152
+ ensure
153
+ @mutex.unlock
140
154
  end
141
155
  end
142
156
 
@@ -11,4 +11,4 @@
11
11
  # eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
12
12
  # fixed.
13
13
 
14
- LOGSTASH_VERSION = "5.4.3"
14
+ LOGSTASH_VERSION = "5.5.0"
@@ -83,10 +83,31 @@ en:
83
83
  logging:
84
84
  unrecognized_option: |-
85
85
  unrecognized option [%{option}]
86
+ modules:
87
+ configuration:
88
+ parse-failed: |-
89
+ Failed to parse the module configuration: [%{error}]
90
+ modules-must-be-unique: >-
91
+ Only a single instance of any module can be run at a time. Duplicate
92
+ modules: %{duplicate_modules}
93
+ modules-invalid-name: >-
94
+ Invalid module name: %{module_name}
95
+ modules-variables-malformed: >-
96
+ Failed to parse module variable %{rawvar}. Must be in -M
97
+ "MODULE_NAME.KEY.SUBKEY=VALUE" format
98
+ modules-unavailable: >-
99
+ The modules specified are not available yet.
100
+ Specified modules: %{specified_modules}
101
+ Available modules: %{available_modules}
102
+ elasticsearch_connection_failed: >-
103
+ Failed to import module configurations to Elasticsearch.
104
+ Module: %{module_name} has hosts: %{hosts}
105
+
86
106
  runner:
87
107
  short-help: |-
88
108
  usage:
89
109
  bin/logstash -f CONFIG_PATH [-t] [-r] [] [-w COUNT] [-l LOG]
110
+ bin/logstash --modules MODULE_NAME [-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"] [-t] [-w COUNT] [-l LOG]
90
111
  bin/logstash -e CONFIG_STR [-t] [--log.level fatal|error|warn|info|debug|trace] [-w COUNT] [-l LOG]
91
112
  bin/logstash -i SHELL [--log.level fatal|error|warn|info|debug|trace]
92
113
  bin/logstash -V [--log.level fatal|error|warn|info|debug|trace]
@@ -96,6 +117,13 @@ en:
96
117
  missing-configuration: >-
97
118
  No configuration file was specified. Perhaps you forgot to provide
98
119
  the '-f yourlogstash.conf' flag?
120
+ config-module-exclusive: >-
121
+ Settings 'path.config' (-f) or 'config.string' (-e) can't be used in conjunction with
122
+ (--modules) or the "modules:" block in the logstash.yml file.
123
+ cli-module-override: >-
124
+ Both command-line and logstash.yml modules configurations detected.
125
+ Using command-line module configuration and ignoring logstash.yml module
126
+ configuration.
99
127
  reload-without-config-path: >-
100
128
  Configuration reloading also requires passing a configuration path with '-f yourlogstash.conf'
101
129
  locked-data-path: >-
@@ -181,6 +209,24 @@ en:
181
209
  "%{default_output}"
182
210
  If you wish to use both defaults, please use
183
211
  the empty string for the '-e' flag.
212
+ modules: |+
213
+ Load Logstash modules.
214
+ Modules can be defined using multiple instances
215
+ '--modules module1 --modules module2',
216
+ or comma-separated syntax
217
+ '--modules=module1,module2'
218
+ Cannot be used in conjunction with '-e' or '-f'
219
+ Use of '--modules' will override modules declared
220
+ in the 'logstash.yml' file.
221
+ modules_variable: |+
222
+ Load variables for module template.
223
+ Multiple instances of '-M' or
224
+ '--modules.variable' are supported.
225
+ Ignored if '--modules' flag is not used.
226
+ Should be in the format of
227
+ '-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"'
228
+ as in
229
+ '-M "example.var.filter.mutate.fieldname=fieldvalue"'
184
230
  configtest: |+
185
231
  Check configuration for valid syntax and then exit.
186
232
  http_host: Web API binding host
@@ -25,11 +25,14 @@ Gem::Specification.new do |gem|
25
25
  gem.add_runtime_dependency "filesize", "0.0.4" #(MIT license) for :bytes config validator
26
26
  gem.add_runtime_dependency "gems", "~> 0.8.3" #(MIT license)
27
27
  gem.add_runtime_dependency "concurrent-ruby", "~> 1.0", ">= 1.0.5"
28
+
29
+ # Later versions are ruby 2.0 only. We should remove the rack dep once we support 9k
30
+ gem.add_runtime_dependency "rack", '1.6.6'
31
+
28
32
  gem.add_runtime_dependency "sinatra", '~> 1.4', '>= 1.4.6'
29
33
  gem.add_runtime_dependency 'puma', '~> 2.16'
30
- gem.add_runtime_dependency "jruby-openssl", "0.9.16" # >= 0.9.13 Required to support TLSv1.2
34
+ gem.add_runtime_dependency "jruby-openssl", "0.9.19" # >= 0.9.13 Required to support TLSv1.2
31
35
  gem.add_runtime_dependency "chronic_duration", "0.10.6"
32
- gem.add_runtime_dependency "jrmonitor", '~> 0.4.2'
33
36
 
34
37
  # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
35
38
  # investigated what the cause might be. -Jordan
@@ -50,6 +53,8 @@ Gem::Specification.new do |gem|
50
53
  # has an rdoc problem that causes a bundler exception. 3.3.9 is the current latest version
51
54
  # which does not have this problem.
52
55
  gem.add_runtime_dependency "ruby-maven", "~> 3.3.9"
56
+ gem.add_runtime_dependency "elasticsearch", "~> 5.0", ">= 5.0.4" # Ruby client for ES (Apache 2.0 license)
57
+ gem.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
53
58
 
54
59
  eval(File.read(File.expand_path("../gemspec_jars.rb", __FILE__)))
55
60
  end
@@ -1,10 +1,15 @@
1
1
  # encoding: utf-8
2
- require_relative "../../spec_helper"
2
+ require "spec_helper"
3
3
 
4
4
  describe LogStash::Api::Commands::Stats do
5
+ include_context "api setup"
5
6
 
6
7
  let(:report_method) { :run }
7
- subject(:report) { do_request { report_class.new.send(report_method) } }
8
+ subject(:report) do
9
+ factory = ::LogStash::Api::CommandFactory.new(LogStash::Api::Service.new(@agent))
10
+
11
+ factory.build(:stats).send(report_method)
12
+ end
8
13
 
9
14
  let(:report_class) { described_class }
10
15
 
@@ -1,5 +1,5 @@
1
1
  # encoding: utf-8
2
- require_relative "../spec_helper"
2
+ require "spec_helper"
3
3
  require "logstash/api/errors"
4
4
 
5
5
  describe LogStash::Api::ApiError do
@@ -1,5 +1,5 @@
1
1
  # encoding: utf-8
2
- require_relative "../../spec_helper"
2
+ require "spec_helper"
3
3
  require "sinatra"
4
4
  require "logstash/api/modules/logging"
5
5
  require "logstash/json"
@@ -10,15 +10,6 @@ describe LogStash::Api::Modules::Logging do
10
10
  describe "#logging" do
11
11
 
12
12
  context "when setting a logger's log level" do
13
- before(:all) do
14
- @runner = LogStashRunner.new
15
- @runner.start
16
- end
17
-
18
- after(:all) do
19
- @runner.stop
20
- end
21
-
22
13
  it "should return a positive acknowledgement on success" do
23
14
  put '/', '{"logger.logstash": "ERROR"}'
24
15
  payload = LogStash::Json.load(last_response.body)
@@ -1,6 +1,5 @@
1
1
  # encoding: utf-8
2
- require_relative "../../../support/shared_examples"
3
- require_relative "../../spec_helper"
2
+ require "spec_helper"
4
3
  require "sinatra"
5
4
  require "logstash/api/modules/plugins"
6
5
  require "logstash/json"
@@ -12,7 +11,7 @@ describe LogStash::Api::Modules::Plugins do
12
11
  extend ResourceDSLMethods
13
12
 
14
13
  before(:each) do
15
- do_request { get "/" }
14
+ get "/"
16
15
  end
17
16
 
18
17
  let(:payload) { LogStash::Json.load(last_response.body) }
@@ -1,6 +1,5 @@
1
1
  # encoding: utf-8
2
- require_relative "../../spec_helper"
3
- require_relative "../../../support/shared_examples"
2
+ require "spec_helper"
4
3
  require "sinatra"
5
4
  require "logstash/api/modules/node"
6
5
  require "logstash/json"
@@ -12,7 +11,7 @@ describe LogStash::Api::Modules::Node do
12
11
  describe "#hot threads" do
13
12
 
14
13
  before(:all) do
15
- do_request { get "/hot_threads" }
14
+ get "/hot_threads"
16
15
  end
17
16
 
18
17
  it "respond OK" do
@@ -26,7 +25,7 @@ describe LogStash::Api::Modules::Node do
26
25
  context "#threads count" do
27
26
 
28
27
  before(:all) do
29
- do_request { get "/hot_threads?threads=5" }
28
+ get "/hot_threads?threads=5"
30
29
  end
31
30
 
32
31
  let(:payload) { LogStash::Json.load(last_response.body) }
@@ -49,7 +48,7 @@ describe LogStash::Api::Modules::Node do
49
48
  ].each do |path|
50
49
 
51
50
  before(:all) do
52
- do_request { get path }
51
+ get path
53
52
  end
54
53
 
55
54
  let(:payload) { last_response.body }
@@ -70,7 +69,7 @@ describe LogStash::Api::Modules::Node do
70
69
  @threads = []
71
70
  5.times { @threads << Thread.new { loop {} } }
72
71
 
73
- do_request { get "/hot_threads?human=t&threads=2"}
72
+ get "/hot_threads?human=t&threads=2"
74
73
  end
75
74
 
76
75
  after(:all) do
@@ -91,7 +90,7 @@ describe LogStash::Api::Modules::Node do
91
90
  "/hot_threads?human=f",
92
91
  ].each do |path|
93
92
  before(:all) do
94
- do_request { get path }
93
+ get path
95
94
  end
96
95
 
97
96
  it "should return a json payload content type" do
@@ -1,6 +1,6 @@
1
1
  # encoding: utf-8
2
- require_relative "../../spec_helper"
3
- require_relative "../../../support/shared_examples"
2
+ require "spec_helper"
3
+
4
4
  require "sinatra"
5
5
  require "logstash/api/modules/node_stats"
6
6
  require "logstash/json"
@@ -1,6 +1,5 @@
1
1
  # encoding: utf-8
2
- require_relative "../../spec_helper"
3
- require_relative "../../../support/shared_examples"
2
+ require "spec_helper"
4
3
  require "sinatra"
5
4
  require "logstash/api/modules/plugins"
6
5
  require "logstash/json"
@@ -40,7 +39,9 @@ describe LogStash::Api::Modules::Plugins do
40
39
 
41
40
  it "return a list of available plugins" do
42
41
  payload["plugins"].each do |plugin|
43
- expect(plugin).to be_available?
42
+ expect do
43
+ Gem::Specification.find_by_name(plugin["name"])
44
+ end.not_to raise_error
44
45
  end
45
46
  end
46
47
 
@@ -1,6 +1,6 @@
1
1
  # encoding: utf-8
2
- require_relative "../../spec_helper"
3
- require_relative "../../../support/shared_examples"
2
+ require "spec_helper"
3
+
4
4
  require "sinatra"
5
5
  require "logstash/api/modules/root"
6
6
  require "logstash/json"
@@ -9,7 +9,7 @@ describe LogStash::Api::Modules::Root do
9
9
  include_context "api setup"
10
10
 
11
11
  it "should respond to root resource" do
12
- do_request { get "/" }
12
+ get "/"
13
13
  expect(last_response).to be_ok
14
14
  end
15
15
 
@@ -192,8 +192,8 @@ describe LogStash::Config::Mixin do
192
192
  expect(clone.uri.to_s).to eql(uri_hidden)
193
193
  end
194
194
 
195
- it "should make the real URI object available under #uri" do
196
- expect(subject.uri.uri).to be_a(::URI)
195
+ it "should make the real java.net.URI object available under #uri" do
196
+ expect(subject.uri.uri).to be_a(java.net.URI)
197
197
  end
198
198
 
199
199
  it "should obfuscate original_params" do
@@ -1,13 +1,32 @@
1
1
  # encoding: utf-8
2
2
  require "spec_helper"
3
+ require "logstash/util/dead_letter_queue_manager"
3
4
  require "logstash/execution_context"
4
5
 
5
6
  describe LogStash::ExecutionContext do
7
+ let(:pipeline) { double("pipeline") }
6
8
  let(:pipeline_id) { :main }
9
+ let(:plugin_id) { "plugin_id" }
10
+ let(:plugin_type) { "plugin_type" }
11
+ let(:dlq_writer) { LogStash::Util::DummyDeadLetterQueueWriter.new }
7
12
 
8
- subject { described_class.new(pipeline_id) }
13
+ before do
14
+ allow(pipeline).to receive(:pipeline_id).and_return(pipeline_id)
15
+ end
16
+
17
+ subject { described_class.new(pipeline, plugin_id, plugin_type, dlq_writer) }
9
18
 
10
19
  it "returns the `pipeline_id`" do
11
20
  expect(subject.pipeline_id).to eq(pipeline_id)
12
21
  end
22
+
23
+ it "returns the pipeline" do
24
+ expect(subject.pipeline).to eq(pipeline)
25
+ end
26
+
27
+ it "returns the plugin-specific dlq writer" do
28
+ expect(subject.dlq_writer.plugin_type).to eq(plugin_type)
29
+ expect(subject.dlq_writer.plugin_id).to eq(plugin_id)
30
+ expect(subject.dlq_writer.inner_writer).to eq(dlq_writer)
31
+ end
13
32
  end