logstash-core 2.0.0.beta1-java → 2.0.0.beta2-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of logstash-core might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 8280b889d663e0310425acfb0f7d75f4e1a024e0
4
- data.tar.gz: a164ff373d9abbd7189adf99af24bacb3bc91238
3
+ metadata.gz: 6026efcfb1f8b27ca3aead70d7055af4b89acc91
4
+ data.tar.gz: 04df43c7a2703f14b7511906d0ab74c60b4e6954
5
5
  SHA512:
6
- metadata.gz: c8b4f816cac02bf178e9b408a28a8178d4b608adbca9d58e0528fe9f9b52b1c0ab37edddd88f83b8f8b9b66de4fed6f9aafe8b4867b7ab5aca48588df8edeccb
7
- data.tar.gz: 4dd5b531749f62087e2816b5855e4093f8039ad07a5c9753b1a8d2a58f9d3a311b7459f129ed4eaf38c311132c16537c766c6f85556bf438fe5809d924b75173
6
+ metadata.gz: 1d3cbf47dd791f55bb26a8c39ded8f13f7c44c44af5b6975e0f9133153d0896d5eb2807ba93564634a17ae25d6cd09ef961c687e1375cef8647ce6f458525602
7
+ data.tar.gz: 4f5f02dc8a9f35ffa97f4a6789712d0e542e70e73a637055da1a143268f5e5882476549ee87918018c3ea8b689b1bac008f83a47bb1d28115b89daff2abd178e
@@ -2,6 +2,7 @@
2
2
  require "clamp" # gem 'clamp'
3
3
  require "logstash/environment"
4
4
  require "logstash/errors"
5
+ require "logstash/config/cpu_core_strategy"
5
6
  require "uri"
6
7
  require "net/http"
7
8
  LogStash::Environment.load_locale!
@@ -21,7 +22,8 @@ class LogStash::Agent < Clamp::Command
21
22
 
22
23
  option ["-w", "--filterworkers"], "COUNT",
23
24
  I18n.t("logstash.agent.flag.filterworkers"),
24
- :attribute_name => :filter_workers, :default => 1, &:to_i
25
+ :attribute_name => :filter_workers,
26
+ :default => LogStash::Config::CpuCoreStrategy.fifty_percent, &:to_i
25
27
 
26
28
  option ["-l", "--log"], "FILE",
27
29
  I18n.t("logstash.agent.flag.log"),
@@ -126,14 +128,14 @@ class LogStash::Agent < Clamp::Command
126
128
  @logger.warn(I18n.t("logstash.agent.sigint"))
127
129
  Thread.new(@logger) {|logger| sleep 5; logger.warn(I18n.t("logstash.agent.slow_shutdown")) }
128
130
  @interrupted_once = true
129
- pipeline.shutdown
131
+ shutdown(pipeline)
130
132
  end
131
133
  end
132
134
 
133
135
  # Make SIGTERM shutdown the pipeline.
134
136
  sigterm_id = Stud::trap("TERM") do
135
137
  @logger.warn(I18n.t("logstash.agent.sigterm"))
136
- pipeline.shutdown
138
+ shutdown(pipeline)
137
139
  end
138
140
 
139
141
  Stud::trap("HUP") do
@@ -172,6 +174,13 @@ class LogStash::Agent < Clamp::Command
172
174
  Stud::untrap("TERM", sigterm_id) unless sigterm_id.nil?
173
175
  end # def execute
174
176
 
177
+ def shutdown(pipeline)
178
+ pipeline.shutdown do
179
+ InflightEventsReporter.logger = @logger
180
+ InflightEventsReporter.start(pipeline.input_to_filter, pipeline.filter_to_output, pipeline.outputs)
181
+ end
182
+ end
183
+
175
184
  def show_version
176
185
  show_version_logstash
177
186
 
@@ -28,7 +28,7 @@ module LogStash::Codecs; class Base < LogStash::Plugin
28
28
  end # def encode
29
29
 
30
30
  public
31
- def teardown; end;
31
+ def close; end;
32
32
 
33
33
  # @param block [Proc(event, data)] the callback proc passing the original event and the encoded event
34
34
  public
@@ -0,0 +1,32 @@
1
+ # encoding: utf-8
2
+ require "logstash/namespace"
3
+ require "logstash/config/defaults"
4
+
5
+ module LogStash module Config module CpuCoreStrategy
6
+
7
+ extend self
8
+
9
+ def maximum
10
+ LogStash::Config::Defaults.cpu_cores
11
+ end
12
+
13
+ def fifty_percent
14
+ [1, (maximum * 0.5)].max.floor
15
+ end
16
+
17
+ def seventy_five_percent
18
+ [1, (maximum * 0.75)].max.floor
19
+ end
20
+
21
+ def twenty_five_percent
22
+ [1, (maximum * 0.25)].max.floor
23
+ end
24
+
25
+ def max_minus_one
26
+ [1, (maximum - 1)].max.floor
27
+ end
28
+
29
+ def max_minus_two
30
+ [1, (maximum - 2)].max.floor
31
+ end
32
+ end end end
@@ -0,0 +1,12 @@
1
+ # encoding: utf-8
2
+ require "logstash/namespace"
3
+ require "concurrent"
4
+
5
+ module LogStash module Config module Defaults
6
+
7
+ extend self
8
+
9
+ def cpu_cores
10
+ Concurrent.processor_count
11
+ end
12
+ end end end
@@ -76,6 +76,13 @@ module LogStash::Config::Mixin
76
76
  "about this, please visit the #logstash channel " +
77
77
  "on freenode irc.", :name => name, :plugin => self)
78
78
  end
79
+ if opts && opts[:obsolete]
80
+ extra = opts[:obsolete].is_a?(String) ? opts[:obsolete] : ""
81
+ extra.gsub!("%PLUGIN%", self.class.config_name)
82
+ raise LogStash::ConfigurationError,
83
+ I18n.t("logstash.agent.configuration.obsolete", :name => name,
84
+ :plugin => self.class.config_name, :extra => extra)
85
+ end
79
86
  end
80
87
 
81
88
  # Set defaults from 'config :foo, :default => somevalue'
@@ -103,6 +110,16 @@ module LogStash::Config::Mixin
103
110
  I18n.t("logstash.agent.configuration.invalid_plugin_settings")
104
111
  end
105
112
 
113
+ # We remove any config options marked as obsolete,
114
+ # no code should be associated to them and their values should not bleed
115
+ # to the plugin context.
116
+ #
117
+ # This need to be done after fetching the options from the parents classed
118
+ params.reject! do |name, value|
119
+ opts = self.class.get_config[name]
120
+ opts.include?(:obsolete)
121
+ end
122
+
106
123
  # set instance variables like '@foo' for each config value given.
107
124
  params.each do |key, value|
108
125
  next if key[0, 1] == "@"
@@ -11,22 +11,11 @@ class LogStash::Filters::Base < LogStash::Plugin
11
11
 
12
12
  config_name "filter"
13
13
 
14
- # Note that all of the specified routing options (`type`,`tags`,`exclude_tags`,`include_fields`,
15
- # `exclude_fields`) must be met in order for the event to be handled by the filter.
14
+ config :type, :validate => :string, :default => "", :obsolete => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
16
15
 
17
- # The type to act on. If a type is given, then this filter will only
18
- # act on messages with the same type. See any input plugin's `type`
19
- # attribute for more.
20
- # Optional.
21
- config :type, :validate => :string, :default => "", :deprecated => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
22
-
23
- # Only handle events with all of these tags.
24
- # Optional.
25
- config :tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
16
+ config :tags, :validate => :array, :default => [], :obsolete => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
26
17
 
27
- # Only handle events without any of these tags.
28
- # Optional.
29
- config :exclude_tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if !(\"sometag\" in [tags]) { %PLUGIN% { ... } }`"
18
+ config :exclude_tags, :validate => :array, :default => [], :obsolete => "You can achieve similar behavior with the new conditionals, like: `if (\"sometag\" not in [tags]) { %PLUGIN% { ... } }`"
30
19
 
31
20
  # If this filter is successful, add arbitrary tags to the event.
32
21
  # Tags can be dynamic and include parts of the event using the `%{field}`
@@ -202,40 +191,12 @@ class LogStash::Filters::Base < LogStash::Plugin
202
191
 
203
192
  protected
204
193
  def filter?(event)
205
- if !@type.empty?
206
- if event["type"] != @type
207
- @logger.debug? and @logger.debug("filters/#{self.class.name}: Skipping event because type doesn't match",
208
- :type=> @type, :event => event)
209
- return false
210
- end
211
- end
212
-
213
- if !@tags.empty?
214
- # this filter has only works on events with certain tags,
215
- # and this event has no tags.
216
- return false if !event["tags"]
217
-
218
- # Is @tags a subset of the event's tags? If not, skip it.
219
- if (event["tags"] & @tags).size != @tags.size
220
- @logger.debug? and @logger.debug("filters/#{self.class.name}: Skipping event because tags don't match",
221
- :tags => tags, :event => event)
222
- return false
223
- end
224
- end
225
-
226
- if !@exclude_tags.empty? && event["tags"]
227
- if (diff_tags = (event["tags"] & @exclude_tags)).size != 0
228
- @logger.debug("filters/#{self.class.name}: Skipping event because tags contains excluded tags:",
229
- :diff_tags => diff_tags, :exclude_tags => @exclude_tags, :event => event)
230
- return false
231
- end
232
- end
233
-
234
- return true
235
- end
194
+ # TODO: noop for now, remove this once we delete this call from all plugins
195
+ true
196
+ end # def filter?
236
197
 
237
198
  public
238
- def teardown
199
+ def close
239
200
  # Nothing to do by default.
240
201
  end
241
202
  end # class LogStash::Filters::Base
@@ -26,31 +26,16 @@ class LogStash::Inputs::Base < LogStash::Plugin
26
26
  # when sent to another Logstash server.
27
27
  config :type, :validate => :string
28
28
 
29
- config :debug, :validate => :boolean, :default => false, :deprecated => "This setting no longer has any effect. In past releases, it existed, but almost no plugin made use of it."
29
+ config :debug, :validate => :boolean, :default => false, :obsolete => "This setting no longer has any effect. In past releases, it existed, but almost no plugin made use of it."
30
30
 
31
- # The format of input data (plain, json, json_event)
32
- config :format, :validate => ["plain", "json", "json_event", "msgpack_event"], :deprecated => "You should use the newer 'codec' setting instead."
31
+ config :format, :validate => ["plain", "json", "json_event", "msgpack_event"], :obsolete => "You should use the newer 'codec' setting instead."
33
32
 
34
- # The codec used for input data. Input codecs are a convenient method for decoding your data before it enters the input, without needing a separate filter in your Logstash pipeline.
35
- config :codec, :validate => :codec, :default => "plain"
33
+ config :charset, :obsolete => "Use the codec setting instead. For example: input { %PLUGIN% { codec => plain { charset => \"UTF-8\" } }"
36
34
 
37
- # The character encoding used in this input. Examples include `UTF-8`
38
- # and `cp1252`
39
- #
40
- # This setting is useful if your log files are in `Latin-1` (aka `cp1252`)
41
- # or in another character set other than `UTF-8`.
42
- #
43
- # This only affects `plain` format logs since json is `UTF-8` already.
44
- config :charset, :deprecated => "Use the codec setting instead. For example: input { %PLUGIN% { codec => plain { charset => \"UTF-8\" } }"
35
+ config :message_format, :validate => :string, :obsolete => "Setting is no longer valid."
45
36
 
46
- # If format is `json`, an event `sprintf` string to build what
47
- # the display `@message` should be given (defaults to the raw JSON).
48
- # `sprintf` format strings look like `%{fieldname}`
49
- #
50
- # If format is `json_event`, ALL fields except for `@type`
51
- # are expected to be present. Not receiving all fields
52
- # will cause unexpected results.
53
- config :message_format, :validate => :string, :deprecated => true
37
+ # The codec used for input data. Input codecs are a convenient method for decoding your data before it enters the input, without needing a separate filter in your Logstash pipeline.
38
+ config :codec, :validate => :codec, :default => "plain"
54
39
 
55
40
  # Add any number of arbitrary tags to your event.
56
41
  #
@@ -67,27 +52,9 @@ class LogStash::Inputs::Base < LogStash::Plugin
67
52
  def initialize(params={})
68
53
  super
69
54
  @threadable = false
55
+ @stop_called = Concurrent::AtomicBoolean.new(false)
70
56
  config_init(params)
71
57
  @tags ||= []
72
-
73
- if @charset && @codec.class.get_config.include?("charset")
74
- # charset is deprecated on inputs, but provide backwards compatibility
75
- # by copying the charset setting into the codec.
76
-
77
- @logger.info("Copying input's charset setting into codec", :input => self, :codec => @codec)
78
- charset = @charset
79
- @codec.instance_eval { @charset = charset }
80
- end
81
-
82
- # Backwards compat for the 'format' setting
83
- case @format
84
- when "plain"; # do nothing
85
- when "json"
86
- @codec = LogStash::Plugin.lookup("codec", "json").new
87
- when "json_event"
88
- @codec = LogStash::Plugin.lookup("codec", "oldlogstashjson").new
89
- end
90
-
91
58
  end # def initialize
92
59
 
93
60
  public
@@ -100,6 +67,28 @@ class LogStash::Inputs::Base < LogStash::Plugin
100
67
  @tags << newtag
101
68
  end # def tag
102
69
 
70
+ public
71
+ # override stop if you need to do more than do_stop to
72
+ # enforce the input plugin to return from `run`.
73
+ # e.g. a tcp plugin might need to close the tcp socket
74
+ # so blocking read operation aborts
75
+ def stop
76
+ # override if necessary
77
+ end
78
+
79
+ public
80
+ def do_stop
81
+ @logger.debug("stopping", :plugin => self)
82
+ @stop_called.make_true
83
+ stop
84
+ end
85
+
86
+ # stop? should never be overriden
87
+ public
88
+ def stop?
89
+ @stop_called.value
90
+ end
91
+
103
92
  protected
104
93
  def to_event(raw, source)
105
94
  raise LogStash::ThisMethodWasRemoved("LogStash::Inputs::Base#to_event - you should use codecs now instead of to_event. Not sure what this means? Get help on https://discuss.elastic.co/c/logstash")
@@ -32,15 +32,23 @@ module LogStash
32
32
  ### JRuby
33
33
 
34
34
  def jruby_load(data, options = {})
35
- options[:symbolize_keys] ? JrJackson::Raw.parse_sym(data) : JrJackson::Raw.parse_raw(data)
35
+ # TODO [guyboertje] remove these comments in 5.0
36
+ # options[:symbolize_keys] ? JrJackson::Raw.parse_sym(data) : JrJackson::Raw.parse_raw(data)
37
+
38
+ JrJackson::Ruby.parse(data, options)
39
+
36
40
  rescue JrJackson::ParseError => e
37
41
  raise LogStash::Json::ParserError.new(e.message)
38
42
  end
39
43
 
40
44
  def jruby_dump(o)
45
+ # TODO [guyboertje] remove these comments in 5.0
41
46
  # test for enumerable here to work around an omission in JrJackson::Json.dump to
42
47
  # also look for Java::JavaUtil::ArrayList, see TODO submit issue
43
- o.is_a?(Enumerable) ? JrJackson::Raw.generate(o) : JrJackson::Json.dump(o)
48
+ # o.is_a?(Enumerable) ? JrJackson::Raw.generate(o) : JrJackson::Json.dump(o)
49
+
50
+ JrJackson::Base.generate(o, {})
51
+
44
52
  rescue => e
45
53
  raise LogStash::Json::GeneratorError.new(e.message)
46
54
  end
@@ -12,19 +12,11 @@ class LogStash::Outputs::Base < LogStash::Plugin
12
12
 
13
13
  config_name "output"
14
14
 
15
- # The type to act on. If a type is given, then this output will only
16
- # act on messages with the same type. See any input plugin's `type`
17
- # attribute for more.
18
- # Optional.
19
- config :type, :validate => :string, :default => "", :deprecated => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
15
+ config :type, :validate => :string, :default => "", :obsolete => "You can achieve this same behavior with the new conditionals, like: `if [type] == \"sometype\" { %PLUGIN% { ... } }`."
20
16
 
21
- # Only handle events with all of these tags.
22
- # Optional.
23
- config :tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
17
+ config :tags, :validate => :array, :default => [], :obsolete => "You can achieve similar behavior with the new conditionals, like: `if \"sometag\" in [tags] { %PLUGIN% { ... } }`"
24
18
 
25
- # Only handle events without any of these tags.
26
- # Optional.
27
- config :exclude_tags, :validate => :array, :default => [], :deprecated => "You can achieve similar behavior with the new conditionals, like: `if !(\"sometag\" in [tags]) { %PLUGIN% { ... } }`"
19
+ config :exclude_tags, :validate => :array, :default => [], :obsolete => "You can achieve similar behavior with the new conditionals, like: `if (\"sometag\" not in [tags]) { %PLUGIN% { ... } }`"
28
20
 
29
21
  # The codec used for output data. Output codecs are a convenient method for encoding your data before it leaves the output, without needing a separate filter in your Logstash pipeline.
30
22
  config :codec, :validate => :codec, :default => "plain"
@@ -94,31 +86,7 @@ class LogStash::Outputs::Base < LogStash::Plugin
94
86
 
95
87
  private
96
88
  def output?(event)
97
- if !@type.empty?
98
- if event["type"] != @type
99
- @logger.debug? and @logger.debug("outputs/#{self.class.name}: Dropping event because type doesn't match",
100
- :type => @type, :event => event)
101
- return false
102
- end
103
- end
104
-
105
- if !@tags.empty?
106
- return false if !event["tags"]
107
- if (event["tags"] & @tags).size != @tags.size
108
- @logger.debug? and @logger.debug("outputs/#{self.class.name}: Dropping event because tags don't match",
109
- :tags => @tags, :event => event)
110
- return false
111
- end
112
- end
113
-
114
- if !@exclude_tags.empty? && event["tags"]
115
- if (diff_tags = (event["tags"] & @exclude_tags)).size != 0
116
- @logger.debug? and @logger.debug("outputs/#{self.class.name}: Dropping event because tags contains excluded tags",
117
- :diff_tags => diff_tags, :exclude_tags => @exclude_tags, :event => event)
118
- return false
119
- end
120
- end
121
-
122
- return true
123
- end
89
+ # TODO: noop for now, remove this once we delete this call from all plugins
90
+ true
91
+ end # def output?
124
92
  end # class LogStash::Outputs::Base
@@ -3,3 +3,4 @@ require "logstash/patches/bugfix_jruby_2558"
3
3
  require "logstash/patches/cabin"
4
4
  require "logstash/patches/profile_require_calls"
5
5
  require "logstash/patches/stronger_openssl_defaults"
6
+ require "logstash/patches/silence_concurrent_ruby_warning"
@@ -0,0 +1,54 @@
1
+ # encoding: utf-8
2
+ require "logstash/namespace"
3
+ require "concurrent/concern/logging"
4
+ require "concurrent/concern/deprecation"
5
+ require "concurrent/version"
6
+ require "cabin"
7
+
8
+ # Concurrent-ruby is throwing warning when the code is run under jdk7, and they
9
+ # will provide best effort support, logstash has to support JDK7 for a few months.
10
+ #
11
+ # By default all deprecation warnings of the concurrent ruby
12
+ # library use the `WARN` level which is show everytime we boot logstash,
13
+ # This monkeypatch change the log level of the deprecation warning to be `debug`
14
+ # instead. This monkey patch might be a bit over kill but there is no
15
+ # easy way to override the java version check.
16
+ #
17
+ # ref: https://github.com/ruby-concurrency/concurrent-ruby/blob/v0.9.1/lib/concurrent/configuration.rb#L284-L295
18
+ #
19
+ # This patch is only valid for 0.9.1
20
+ if Concurrent::VERSION == "0.9.1"
21
+ module Concurrent
22
+ module Concern
23
+ module Deprecation
24
+ include Concern::Logging
25
+
26
+ def deprecated(message, strip = 2)
27
+ caller_line = caller(strip).first if strip > 0
28
+ klass = if Module === self
29
+ self
30
+ else
31
+ self.class
32
+ end
33
+ message = if strip > 0
34
+ format("[DEPRECATED] %s\ncalled on: %s", message, caller_line)
35
+ else
36
+ format('[DEPRECATED] %s', message)
37
+ end
38
+
39
+ # lets use our logger
40
+ logger = Cabin::Channel.get(LogStash)
41
+ logger.debug(message, :class => klass.to_s)
42
+ end
43
+
44
+ extend self
45
+ end
46
+ end
47
+ end
48
+ else
49
+ # This is added a guard to check if we need to update this code or not.
50
+ # Keep in mind, the latest releases of concurrent-ruby brokes a few stuff.
51
+ #
52
+ # Even the latest master version changed how they handle deprecation.
53
+ raise "Logstash expects concurrent-ruby version 0.9.1 and version #{Concurrent::VERSION} is installed, please verify this patch: #{__FILE__}"
54
+ end