logstash-core 5.0.0.alpha6.snapshot1-java → 5.0.0.alpha6.snapshot2-java

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of logstash-core might be problematic. Click here for more details.

Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/lib/jars.rb +7 -0
  3. data/lib/logstash-core/version.rb +1 -1
  4. data/lib/logstash/agent.rb +2 -2
  5. data/lib/logstash/api/rack_app.rb +14 -15
  6. data/lib/logstash/config/config_ast.rb +2 -2
  7. data/lib/logstash/config/file.rb +1 -2
  8. data/lib/logstash/config/mixin.rb +9 -11
  9. data/lib/logstash/environment.rb +2 -1
  10. data/lib/logstash/filters/base.rb +1 -0
  11. data/lib/logstash/inputs/base.rb +1 -0
  12. data/lib/logstash/java_integration.rb +1 -0
  13. data/lib/logstash/logging.rb +1 -89
  14. data/lib/logstash/logging/json.rb +1 -1
  15. data/lib/logstash/logging/logger.rb +72 -0
  16. data/lib/logstash/output_delegator.rb +8 -16
  17. data/lib/logstash/output_delegator_strategies/legacy.rb +2 -1
  18. data/lib/logstash/output_delegator_strategies/shared.rb +1 -0
  19. data/lib/logstash/output_delegator_strategies/single.rb +1 -0
  20. data/lib/logstash/outputs/base.rb +5 -0
  21. data/lib/logstash/patches/clamp.rb +21 -0
  22. data/lib/logstash/pipeline.rb +24 -24
  23. data/lib/logstash/pipeline_reporter.rb +1 -1
  24. data/lib/logstash/plugin.rb +3 -10
  25. data/lib/logstash/plugins/registry.rb +3 -1
  26. data/lib/logstash/runner.rb +47 -74
  27. data/lib/logstash/shutdown_watcher.rb +1 -8
  28. data/lib/logstash/util/decorators.rb +3 -4
  29. data/lib/logstash/util/java_version.rb +0 -5
  30. data/lib/logstash/util/loggable.rb +7 -17
  31. data/lib/logstash/util/safe_uri.rb +2 -0
  32. data/lib/logstash/util/wrapped_synchronous_queue.rb +5 -5
  33. data/lib/logstash/version.rb +1 -1
  34. data/locales/en.yml +19 -8
  35. data/logstash-core.gemspec +4 -3
  36. data/spec/api/lib/rack_app_spec.rb +6 -4
  37. data/spec/logstash/config/loader_spec.rb +3 -1
  38. data/spec/logstash/config/mixin_spec.rb +28 -9
  39. data/spec/logstash/output_delegator_spec.rb +15 -2
  40. data/spec/logstash/outputs/base_spec.rb +0 -1
  41. data/spec/logstash/pipeline_reporter_spec.rb +1 -0
  42. data/spec/logstash/pipeline_spec.rb +14 -2
  43. data/spec/logstash/plugin_spec.rb +7 -6
  44. data/spec/logstash/runner_spec.rb +62 -44
  45. data/spec/logstash/shutdown_watcher_spec.rb +0 -4
  46. metadata +21 -18
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: e89e8c25486877ae001285428c6e6d9fbe3dfc6b
4
- data.tar.gz: 5e6318f6ae9f263864588fa2361f5e9ebdc94fb4
3
+ metadata.gz: 64e38afe34f986adc6451f62d9adb961977eb5d3
4
+ data.tar.gz: a78880a388cbae77c13de3a11bfd8789dad96898
5
5
  SHA512:
6
- metadata.gz: af57be94f22a74d3ffbbde98eca0eb92fa503c15254b666c5fdfe518c401af10260203b520b9ce26f2b9043c93fdd2289f2af5ecb1387deaf7f65bdfa6ad1a19
7
- data.tar.gz: 9d6a1a9a69f77fb49cb0ec3f5d2be1f8a0f2f3a460590a4722ae071cd711fb5a792baa663c233a0f06b94dbea6e7a09a3fd9c0f60ee0ac7d488097b5a96f2a52
6
+ metadata.gz: 1bac0882c5f848de4d0c7c3d9c115efabd49ad102d37cf652b294927c66c540ce37361ba8ef259e9080d93d6728f63781b266ebd9482552a9a6f5984fbdc8bb4
7
+ data.tar.gz: f1d4e71fefb6775e6a09523fc2ca0f1ead2e294c2c7eed06c748bd06360eacd2518ff459c422138d7c372fe8b42e9e5d52b148aefa0443aa6a231be925a7686f
@@ -0,0 +1,7 @@
1
+ require 'jar_dependencies'
2
+ require_jar('org.apache.logging.log4j', 'log4j-1.2-api', '2.6.2')
3
+ require_jar('org.apache.logging.log4j', 'log4j-api', '2.6.2')
4
+ require_jar('org.apache.logging.log4j', 'log4j-core', '2.6.2')
5
+ require_jar('com.fasterxml.jackson.core', 'jackson-core', '2.7.4')
6
+ require_jar('com.fasterxml.jackson.core', 'jackson-databind', '2.7.4')
7
+ require_jar('org.logstash', 'logstash-core', '5.0.0-alpha6')
@@ -5,4 +5,4 @@
5
5
  # Note to authors: this should not include dashes because 'gem' barfs if
6
6
  # you include a dash in the version string.
7
7
 
8
- LOGSTASH_CORE_VERSION = "5.0.0.alpha6.snapshot1"
8
+ LOGSTASH_CORE_VERSION = "5.0.0.alpha6.snapshot2"
@@ -18,6 +18,7 @@ require "securerandom"
18
18
  LogStash::Environment.load_locale!
19
19
 
20
20
  class LogStash::Agent
21
+ include LogStash::Util::Loggable
21
22
  STARTED_AT = Time.now.freeze
22
23
 
23
24
  attr_reader :metric, :node_name, :pipelines, :settings, :webserver
@@ -28,10 +29,9 @@ class LogStash::Agent
28
29
  # :node_name [String] - identifier for the agent
29
30
  # :auto_reload [Boolean] - enable reloading of pipelines
30
31
  # :reload_interval [Integer] - reload pipelines every X seconds
31
- # :logger [Cabin::Channel] - logger instance
32
32
  def initialize(settings = LogStash::SETTINGS)
33
+ @logger = self.class.logger
33
34
  @settings = settings
34
- @logger = Cabin::Channel.get(LogStash)
35
35
  @auto_reload = setting("config.reload.automatic")
36
36
 
37
37
  @pipelines = {}
@@ -10,18 +10,17 @@ require "logstash/api/modules/stats"
10
10
  module LogStash
11
11
  module Api
12
12
  module RackApp
13
- # Cabin is not compatible with CommonLogger, and this gives us more control anyway
14
13
  METADATA_FIELDS = [:request_method, :path_info, :query_string, :http_version, :http_accept].freeze
15
14
  def self.log_metadata(status, env)
16
15
  METADATA_FIELDS.reduce({:status => status}) do |acc, field|
17
16
  acc[field] = env[field.to_s.upcase]
18
17
  acc
19
- end
18
+ end
20
19
  end
21
-
20
+
22
21
  class ApiLogger
23
22
  LOG_MESSAGE = "API HTTP Request".freeze
24
-
23
+
25
24
  def initialize(app, logger)
26
25
  @app = app
27
26
  @logger = logger
@@ -30,24 +29,24 @@ module LogStash
30
29
  def call(env)
31
30
  res = @app.call(env)
32
31
  status, headers, body = res
33
-
32
+
34
33
  if fatal_error?(status)
35
- @logger.warn? && @logger.warn(LOG_MESSAGE, RackApp.log_metadata(status, env))
36
- else
37
- @logger.info? && @logger.info(LOG_MESSAGE, RackApp.log_metadata(status, env))
34
+ @logger.warn? && @logger.warn(LOG_MESSAGE, RackApp.log_metadata(status, env))
35
+ else
36
+ @logger.info? && @logger.info(LOG_MESSAGE, RackApp.log_metadata(status, env))
38
37
  end
39
38
 
40
- res
39
+ res
41
40
  end
42
41
 
43
42
  def fatal_error?(status)
44
43
  status >= 500 && status < 600
45
44
  end
46
45
  end
47
-
46
+
48
47
  class ApiErrorHandler
49
48
  LOG_MESSAGE = "Internal API server error".freeze
50
-
49
+
51
50
  def initialize(app, logger)
52
51
  @app = app
53
52
  @logger = logger
@@ -65,21 +64,21 @@ module LogStash
65
64
  })
66
65
 
67
66
  @logger.error(LOG_MESSAGE, body)
68
-
67
+
69
68
  [500,
70
69
  {'Content-Type' => 'application/json'},
71
70
  [LogStash::Json.dump(body)]
72
71
  ]
73
72
  end
74
73
  end
75
-
74
+
76
75
  def self.app(logger, agent, environment)
77
76
  namespaces = rack_namespaces(agent)
78
77
 
79
78
  Rack::Builder.new do
80
79
  # Custom logger object. Rack CommonLogger does not work with cabin
81
80
  use ApiLogger, logger
82
-
81
+
83
82
  # In test env we want errors to propogate up the chain
84
83
  # so we get easy to understand test failures.
85
84
  # In production / dev we don't want a bad API endpoint
@@ -87,7 +86,7 @@ module LogStash
87
86
  if environment != "test"
88
87
  use ApiErrorHandler, logger
89
88
  end
90
-
89
+
91
90
  run LogStash::Api::Modules::Root.new(nil, agent)
92
91
  namespaces.each_pair do |namespace, app|
93
92
  map(namespace) do
@@ -123,7 +123,7 @@ module LogStash; module Config; module AST
123
123
  definitions << "define_singleton_method :#{type}_func do |event|"
124
124
  definitions << " targeted_outputs = []" if type == "output"
125
125
  definitions << " events = [event]" if type == "filter"
126
- definitions << " @logger.debug? && @logger.debug(\"#{type} received\", :event => event.to_hash)"
126
+ definitions << " @logger.debug? && @logger.debug(\"#{type} received\", \"event\" => event.to_hash)"
127
127
 
128
128
  sections.select { |s| s.plugin_type.text_value == type }.each do |s|
129
129
  definitions << s.compile.split("\n", -1).map { |e| " #{e}" }
@@ -174,7 +174,7 @@ module LogStash; module Config; module AST
174
174
 
175
175
  return if events.nil? || events.empty?
176
176
 
177
- @logger.debug? && @logger.debug(\"Flushing\", :plugin => @generated_objects[:#{name}], :events => events)
177
+ @logger.debug? && @logger.debug(\"Flushing\", :plugin => @generated_objects[:#{name}], :events => events.map { |x| x.to_hash })
178
178
 
179
179
  #{plugin.compile_starting_here.gsub(/^/, " ")}
180
180
 
@@ -8,11 +8,10 @@ require "logger"
8
8
 
9
9
  class LogStash::Config::File
10
10
  include Enumerable
11
- attr_accessor :logger
11
+ include LogStash::Util::Loggable
12
12
 
13
13
  public
14
14
  def initialize(text)
15
- @logger = Cabin::Channel.get(LogStash)
16
15
  @text = text
17
16
  @config = parse(text)
18
17
  end # def initialize
@@ -66,7 +66,7 @@ module LogStash::Config::Mixin
66
66
  if opts && opts[:deprecated]
67
67
  extra = opts[:deprecated].is_a?(String) ? opts[:deprecated] : ""
68
68
  extra.gsub!("%PLUGIN%", self.class.config_name)
69
- @logger.warn("You are using a deprecated config setting " +
69
+ self.logger.warn("You are using a deprecated config setting " +
70
70
  "#{name.inspect} set in #{self.class.config_name}. " +
71
71
  "Deprecated settings will continue to work, " +
72
72
  "but are scheduled for removal from logstash " +
@@ -141,7 +141,7 @@ module LogStash::Config::Mixin
141
141
  next if key[0, 1] == "@"
142
142
 
143
143
  # Set this key as an instance variable only if it doesn't start with an '@'
144
- @logger.debug("config #{self.class.name}/@#{key} = #{value.inspect}")
144
+ self.logger.debug("config #{self.class.name}/@#{key} = #{value.inspect}")
145
145
  instance_variable_set("@#{key}", value)
146
146
  end
147
147
 
@@ -198,8 +198,7 @@ module LogStash::Config::Mixin
198
198
  # Deprecated: Declare the version of the plugin
199
199
  # inside the gemspec.
200
200
  def milestone(m = nil)
201
- @logger = Cabin::Channel.get(LogStash)
202
- @logger.debug(I18n.t('logstash.plugin.deprecated_milestone', :plugin => config_name))
201
+ self.logger.debug(I18n.t('logstash.plugin.deprecated_milestone', :plugin => config_name))
203
202
  end
204
203
 
205
204
  # Define a new configuration setting
@@ -262,7 +261,6 @@ module LogStash::Config::Mixin
262
261
  def validate(params)
263
262
  @plugin_name = config_name
264
263
  @plugin_type = ancestors.find { |a| a.name =~ /::Base$/ }.config_name
265
- @logger = Cabin::Channel.get(LogStash)
266
264
  is_valid = true
267
265
 
268
266
  print_version_notice
@@ -282,12 +280,12 @@ module LogStash::Config::Mixin
282
280
 
283
281
  if plugin_version < PLUGIN_VERSION_1_0_0
284
282
  if plugin_version < PLUGIN_VERSION_0_9_0
285
- @logger.info(I18n.t("logstash.plugin.version.0-1-x",
283
+ self.logger.info(I18n.t("logstash.plugin.version.0-1-x",
286
284
  :type => @plugin_type,
287
285
  :name => @config_name,
288
286
  :LOGSTASH_VERSION => LOGSTASH_VERSION))
289
287
  else
290
- @logger.info(I18n.t("logstash.plugin.version.0-9-x",
288
+ self.logger.info(I18n.t("logstash.plugin.version.0-9-x",
291
289
  :type => @plugin_type,
292
290
  :name => @config_name,
293
291
  :LOGSTASH_VERSION => LOGSTASH_VERSION))
@@ -297,7 +295,7 @@ module LogStash::Config::Mixin
297
295
  # If we cannot find a version in the currently installed gems we
298
296
  # will display this message. This could happen in the test, if you
299
297
  # create an anonymous class to test a plugin.
300
- @logger.warn(I18n.t("logstash.plugin.no_version",
298
+ self.logger.warn(I18n.t("logstash.plugin.no_version",
301
299
  :type => @plugin_type,
302
300
  :name => @config_name,
303
301
  :LOGSTASH_VERSION => LOGSTASH_VERSION))
@@ -321,7 +319,7 @@ module LogStash::Config::Mixin
321
319
 
322
320
  if invalid_params.size > 0
323
321
  invalid_params.each do |name|
324
- @logger.error("Unknown setting '#{name}' for #{@plugin_name}")
322
+ self.logger.error("Unknown setting '#{name}' for #{@plugin_name}")
325
323
  end
326
324
  return false
327
325
  end # if invalid_params.size > 0
@@ -348,7 +346,7 @@ module LogStash::Config::Mixin
348
346
 
349
347
  value = params[config_key]
350
348
  if value.nil? || (config[:list] && Array(value).empty?)
351
- @logger.error(I18n.t("logstash.runner.configuration.setting_missing",
349
+ self.logger.error(I18n.t("logstash.runner.configuration.setting_missing",
352
350
  :setting => config_key, :plugin => @plugin_name,
353
351
  :type => @plugin_type))
354
352
  is_valid = false
@@ -396,7 +394,7 @@ module LogStash::Config::Mixin
396
394
  # Used for converting values in the config to proper objects.
397
395
  params[key] = processed_value
398
396
  else
399
- @logger.error(I18n.t("logstash.runner.configuration.setting_invalid",
397
+ self.logger.error(I18n.t("logstash.runner.configuration.setting_invalid",
400
398
  :plugin => @plugin_name, :type => @plugin_type,
401
399
  :setting => key, :value => value.inspect,
402
400
  :value_type => config_settings[:validate],
@@ -1,5 +1,6 @@
1
1
  # encoding: utf-8
2
2
  require "logstash/errors"
3
+ require "logstash/java_integration"
3
4
  require "logstash/config/cpu_core_strategy"
4
5
  require "logstash/settings"
5
6
  require "socket"
@@ -32,7 +33,7 @@ module LogStash
32
33
  Setting.new("path.plugins", Array, []),
33
34
  Setting::String.new("interactive", nil, false),
34
35
  Setting::Boolean.new("config.debug", false),
35
- Setting::String.new("log.level", "warn", true, ["quiet", "verbose", "warn", "debug"]),
36
+ Setting::String.new("log.level", "warn", true, ["fatal", "error", "warn", "debug", "info", "trace"]),
36
37
  Setting::Boolean.new("version", false),
37
38
  Setting::Boolean.new("help", false),
38
39
  Setting::String.new("path.log", nil, false),
@@ -7,6 +7,7 @@ require "logstash/config/mixin"
7
7
  require "logstash/util/decorators"
8
8
 
9
9
  class LogStash::Filters::Base < LogStash::Plugin
10
+ include LogStash::Util::Loggable
10
11
  include LogStash::Config::Mixin
11
12
 
12
13
  config_name "filter"
@@ -9,6 +9,7 @@ require "logstash/util/decorators"
9
9
 
10
10
  # This is the base class for Logstash inputs.
11
11
  class LogStash::Inputs::Base < LogStash::Plugin
12
+ include LogStash::Util::Loggable
12
13
  include LogStash::Config::Mixin
13
14
 
14
15
  config_name "input"
@@ -1,5 +1,6 @@
1
1
  # encoding: utf-8
2
2
  require "java"
3
+ require "jars"
3
4
 
4
5
  # this is mainly for usage with JrJackson json parsing in :raw mode which genenerates
5
6
  # Java::JavaUtil::ArrayList and Java::JavaUtil::LinkedHashMap native objects for speed.
@@ -1,91 +1,3 @@
1
1
  # encoding: utf-8
2
+ require "logstash/logging/logger"
2
3
  require "logstash/namespace"
3
- require "cabin"
4
- require "logger"
5
-
6
- class LogStash::Logger
7
- attr_accessor :target
8
-
9
- public
10
- def initialize(*args)
11
- super()
12
-
13
- #self[:program] = File.basename($0)
14
- #subscribe(::Logger.new(*args))
15
- @target = args[0]
16
- @channel = Cabin::Channel.get(LogStash)
17
-
18
- # lame hack until cabin's smart enough not to doubley-subscribe something.
19
- # without this subscription count check, running the test suite
20
- # causes Cabin to subscribe to STDOUT maaaaaany times.
21
- subscriptions = @channel.instance_eval { @subscribers.count }
22
- @channel.subscribe(@target) unless subscriptions > 0
23
-
24
- # Set default loglevel to WARN unless $DEBUG is set (run with 'ruby -d')
25
- @level = $DEBUG ? :debug : :warn
26
- if ENV["LOGSTASH_DEBUG"]
27
- @level = :debug
28
- end
29
-
30
- # Direct metrics elsewhere.
31
- @channel.metrics.channel = Cabin::Channel.new
32
- end # def initialize
33
-
34
- # Delegation
35
- def level=(value) @channel.level = value; end
36
- def debug(*args); @channel.debug(*args); end
37
- def debug?(*args); @channel.debug?(*args); end
38
- def info(*args); @channel.info(*args); end
39
- def info?(*args); @channel.info?(*args); end
40
- def warn(*args); @channel.warn(*args); end
41
- def warn?(*args); @channel.warn?(*args); end
42
- def error(*args); @channel.error(*args); end
43
- def error?(*args); @channel.error?(*args); end
44
- def fatal(*args); @channel.fatal(*args); end
45
- def fatal?(*args); @channel.fatal?(*args); end
46
-
47
- def self.setup_log4j(logger)
48
- require "java"
49
-
50
- properties = java.util.Properties.new
51
- log4j_level = "WARN"
52
- case logger.level
53
- when :debug
54
- log4j_level = "DEBUG"
55
- when :info
56
- log4j_level = "INFO"
57
- when :warn
58
- log4j_level = "WARN"
59
- end # case level
60
- properties.setProperty("log4j.rootLogger", "#{log4j_level},logstash")
61
-
62
- # TODO(sissel): This is a shitty hack to work around the fact that
63
- # LogStash::Logger isn't used anymore. We should fix that.
64
- target = logger.instance_eval { @subscribers }.values.first.instance_eval { @io }
65
- case target
66
- when STDOUT
67
- properties.setProperty("log4j.appender.logstash",
68
- "org.apache.log4j.ConsoleAppender")
69
- properties.setProperty("log4j.appender.logstash.Target", "System.out")
70
- when STDERR
71
- properties.setProperty("log4j.appender.logstash",
72
- "org.apache.log4j.ConsoleAppender")
73
- properties.setProperty("log4j.appender.logstash.Target", "System.err")
74
- when target.is_a?(File)
75
- properties.setProperty("log4j.appender.logstash",
76
- "org.apache.log4j.FileAppender")
77
- properties.setProperty("log4j.appender.logstash.File", target.path)
78
- else
79
- properties.setProperty("log4j.appender.logstash", "org.apache.log4j.varia.NullAppender")
80
- end # case target
81
-
82
- properties.setProperty("log4j.appender.logstash.layout",
83
- "org.apache.log4j.PatternLayout")
84
- properties.setProperty("log4j.appender.logstash.layout.conversionPattern",
85
- "log4j, [%d{yyyy-MM-dd}T%d{HH:mm:ss.SSS}] %5p: %c: %m%n")
86
-
87
- org.apache.log4j.LogManager.resetConfiguration
88
- org.apache.log4j.PropertyConfigurator.configure(properties)
89
- logger.debug("log4j java properties setup", :log4j_level => log4j_level)
90
- end
91
- end # class LogStash::Logger
@@ -3,7 +3,7 @@ require "logstash/namespace"
3
3
  require "logstash/logging"
4
4
  require "logstash/json"
5
5
 
6
- module LogStash; class Logging; class JSON
6
+ module LogStash; module Logging; class JSON
7
7
  def initialize(io)
8
8
  raise ArgumentError, "Expected IO, got #{io.class.name}" unless io.is_a?(IO)
9
9
 
@@ -0,0 +1,72 @@
1
+ require "logstash/java_integration"
2
+
3
+ module LogStash
4
+ module Logging
5
+ class Logger
6
+ java_import org.apache.logging.log4j.Level
7
+ java_import org.apache.logging.log4j.LogManager
8
+ java_import org.apache.logging.log4j.core.config.Configurator
9
+
10
+ def initialize(name)
11
+ @logger = LogManager.getLogger(name)
12
+ end
13
+
14
+ def debug?
15
+ @logger.is_debug_enabled
16
+ end
17
+
18
+ def info?
19
+ @logger.is_info_enabled
20
+ end
21
+
22
+ def error?
23
+ @logger.is_error_enabled
24
+ end
25
+
26
+ def warn?
27
+ @logger.is_warn_enabled
28
+ end
29
+
30
+ def fatal?
31
+ @logger.is_fatal_enabled
32
+ end
33
+
34
+ def trace?
35
+ @logger.is_trace_enabled
36
+ end
37
+
38
+ def debug(message, data = {})
39
+ @logger.debug(message, data)
40
+ end
41
+
42
+ def warn(message, data = {})
43
+ @logger.warn(message, data)
44
+ end
45
+
46
+ def info(message, data = {})
47
+ @logger.info(message, data)
48
+ end
49
+
50
+ def error(message, data = {})
51
+ @logger.error(message, data)
52
+ end
53
+
54
+ def fatal(message, data = {})
55
+ @logger.fatal(message, data)
56
+ end
57
+
58
+ def trace(message, data = {})
59
+ @logger.trace(message, data)
60
+ end
61
+
62
+ # Point logging at a specific path.
63
+ def self.configure_logging(level, path = LogManager::ROOT_LOGGER_NAME)
64
+ Configurator.setLevel(path, Level.toLevel(level))
65
+ end # def configure_logging
66
+
67
+ def self.initialize(config_location)
68
+ Configurator.initialize(nil, config_location)
69
+ end
70
+ end
71
+ end
72
+ end