logstash-core 2.2.4.snapshot1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of logstash-core might be problematic. Click here for more details.

Files changed (83) hide show
  1. checksums.yaml +7 -0
  2. data/lib/logstash-core.rb +1 -0
  3. data/lib/logstash-core/logstash-core.rb +3 -0
  4. data/lib/logstash-core/version.rb +8 -0
  5. data/lib/logstash/agent.rb +391 -0
  6. data/lib/logstash/codecs/base.rb +50 -0
  7. data/lib/logstash/config/config_ast.rb +550 -0
  8. data/lib/logstash/config/cpu_core_strategy.rb +32 -0
  9. data/lib/logstash/config/defaults.rb +12 -0
  10. data/lib/logstash/config/file.rb +39 -0
  11. data/lib/logstash/config/grammar.rb +3503 -0
  12. data/lib/logstash/config/mixin.rb +518 -0
  13. data/lib/logstash/config/registry.rb +13 -0
  14. data/lib/logstash/environment.rb +98 -0
  15. data/lib/logstash/errors.rb +12 -0
  16. data/lib/logstash/filters/base.rb +205 -0
  17. data/lib/logstash/inputs/base.rb +116 -0
  18. data/lib/logstash/inputs/threadable.rb +18 -0
  19. data/lib/logstash/java_integration.rb +116 -0
  20. data/lib/logstash/json.rb +61 -0
  21. data/lib/logstash/logging.rb +91 -0
  22. data/lib/logstash/namespace.rb +13 -0
  23. data/lib/logstash/output_delegator.rb +172 -0
  24. data/lib/logstash/outputs/base.rb +91 -0
  25. data/lib/logstash/patches.rb +5 -0
  26. data/lib/logstash/patches/bugfix_jruby_2558.rb +51 -0
  27. data/lib/logstash/patches/cabin.rb +35 -0
  28. data/lib/logstash/patches/profile_require_calls.rb +47 -0
  29. data/lib/logstash/patches/rubygems.rb +38 -0
  30. data/lib/logstash/patches/stronger_openssl_defaults.rb +68 -0
  31. data/lib/logstash/pipeline.rb +499 -0
  32. data/lib/logstash/pipeline_reporter.rb +114 -0
  33. data/lib/logstash/plugin.rb +120 -0
  34. data/lib/logstash/program.rb +14 -0
  35. data/lib/logstash/runner.rb +124 -0
  36. data/lib/logstash/shutdown_watcher.rb +100 -0
  37. data/lib/logstash/util.rb +203 -0
  38. data/lib/logstash/util/buftok.rb +139 -0
  39. data/lib/logstash/util/charset.rb +35 -0
  40. data/lib/logstash/util/decorators.rb +52 -0
  41. data/lib/logstash/util/defaults_printer.rb +31 -0
  42. data/lib/logstash/util/filetools.rb +186 -0
  43. data/lib/logstash/util/java_version.rb +66 -0
  44. data/lib/logstash/util/password.rb +25 -0
  45. data/lib/logstash/util/plugin_version.rb +56 -0
  46. data/lib/logstash/util/prctl.rb +10 -0
  47. data/lib/logstash/util/retryable.rb +40 -0
  48. data/lib/logstash/util/socket_peer.rb +7 -0
  49. data/lib/logstash/util/unicode_trimmer.rb +81 -0
  50. data/lib/logstash/util/worker_threads_default_printer.rb +29 -0
  51. data/lib/logstash/util/wrapped_synchronous_queue.rb +41 -0
  52. data/lib/logstash/version.rb +14 -0
  53. data/locales/en.yml +204 -0
  54. data/logstash-core.gemspec +58 -0
  55. data/spec/conditionals_spec.rb +429 -0
  56. data/spec/logstash/agent_spec.rb +85 -0
  57. data/spec/logstash/config/config_ast_spec.rb +146 -0
  58. data/spec/logstash/config/cpu_core_strategy_spec.rb +123 -0
  59. data/spec/logstash/config/defaults_spec.rb +10 -0
  60. data/spec/logstash/config/mixin_spec.rb +158 -0
  61. data/spec/logstash/environment_spec.rb +56 -0
  62. data/spec/logstash/filters/base_spec.rb +251 -0
  63. data/spec/logstash/inputs/base_spec.rb +74 -0
  64. data/spec/logstash/java_integration_spec.rb +304 -0
  65. data/spec/logstash/json_spec.rb +96 -0
  66. data/spec/logstash/output_delegator_spec.rb +144 -0
  67. data/spec/logstash/outputs/base_spec.rb +40 -0
  68. data/spec/logstash/patches_spec.rb +90 -0
  69. data/spec/logstash/pipeline_reporter_spec.rb +85 -0
  70. data/spec/logstash/pipeline_spec.rb +455 -0
  71. data/spec/logstash/plugin_spec.rb +169 -0
  72. data/spec/logstash/runner_spec.rb +68 -0
  73. data/spec/logstash/shutdown_watcher_spec.rb +113 -0
  74. data/spec/logstash/util/buftok_spec.rb +31 -0
  75. data/spec/logstash/util/charset_spec.rb +74 -0
  76. data/spec/logstash/util/defaults_printer_spec.rb +50 -0
  77. data/spec/logstash/util/java_version_spec.rb +79 -0
  78. data/spec/logstash/util/plugin_version_spec.rb +64 -0
  79. data/spec/logstash/util/unicode_trimmer_spec.rb +55 -0
  80. data/spec/logstash/util/worker_threads_default_printer_spec.rb +45 -0
  81. data/spec/logstash/util/wrapped_synchronous_queue_spec.rb +28 -0
  82. data/spec/logstash/util_spec.rb +35 -0
  83. metadata +364 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a0949964fd2c1643be200ae54e1375dfcae07698
4
+ data.tar.gz: 26c4397ec2ba5239f1c3197974ac1ccdf6acb16a
5
+ SHA512:
6
+ metadata.gz: 27425ccf12b1a2630d47ad42d06e23098f28407f9e3a9000588636927e87f1b0dbf76523f3733447676136f37e751bb98763f07335ef54314eed5251c939083e
7
+ data.tar.gz: fea0bd1ddadf63f045db39687c5512e1973dbddbda50c95166734f9cd0ebd1764f2500f8eca79d8b98084466daba963feda12b91b3b904f72b2866c2bef1a92a
@@ -0,0 +1 @@
1
+ require "logstash-core/logstash-core"
@@ -0,0 +1,3 @@
1
+ # encoding: utf-8
2
+ module LogStash
3
+ end
@@ -0,0 +1,8 @@
1
+ # encoding: utf-8
2
+
3
+ # The version of logstash core gem.
4
+ #
5
+ # Note to authors: this should not include dashes because 'gem' barfs if
6
+ # you include a dash in the version string.
7
+
8
+ LOGSTASH_CORE_VERSION = "2.2.4.snapshot1"
@@ -0,0 +1,391 @@
1
+ # encoding: utf-8
2
+ require "clamp" # gem 'clamp'
3
+ require "logstash/environment"
4
+ require "logstash/errors"
5
+ require "logstash/config/cpu_core_strategy"
6
+ require "uri"
7
+ require "net/http"
8
+ require "logstash/pipeline"
9
+ LogStash::Environment.load_locale!
10
+
11
+ class LogStash::Agent < Clamp::Command
12
+
13
+ DEFAULT_INPUT = "input { stdin { type => stdin } }"
14
+ DEFAULT_OUTPUT = "output { stdout { codec => rubydebug } }"
15
+
16
+ option ["-f", "--config"], "CONFIG_PATH",
17
+ I18n.t("logstash.agent.flag.config"),
18
+ :attribute_name => :config_path
19
+
20
+ option "-e", "CONFIG_STRING",
21
+ I18n.t("logstash.agent.flag.config-string",
22
+ :default_input => DEFAULT_INPUT, :default_output => DEFAULT_OUTPUT),
23
+ :default => "", :attribute_name => :config_string
24
+
25
+ option ["-w", "--pipeline-workers"], "COUNT",
26
+ I18n.t("logstash.agent.flag.pipeline-workers"),
27
+ :attribute_name => :pipeline_workers,
28
+ :default => LogStash::Pipeline::DEFAULT_SETTINGS[:default_pipeline_workers]
29
+
30
+ option ["-b", "--pipeline-batch-size"], "SIZE",
31
+ I18n.t("logstash.agent.flag.pipeline-batch-size"),
32
+ :attribute_name => :pipeline_batch_size,
33
+ :default => LogStash::Pipeline::DEFAULT_SETTINGS[:pipeline_batch_size]
34
+
35
+ option ["-u", "--pipeline-batch-delay"], "DELAY_IN_MS",
36
+ I18n.t("logstash.agent.flag.pipeline-batch-delay"),
37
+ :attribute_name => :pipeline_batch_delay,
38
+ :default => LogStash::Pipeline::DEFAULT_SETTINGS[:pipeline_batch_delay]
39
+
40
+ option ["--filterworkers"], "COUNT",
41
+ I18n.t("logstash.agent.flag.filterworkers"),
42
+ :attribute_name => :filter_workers
43
+
44
+ option ["-l", "--log"], "FILE",
45
+ I18n.t("logstash.agent.flag.log"),
46
+ :attribute_name => :log_file
47
+
48
+ # Old support for the '-v' flag'
49
+ option "-v", :flag,
50
+ I18n.t("logstash.agent.flag.verbosity"),
51
+ :attribute_name => :verbosity, :multivalued => true
52
+
53
+ option "--quiet", :flag, I18n.t("logstash.agent.flag.quiet")
54
+ option "--verbose", :flag, I18n.t("logstash.agent.flag.verbose")
55
+ option "--debug", :flag, I18n.t("logstash.agent.flag.debug")
56
+
57
+ option "--debug-config", :flag,
58
+ I18n.t("logstash.runner.flag.debug_config"),
59
+ :attribute_name => :debug_config, :default => false
60
+
61
+ option ["-V", "--version"], :flag,
62
+ I18n.t("logstash.agent.flag.version")
63
+
64
+ option ["-p", "--pluginpath"] , "PATH",
65
+ I18n.t("logstash.agent.flag.pluginpath"),
66
+ :multivalued => true,
67
+ :attribute_name => :plugin_paths
68
+
69
+ option ["-t", "--configtest"], :flag,
70
+ I18n.t("logstash.agent.flag.configtest"),
71
+ :attribute_name => :config_test
72
+
73
+ option "--[no-]allow-unsafe-shutdown", :flag,
74
+ I18n.t("logstash.agent.flag.unsafe_shutdown"),
75
+ :attribute_name => :unsafe_shutdown,
76
+ :default => false
77
+
78
+ def initialize(*args)
79
+ super(*args)
80
+ @pipeline_settings ||= { :pipeline_id => "base" }
81
+ end
82
+
83
+ def pipeline_workers=(pipeline_workers_value)
84
+ @pipeline_settings[:pipeline_workers] = validate_positive_integer(pipeline_workers_value)
85
+ end
86
+
87
+ def pipeline_batch_size=(pipeline_batch_size_value)
88
+ @pipeline_settings[:pipeline_batch_size] = validate_positive_integer(pipeline_batch_size_value)
89
+ end
90
+
91
+ def pipeline_batch_delay=(pipeline_batch_delay_value)
92
+ @pipeline_settings[:pipeline_batch_delay] = validate_positive_integer(pipeline_batch_delay_value)
93
+ end
94
+
95
+ def validate_positive_integer(str_arg)
96
+ int_arg = str_arg.to_i
97
+ if str_arg !~ /^\d+$/ || int_arg < 1
98
+ raise ArgumentError, "Expected a positive integer, got '#{str_arg}'"
99
+ end
100
+
101
+ int_arg
102
+ end
103
+
104
+ # Emit a warning message.
105
+ def warn(message)
106
+ # For now, all warnings are fatal.
107
+ raise LogStash::ConfigurationError, message
108
+ end # def warn
109
+
110
+ # Emit a failure message and abort.
111
+ def fail(message)
112
+ raise LogStash::ConfigurationError, message
113
+ end # def fail
114
+
115
+ # Run the agent. This method is invoked after clamp parses the
116
+ # flags given to this program.
117
+ def execute
118
+ require "logstash/pipeline"
119
+ require "cabin" # gem 'cabin'
120
+ require "logstash/plugin"
121
+ @logger = Cabin::Channel.get(LogStash)
122
+
123
+ LogStash::ShutdownWatcher.unsafe_shutdown = unsafe_shutdown?
124
+ LogStash::ShutdownWatcher.logger = @logger
125
+
126
+ if version?
127
+ show_version
128
+ return 0
129
+ end
130
+
131
+ # temporarily send logs to stdout as well if a --log is specified
132
+ # and stdout appears to be a tty
133
+ show_startup_errors = log_file && STDOUT.tty?
134
+
135
+ if show_startup_errors
136
+ stdout_logs = @logger.subscribe(STDOUT)
137
+ end
138
+ configure
139
+
140
+
141
+ if filter_workers
142
+ @logger.warn("--filter-workers is deprecated! Please use --pipeline-workers or -w. This setting will be removed in the next major version!")
143
+ self.pipeline_workers = filter_workers
144
+ end
145
+
146
+ # You must specify a config_string or config_path
147
+ if @config_string.nil? && @config_path.nil?
148
+ fail(help + "\n" + I18n.t("logstash.agent.missing-configuration"))
149
+ end
150
+
151
+ @config_string = @config_string.to_s
152
+
153
+ if @config_path
154
+ # Append the config string.
155
+ # This allows users to provide both -f and -e flags. The combination
156
+ # is rare, but useful for debugging.
157
+ @config_string = @config_string + load_config(@config_path)
158
+ else
159
+ # include a default stdin input if no inputs given
160
+ if @config_string !~ /input *{/
161
+ @config_string += DEFAULT_INPUT
162
+ end
163
+ # include a default stdout output if no outputs given
164
+ if @config_string !~ /output *{/
165
+ @config_string += DEFAULT_OUTPUT
166
+ end
167
+ end
168
+
169
+
170
+ begin
171
+ pipeline = LogStash::Pipeline.new(@config_string, @pipeline_settings)
172
+ rescue LoadError => e
173
+ fail("Configuration problem.")
174
+ end
175
+
176
+ # Make SIGINT shutdown the pipeline.
177
+ sigint_id = Stud::trap("INT") do
178
+
179
+ if @interrupted_once
180
+ @logger.fatal(I18n.t("logstash.agent.forced_sigint"))
181
+ exit
182
+ else
183
+ @logger.warn(I18n.t("logstash.agent.sigint"))
184
+ Thread.new(@logger) {|logger| sleep 5; logger.warn(I18n.t("logstash.agent.slow_shutdown")) }
185
+ @interrupted_once = true
186
+ shutdown(pipeline)
187
+ end
188
+ end
189
+
190
+ # Make SIGTERM shutdown the pipeline.
191
+ sigterm_id = Stud::trap("TERM") do
192
+ @logger.warn(I18n.t("logstash.agent.sigterm"))
193
+ shutdown(pipeline)
194
+ end
195
+
196
+ Stud::trap("HUP") do
197
+ @logger.info(I18n.t("logstash.agent.sighup"))
198
+ configure_logging(log_file)
199
+ end
200
+
201
+ # Stop now if we are only asking for a config test.
202
+ if config_test?
203
+ @logger.terminal "Configuration OK"
204
+ return
205
+ end
206
+
207
+ @logger.unsubscribe(stdout_logs) if show_startup_errors
208
+
209
+ # TODO(sissel): Get pipeline completion status.
210
+ pipeline.run
211
+ return 0
212
+ rescue LogStash::ConfigurationError => e
213
+ @logger.unsubscribe(stdout_logs) if show_startup_errors
214
+ @logger.error I18n.t("logstash.agent.error", :error => e)
215
+ if !config_test?
216
+ @logger.info I18n.t("logstash.agent.configtest-flag-information")
217
+ end
218
+ return 1
219
+ rescue => e
220
+ @logger.unsubscribe(stdout_logs) if show_startup_errors
221
+ @logger.warn(I18n.t("oops"), :error => e, :class => e.class.name, :backtrace => e.backtrace)
222
+ return 1
223
+ ensure
224
+ @log_fd.close if @log_fd
225
+ Stud::untrap("INT", sigint_id) unless sigint_id.nil?
226
+ Stud::untrap("TERM", sigterm_id) unless sigterm_id.nil?
227
+ end # def execute
228
+
229
+ def shutdown(pipeline)
230
+ pipeline.shutdown do
231
+ ::LogStash::ShutdownWatcher.start(pipeline)
232
+ end
233
+ end
234
+
235
+ def show_version
236
+ show_version_logstash
237
+
238
+ if [:info, :debug].include?(verbosity?) || debug? || verbose?
239
+ show_version_ruby
240
+ show_version_java if LogStash::Environment.jruby?
241
+ show_gems if [:debug].include?(verbosity?) || debug?
242
+ end
243
+ end # def show_version
244
+
245
+ def show_version_logstash
246
+ require "logstash/version"
247
+ puts "logstash #{LOGSTASH_VERSION}"
248
+ end # def show_version_logstash
249
+
250
+ def show_version_ruby
251
+ puts RUBY_DESCRIPTION
252
+ end # def show_version_ruby
253
+
254
+ def show_version_java
255
+ properties = java.lang.System.getProperties
256
+ puts "java #{properties["java.version"]} (#{properties["java.vendor"]})"
257
+ puts "jvm #{properties["java.vm.name"]} / #{properties["java.vm.version"]}"
258
+ end # def show_version_java
259
+
260
+ def show_gems
261
+ require "rubygems"
262
+ Gem::Specification.each do |spec|
263
+ puts "gem #{spec.name} #{spec.version}"
264
+ end
265
+ end # def show_gems
266
+
267
+ # Do any start-time configuration.
268
+ #
269
+ # Log file stuff, plugin path checking, etc.
270
+ def configure
271
+ @pipeline_settings[:debug_config] = debug_config?
272
+ configure_logging(log_file)
273
+ configure_plugin_paths(plugin_paths)
274
+ end # def configure
275
+
276
+ # Point logging at a specific path.
277
+ def configure_logging(path)
278
+ # Set with the -v (or -vv...) flag
279
+ if quiet?
280
+ @logger.level = :error
281
+ elsif verbose?
282
+ @logger.level = :info
283
+ elsif debug?
284
+ @logger.level = :debug
285
+ else
286
+ # Old support for the -v and -vv stuff.
287
+ if verbosity? && verbosity?.any?
288
+ # this is an array with length of how many times the flag is given
289
+ if verbosity?.length == 1
290
+ @logger.warn("The -v flag is deprecated and will be removed in a future release. You should use --verbose instead.")
291
+ @logger.level = :info
292
+ else
293
+ @logger.warn("The -vv flag is deprecated and will be removed in a future release. You should use --debug instead.")
294
+ @logger.level = :debug
295
+ end
296
+ else
297
+ @logger.level = :warn
298
+ end
299
+ end
300
+
301
+ if log_file
302
+ # TODO(sissel): Implement file output/rotation in Cabin.
303
+ # TODO(sissel): Catch exceptions, report sane errors.
304
+ begin
305
+ @log_fd.close if @log_fd
306
+ @log_fd = File.new(path, "a")
307
+ rescue => e
308
+ fail(I18n.t("logstash.agent.configuration.log_file_failed",
309
+ :path => path, :error => e))
310
+ end
311
+
312
+ puts "Sending logstash logs to #{path}."
313
+ @logger.unsubscribe(@logger_subscription) if @logger_subscription
314
+ @logger_subscription = @logger.subscribe(@log_fd)
315
+ else
316
+ @logger.subscribe(STDOUT)
317
+ end
318
+
319
+ # TODO(sissel): redirect stdout/stderr to the log as well
320
+ # http://jira.codehaus.org/browse/JRUBY-7003
321
+ end # def configure_logging
322
+
323
+ # add the given paths for ungemified/bare plugins lookups
324
+ # @param paths [String, Array<String>] plugins path string or list of path strings to add
325
+ def configure_plugin_paths(paths)
326
+ Array(paths).each do |path|
327
+ fail(I18n.t("logstash.agent.configuration.plugin_path_missing", :path => path)) unless File.directory?(path)
328
+ LogStash::Environment.add_plugin_path(path)
329
+ end
330
+ end
331
+
332
+ def load_config(path)
333
+ begin
334
+ uri = URI.parse(path)
335
+
336
+ case uri.scheme
337
+ when nil then
338
+ local_config(path)
339
+ when /http/ then
340
+ fetch_config(uri)
341
+ when "file" then
342
+ local_config(uri.path)
343
+ else
344
+ fail(I18n.t("logstash.agent.configuration.scheme-not-supported", :path => path))
345
+ end
346
+ rescue URI::InvalidURIError
347
+ # fallback for windows.
348
+ # if the parsing of the file failed we assume we can reach it locally.
349
+ # some relative path on windows arent parsed correctly (.\logstash.conf)
350
+ local_config(path)
351
+ end
352
+ end
353
+
354
+ def local_config(path)
355
+ path = File.expand_path(path)
356
+ path = File.join(path, "*") if File.directory?(path)
357
+
358
+ if Dir.glob(path).length == 0
359
+ fail(I18n.t("logstash.agent.configuration.file-not-found", :path => path))
360
+ end
361
+
362
+ config = ""
363
+ encoding_issue_files = []
364
+ Dir.glob(path).sort.each do |file|
365
+ next unless File.file?(file)
366
+ if file.match(/~$/)
367
+ @logger.debug("NOT reading config file because it is a temp file", :file => file)
368
+ next
369
+ end
370
+ @logger.debug("Reading config file", :file => file)
371
+ cfg = File.read(file)
372
+ if !cfg.ascii_only? && !cfg.valid_encoding?
373
+ encoding_issue_files << file
374
+ end
375
+ config << cfg + "\n"
376
+ end
377
+ if (encoding_issue_files.any?)
378
+ fail("The following config files contains non-ascii characters but are not UTF-8 encoded #{encoding_issue_files}")
379
+ end
380
+ return config
381
+ end # def load_config
382
+
383
+ def fetch_config(uri)
384
+ begin
385
+ Net::HTTP.get(uri) + "\n"
386
+ rescue Exception => e
387
+ fail(I18n.t("logstash.agent.configuration.fetch-failed", :path => uri.to_s, :message => e.message))
388
+ end
389
+ end
390
+
391
+ end # class LogStash::Agent
@@ -0,0 +1,50 @@
1
+ # encoding: utf-8
2
+ require "logstash/namespace"
3
+ require "logstash/event"
4
+ require "logstash/plugin"
5
+ require "logstash/logging"
6
+
7
+ # This is the base class for logstash codecs.
8
+ module LogStash::Codecs; class Base < LogStash::Plugin
9
+ include LogStash::Config::Mixin
10
+ config_name "codec"
11
+
12
+ def initialize(params={})
13
+ super
14
+ config_init(@params)
15
+ register if respond_to?(:register)
16
+ end
17
+
18
+ public
19
+ def decode(data)
20
+ raise "#{self.class}#decode must be overidden"
21
+ end # def decode
22
+
23
+ alias_method :<<, :decode
24
+
25
+ public
26
+ def encode(event)
27
+ raise "#{self.class}#encode must be overidden"
28
+ end # def encode
29
+
30
+ public
31
+ def close; end;
32
+
33
+ # @param block [Proc(event, data)] the callback proc passing the original event and the encoded event
34
+ public
35
+ def on_event(&block)
36
+ @on_event = block
37
+ end
38
+
39
+ public
40
+ def flush(&block)
41
+ # does nothing by default.
42
+ # if your codec needs a flush method (like you are spooling things)
43
+ # you must implement this.
44
+ end
45
+
46
+ public
47
+ def clone
48
+ return self.class.new(params)
49
+ end
50
+ end; end # class LogStash::Codecs::Base