logstash-lite 0.2.20101201111523 → 0.2.20101207114354

Sign up to get free protection for your applications and to get access to all the features.
data/bin/logstash CHANGED
@@ -8,7 +8,7 @@ require "logstash/agent"
8
8
  require "optparse"
9
9
  require "yaml"
10
10
 
11
- Settings = Struct.new(:config_file, :daemonize)
11
+ Settings = Struct.new(:config_file, :daemonize, :logfile)
12
12
 
13
13
  settings = Settings.new
14
14
  settings.daemonize = false
@@ -23,9 +23,13 @@ opts = OptionParser.new do |opts|
23
23
  settings.config_file = arg
24
24
  end
25
25
 
26
- #opts.on("-d", "--daemonize", "Daemonize (default is run in foreground)") do
27
- #settings.daemonize = true
28
- #end
26
+ opts.on("-d", "--daemonize", "Daemonize (default is run in foreground)") do
27
+ settings.daemonize = true
28
+ end
29
+
30
+ opts.on("-l", "--log FILE", "Log to a given path. Default is stdout.") do |path|
31
+ settings.logfile = path
32
+ end
29
33
  end
30
34
 
31
35
  begin
@@ -33,7 +37,6 @@ begin
33
37
  if settings.config_file == "" or settings.config_file == nil
34
38
  raise "No config file given. (missing -f or --config flag?)"
35
39
  end
36
-
37
40
  rescue
38
41
  $stderr.puts "#{progname}: #{$!}"
39
42
  $stderr.puts opts
@@ -52,5 +55,15 @@ rescue => e
52
55
  exit 1
53
56
  end
54
57
 
58
+ if settings.daemonize
59
+ if Process.fork == nil
60
+ Process.setsid
61
+ else
62
+ exit(0)
63
+ end
64
+ end
55
65
  agent = LogStash::Agent.new(config)
56
- agent.run
66
+ if settings.logfile
67
+ agent.log_to(settings.logfile)
68
+ end
69
+ agent.run
@@ -0,0 +1,31 @@
1
+ ---
2
+ inputs:
3
+ linux-syslog:
4
+ - /var/log/messages
5
+ filters:
6
+ - grok:
7
+ linux-syslog: # for logs of type 'linux-syslog'
8
+ patterns:
9
+ - %{SYSLOGLINE}
10
+ - date:
11
+ linux-syslog:
12
+ timestamp: "%b %e %H:%M:%S"
13
+ timestamp8601: ISO8601
14
+ - grep:
15
+ linux-syslog:
16
+ - match:
17
+ message: test
18
+ add_fields:
19
+ nagios_alert: test_alert
20
+ add_tags:
21
+ - nagios
22
+ - test
23
+ - match:
24
+ message: (?i)foo.*bar
25
+ program: test
26
+ add_fields:
27
+ nagios_alert: foo_alert
28
+ add_tags:
29
+ - nagios
30
+ outputs:
31
+ - stdout:///
@@ -15,4 +15,5 @@ inputs:
15
15
  unknown:
16
16
  - /b/randomdata
17
17
  outputs:
18
- - amqp://localhost/topic/rawlogs
18
+ #- amqp://localhost/topic/rawlogs
19
+ - websocket://0.0.0.0:3232/
@@ -0,0 +1,7 @@
1
+ ---
2
+ inputs:
3
+ stomp:
4
+ - stomp://logs:password@localhost:6163/topic/logs
5
+ outputs:
6
+ - stdout:///
7
+
@@ -0,0 +1,7 @@
1
+ ---
2
+ inputs:
3
+ tail-syslog:
4
+ - /var/log/syslog
5
+ outputs:
6
+ - stomp://logs:password@localhost:6163/topic/logs
7
+
@@ -14,7 +14,7 @@ class LogStash::Agent
14
14
  attr_reader :filters
15
15
 
16
16
  def initialize(config)
17
- @logger = LogStash::Logger.new(STDERR)
17
+ log_to(STDERR)
18
18
 
19
19
  @config = config
20
20
  @outputs = []
@@ -26,6 +26,11 @@ class LogStash::Agent
26
26
  # - where to ship to
27
27
  end # def initialize
28
28
 
29
+ public
30
+ def log_to(target)
31
+ @logger = LogStash::Logger.new(target)
32
+ end # def log_to
33
+
29
34
  # Register any event handlers with EventMachine
30
35
  # Technically, this agent could listen for anything (files, sockets, amqp,
31
36
  # stomp, etc).
@@ -55,6 +60,7 @@ class LogStash::Agent
55
60
  urls.each do |url|
56
61
  @logger.debug("Using input #{url} of type #{type}")
57
62
  input = LogStash::Inputs.from_url(url, type) { |event| receive(event) }
63
+ input.logger = @logger
58
64
  input.register
59
65
  @inputs << input
60
66
  end
@@ -67,6 +73,7 @@ class LogStash::Agent
67
73
  name, value = filter
68
74
  @logger.debug("Using filter #{name} => #{value.inspect}")
69
75
  filter = LogStash::Filters.from_name(name, value)
76
+ filter.logger = @logger
70
77
  filter.register
71
78
  @filters << filter
72
79
  end # each filter
@@ -76,6 +83,7 @@ class LogStash::Agent
76
83
  @config["outputs"].each do |url|
77
84
  @logger.debug("Using output #{url}")
78
85
  output = LogStash::Outputs.from_url(url)
86
+ output.logger = @logger
79
87
  output.register
80
88
  @outputs << output
81
89
  end # each output
@@ -135,6 +143,10 @@ class LogStash::Agent
135
143
  @sigchannel.push(:USR1)
136
144
  end
137
145
 
146
+ Signal.trap("INT") do
147
+ @sigchannel.push(:INT)
148
+ end
149
+
138
150
  @sigchannel.subscribe do |msg|
139
151
  case msg
140
152
  when :USR1
@@ -153,7 +165,12 @@ class LogStash::Agent
153
165
  counts.sort { |a,b| a[1] <=> b[1] or a[0] <=> b[0] }.each do |key, value|
154
166
  @logger.info("Class: [#{value}] #{key}")
155
167
  end
156
- end
157
- end
158
- end
168
+ when :INT
169
+ @logger.warn("SIGINT received. Shutting down.")
170
+ EventMachine::stop_event_loop
171
+ # TODO(sissel): Should have input/output/filter register shutdown
172
+ # hooks.
173
+ end # case msg
174
+ end # @sigchannel.subscribe
175
+ end # def sighandler
159
176
  end # class LogStash::Components::Agent
@@ -2,6 +2,7 @@ require "logstash/namespace"
2
2
  require "logstash/logging"
3
3
 
4
4
  class LogStash::Filters::Base
5
+ attr_accessor :logger
5
6
  def initialize(config = {})
6
7
  @logger = LogStash::Logger.new(STDERR)
7
8
  @config = config
@@ -0,0 +1,92 @@
1
+ require "logstash/filters/base"
2
+
3
+ class LogStash::Filters::Grep < LogStash::Filters::Base
4
+ def initialize(config = {})
5
+ super
6
+
7
+ @config = config
8
+ end # def initialize
9
+
10
+ def register
11
+ @config.each do |type, matches|
12
+ if ! matches.is_a?(Array)
13
+ @logger.warn("grep: #{type} misconfigured; must be an array")
14
+ next
15
+ end
16
+
17
+ matches.each_index do |i|
18
+ match = matches[i]
19
+ if ! match.member?("match")
20
+ @logger.warn(["grep: #{type}/#{i}: no 'match' section defined", match])
21
+ next
22
+ end
23
+ match["match"].each do |field, re_str|
24
+ re = Regexp.new(re_str)
25
+ @config[type][i]["match"][field] = re
26
+ @logger.debug(["grep: #{type}/#{i}/#{field}", re_str, re])
27
+ end
28
+ end # matches.each
29
+ end # @config.each
30
+ end # def register
31
+
32
+ def filter(event)
33
+ config = @config[event.type]
34
+ if not config
35
+ @logger.debug("grep: skipping type #{event.type} from #{event.source}")
36
+ return
37
+ end
38
+
39
+ @logger.debug(["Running grep filter", event, config])
40
+ matched = false
41
+ config.each do |match|
42
+ if ! match["match"]
43
+ @logging.debug(["Skipping match object, no match key", match])
44
+ next
45
+ end
46
+
47
+ # For each match object, we have to match everything in order to
48
+ # apply any fields/tags.
49
+ match_count = 0
50
+ match["match"].each do |field, re|
51
+ next unless event[field]
52
+
53
+ event[field].each do |value|
54
+ next unless re.match(value)
55
+ @logger.debug("grep matched on field #{field}")
56
+ match_count += 1
57
+ end
58
+ end # match["match"].each
59
+
60
+ if match_count == match["match"].length
61
+ matched = true
62
+ @logger.debug("matched all fields (#{match_count})")
63
+
64
+ if match["add_fields"]
65
+ match["add_fields"].each do |field, value|
66
+ event[field] ||= []
67
+ event[field] << value
68
+ @logger.debug("grep: adding #{value} to field #{field}")
69
+ end
70
+ end # if match["add_fields"]
71
+
72
+ if match["add_tags"]
73
+ match["add_tags"].each do |tag|
74
+ event.tags << tag
75
+ @logger.debug("grep: adding tag #{tag}")
76
+ end
77
+ end # if match["add_tags"]
78
+ else
79
+ @logger.debug("match block failed " \
80
+ "(#{match_count}/#{match["match"].length} matches)")
81
+ end # match["match"].each
82
+ end # config.each
83
+
84
+ if not matched
85
+ @logger.debug("grep: dropping event, no matches")
86
+ event.cancel
87
+ return
88
+ end
89
+
90
+ @logger.debug(["Event after grep filter", event.to_hash])
91
+ end # def filter
92
+ end # class LogStash::Filters::Grep
@@ -4,6 +4,7 @@ require "logstash/logging"
4
4
  require "uri"
5
5
 
6
6
  class LogStash::Inputs::Base
7
+ attr_accessor :logger
7
8
  def initialize(url, type, config={}, &block)
8
9
  @logger = LogStash::Logger.new(STDERR)
9
10
  @url = url
@@ -0,0 +1,27 @@
1
+ require "logstash/inputs/base"
2
+ require "logstash/stomp/handler"
3
+
4
+ class LogStash::Inputs::Stomp < LogStash::Inputs::Base
5
+
6
+ class InputHandler < LogStash::Stomp::Handler
7
+ def receive_msg(message)
8
+ super
9
+
10
+ unless message.command == "CONNECTED"
11
+ event = LogStash::Event.from_json(message.body)
12
+ @input.receive(event)
13
+ end
14
+ end # def receive_msg
15
+ end # class StompHandler
16
+
17
+ def initialize(url, config={}, &block)
18
+ super
19
+
20
+ @logger.debug(["Connecting", { :url => @url }])
21
+ end # def initialize
22
+
23
+ def register
24
+ @logger.info(["Registering input", { :url => @url}])
25
+ EventMachine::connect(@url.host, @url.port, InputHandler, self, @logger, @url)
26
+ end # def register
27
+ end # class LogStash::Inputs::Amqp
@@ -5,6 +5,7 @@ require "cgi"
5
5
  require "uri"
6
6
 
7
7
  class LogStash::Outputs::Base
8
+ attr_accessor :logger
8
9
  def initialize(url, config={}, &block)
9
10
  @url = url
10
11
  @url = URI.parse(url) if url.is_a? String
@@ -0,0 +1,22 @@
1
+ require "logstash/outputs/base"
2
+ require "logstash/stomp/handler"
3
+
4
+ class LogStash::Outputs::Stomp < LogStash::Outputs::Base
5
+ attr_reader :url
6
+
7
+ def initialize(url, config={}, &block)
8
+ super
9
+
10
+ @logger.debug(["Initialize", { :url => @url }])
11
+ end # def initialize
12
+
13
+ def register
14
+ @logger.info(["Registering output", { :url => @url }])
15
+ @connection = EventMachine::connect(@url.host, @url.port, LogStash::Stomp::Handler, self, @logger, @url)
16
+ end # def register
17
+
18
+ def receive(event)
19
+ @logger.debug(["Sending event", { :url => @url, :event => event }])
20
+ @connection.send(@url.path, event.to_json)
21
+ end # def receive
22
+ end # class LogStash::Outputs::Stomp
@@ -9,10 +9,10 @@ class LogStash::Outputs::Websocket < LogStash::Outputs::Base
9
9
  def register
10
10
  @channel = EventMachine::Channel.new
11
11
  @subscribers = 0
12
- host = (@url.host or "0.0.0.0")
13
- port = (@url.port or 3000)
12
+ @url.host = (@url.host or "0.0.0.0")
13
+ @url.port = (@url.port or 3232)
14
14
  @logger.info("Registering websocket on #{@url}")
15
- EventMachine::WebSocket.start(:host => host, :port => port) do |ws|
15
+ EventMachine::WebSocket.start(:host => @url.host, :port => @url.port) do |ws|
16
16
  ws.onopen do
17
17
  @subscribers += 1
18
18
  @logger.info("New #{self.class.name} connection")
@@ -0,0 +1,34 @@
1
+ # Base of Stomp Handler
2
+ # it handles connecting and subscribing to the stomp broker which
3
+ # is used in both stomp input and output
4
+ class LogStash::Stomp
5
+ class Handler < EventMachine::Connection
6
+ include EM::Protocols::Stomp
7
+
8
+ def initialize(*args)
9
+ super
10
+
11
+ @input = args[0]
12
+ @logger = args[1]
13
+ @url = args[2]
14
+ end # def initialize
15
+
16
+ def connection_completed
17
+ @logger.debug("Connected")
18
+ connect :login => @url.user, :passcode => @url.password
19
+ end # def connection_completed
20
+
21
+ def unbind
22
+ @logger.error(["Error when connecting to stomp broker", { :url => @url }])
23
+ end # def unbind
24
+
25
+ def receive_msg(message)
26
+ @logger.debug(["receiving message", { :msg => message }])
27
+ if message.command == "CONNECTED"
28
+ @logger.debug(["subscribing to", { :path => @url.path }])
29
+ subscribe @url.path
30
+ return
31
+ end
32
+ end # def receive_msg
33
+ end # class Handler
34
+ end # class LogStash::Stomp
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-lite
3
3
  version: !ruby/object:Gem::Version
4
- hash: 40202402223057
4
+ hash: 40202414228723
5
5
  prerelease: false
6
6
  segments:
7
7
  - 0
8
8
  - 2
9
- - 20101201111523
10
- version: 0.2.20101201111523
9
+ - 20101207114354
10
+ version: 0.2.20101207114354
11
11
  platform: ruby
12
12
  authors:
13
13
  - Jordan Sissel
@@ -15,7 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2010-12-01 00:00:00 -08:00
18
+ date: 2010-12-07 00:00:00 -08:00
19
19
  default_executable:
20
20
  dependencies:
21
21
  - !ruby/object:Gem::Dependency
@@ -94,6 +94,7 @@ files:
94
94
  - lib/logstash/inputs/file.rb
95
95
  - lib/logstash/inputs/base.rb
96
96
  - lib/logstash/inputs/amqp.rb
97
+ - lib/logstash/inputs/stomp.rb
97
98
  - lib/logstash/inputs/tcp.rb
98
99
  - lib/logstash/outputs/gelf.rb
99
100
  - lib/logstash/outputs/elasticsearch.rb
@@ -103,14 +104,17 @@ files:
103
104
  - lib/logstash/outputs/websocket.rb
104
105
  - lib/logstash/outputs/base.rb
105
106
  - lib/logstash/outputs/amqp.rb
107
+ - lib/logstash/outputs/stomp.rb
106
108
  - lib/logstash/outputs/tcp.rb
107
109
  - lib/logstash/namespace.rb
108
110
  - lib/logstash/time.rb
109
111
  - lib/logstash/filters.rb
110
112
  - lib/logstash/outputs.rb
113
+ - lib/logstash/stomp/handler.rb
111
114
  - lib/logstash/filters/grokdiscovery.rb
112
115
  - lib/logstash/filters/multiline.rb
113
116
  - lib/logstash/filters/grok.rb
117
+ - lib/logstash/filters/grep.rb
114
118
  - lib/logstash/filters/base.rb
115
119
  - lib/logstash/filters/field.rb
116
120
  - lib/logstash/filters/date.rb
@@ -208,6 +212,7 @@ files:
208
212
  - etc/tograylog.yaml
209
213
  - etc/logstash-elasticsearch-rabbitmq-river.yaml
210
214
  - etc/logstash-reader.yaml
215
+ - etc/logstash-stomp-input.yaml
211
216
  - etc/logstash-parser.yaml
212
217
  - etc/logstash-mongodb-storage.yaml
213
218
  - etc/logstash-standalone.yaml
@@ -216,7 +221,9 @@ files:
216
221
  - etc/redhat/logstash.spec
217
222
  - etc/redhat/logstash
218
223
  - etc/redhat/logstash-agent
224
+ - etc/logstash-stomp.yaml
219
225
  - etc/prod.yaml
226
+ - etc/logstash-grep.yaml
220
227
  - etc/logstash-shipper.yaml
221
228
  - patterns/linux-syslog
222
229
  - patterns/haproxy