logstash-lite 0.2.20101120024757 → 0.2.20101123133737

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,56 @@
1
+ #!/usr/bin/env ruby
2
+ $: << "lib"
3
+
4
+ require "rubygems"
5
+ require "logstash/agent"
6
+ require "yaml"
7
+
8
+ collector_config = YAML.load <<"YAML"
9
+ ---
10
+ inputs:
11
+ foo:
12
+ - internal:///
13
+ outputs:
14
+ - amqp://localhost/topic/logstash/testing
15
+ YAML
16
+
17
+ receiver_config = YAML.load <<"YAML"
18
+ ---
19
+ inputs:
20
+ foo:
21
+ - amqp://localhost/topic/logstash/testing
22
+ outputs:
23
+ - internal:///
24
+ YAML
25
+
26
+ collector_agent = LogStash::Agent.new(collector_config)
27
+ receiver_agent = LogStash::Agent.new(receiver_config)
28
+
29
+ data = ["hello world", "foo", "bar"]
30
+
31
+ EventMachine::run do
32
+ receiver_agent.register
33
+ collector_agent.register
34
+
35
+ EM::next_tick do
36
+ # Register output callback on the receiver
37
+ receiver_agent.outputs\
38
+ .find { |o| o.is_a?(LogStash::Outputs::Internal) }\
39
+ .callback do |event|
40
+ puts event
41
+ #puts expect.first == event.message
42
+ #expect.shift
43
+ #agent.stop
44
+ end
45
+
46
+ EM::next_tick do
47
+ # Send input to the collector
48
+ expect = data.clone
49
+ input = collector_agent.inputs\
50
+ .find { |i| i.is_a?(LogStash::Inputs::Internal) }
51
+ channel = input.channel
52
+ data.each { |message| channel.push(message) }
53
+ end
54
+ end
55
+ end
56
+
@@ -9,6 +9,9 @@ require "logstash/logging"
9
9
  # Collect logs, ship them out.
10
10
  class LogStash::Agent
11
11
  attr_reader :config
12
+ attr_reader :inputs
13
+ attr_reader :outputs
14
+ attr_reader :filters
12
15
 
13
16
  def initialize(config)
14
17
  @logger = LogStash::Logger.new(STDERR)
@@ -26,7 +29,7 @@ class LogStash::Agent
26
29
  # Register any event handlers with EventMachine
27
30
  # Technically, this agent could listen for anything (files, sockets, amqp,
28
31
  # stomp, etc).
29
- protected
32
+ public
30
33
  def register
31
34
  # TODO(sissel): warn when no inputs and no outputs are defined.
32
35
  # TODO(sissel): Refactor this madness into a config lib
@@ -80,12 +83,20 @@ class LogStash::Agent
80
83
  end # def register
81
84
 
82
85
  public
83
- def run
86
+ def run(&block)
84
87
  EventMachine.run do
85
88
  self.register
89
+ yield if block_given?
86
90
  end # EventMachine.run
87
91
  end # def run
88
92
 
93
+ public
94
+ def stop
95
+ # TODO(sissel): Stop inputs, fluch outputs, wait for finish,
96
+ # then stop the event loop
97
+ EventMachine.stop_event_loop
98
+ end
99
+
89
100
  protected
90
101
  def filter(event)
91
102
  @filters.each do |f|
@@ -1,5 +1,6 @@
1
1
 
2
2
  require "logstash/namespace"
3
+ require "logstash/ruby_fixes"
3
4
  require "uri"
4
5
 
5
6
  module LogStash::Inputs
@@ -23,7 +23,7 @@ class LogStash::Inputs::Amqp < LogStash::Inputs::Base
23
23
  end
24
24
 
25
25
  def register
26
- @logger.info("Registering #{@url}")
26
+ @logger.info("Registering input #{@url}")
27
27
  @amqp = AMQP.connect(:host => @url.host)
28
28
  @mq = MQ.new(@amqp)
29
29
  @target = nil
@@ -0,0 +1,36 @@
1
+
2
+ require "logstash/inputs/base"
3
+ require "eventmachine-tail"
4
+ require "socket" # for Socket.gethostname
5
+
6
+ class LogStash::Inputs::Internal < LogStash::Inputs::Base
7
+ attr_reader :channel
8
+
9
+ def initialize(url, type, config={}, &block)
10
+ super
11
+
12
+ # Default host to the machine's hostname if it's not set
13
+ @url.host ||= Socket.gethostname
14
+ @channel = EventMachine::Channel.new
15
+ end
16
+
17
+ def register
18
+ @logger.info("Registering input #{@url}")
19
+ @channel.subscribe do |event|
20
+ receive(event)
21
+ end
22
+ end # def register
23
+
24
+ def receive(event)
25
+ if !event.is_a?(LogStash::Event)
26
+ event = LogStash::Event.new({
27
+ "@message" => event,
28
+ "@type" => @type,
29
+ "@tags" => @tags.clone,
30
+ "@source" => @url,
31
+ })
32
+ end
33
+ @logger.debug(["Got event", event])
34
+ @callback.call(event)
35
+ end # def receive
36
+ end # class LogStash::Inputs::Internal
@@ -1,4 +1,5 @@
1
1
  require "logstash/namespace"
2
+ require "logstash/ruby_fixes"
2
3
  require "logger"
3
4
 
4
5
  class LogStash::Logger < Logger
@@ -18,16 +19,22 @@ class LogStash::Logger < Logger
18
19
 
19
20
  # Set default loglevel to WARN unless $DEBUG is set (run with 'ruby -d')
20
21
  self.level = $DEBUG ? Logger::DEBUG: Logger::INFO
22
+ if ENV["LOGSTASH_DEBUG"]
23
+ self.level = Logger::DEBUG
24
+ end
25
+
26
+ @formatter.progname = self.progname = File.basename($0)
21
27
 
22
28
  # Conditional support for awesome_print
23
29
  if !@@have_awesome_print && @@notify_awesome_print_load_failed
24
- info [ "Failed: require 'ap' (aka awesome_print); some " \
25
- "logging features may be disabled",
26
- @@notify_awesome_print_load_failed ]
30
+ debug [ "awesome_print not found, falling back to Object#inspect." \
31
+ "If you want prettier log output, run 'gem install "\
32
+ "awesome_print'",
33
+ { :exception @@notify_awesome_print_load_failed }]
34
+
35
+ # Only show this once.
27
36
  @@notify_awesome_print_load_failed = nil
28
37
  end
29
-
30
- @formatter.progname = self.progname = File.basename($0)
31
38
  end # def initialize
32
39
 
33
40
  def level=(level)
@@ -19,6 +19,7 @@ class LogStash::Outputs::Amqp < LogStash::Outputs::Base
19
19
  end # def initialize
20
20
 
21
21
  def register
22
+ @logger.info("Registering output #{@url}")
22
23
  @amqp = AMQP.connect(:host => @url.host)
23
24
  @mq = MQ.new(@amqp)
24
25
  @target = nil
@@ -26,6 +26,7 @@ class LogStash::Outputs::Gelf < LogStash::Outputs::Base
26
26
  gelf.file = event["@source_path"]
27
27
 
28
28
  event.fields.each do |name, value|
29
+ next if value == nil or value.empty?
29
30
  gelf.add_additional name, value
30
31
  end
31
32
  gelf.add_additional "event_timestamp", event.timestamp
@@ -0,0 +1,30 @@
1
+ require "logstash/outputs/base"
2
+
3
+ class LogStash::Outputs::Internal < LogStash::Outputs::Base
4
+ def initialize(url, config={}, &block)
5
+ super
6
+ @callback = block
7
+ end
8
+
9
+ def register
10
+ @logger.info("Registering output #{@url}")
11
+ end # def register
12
+
13
+ def receive(event)
14
+ if !@callback
15
+ @logger.error("No callback for output #{@url}, cannot receive")
16
+ return
17
+ end
18
+ @callback.call(event)
19
+ end # def event
20
+
21
+ # Set the callback by passing a block of code
22
+ def callback(&block)
23
+ @callback = block
24
+ end
25
+
26
+ # Set the callback by passing a proc object
27
+ def callback=(proc_block)
28
+ @callback = proc_block
29
+ end
30
+ end # class LogStash::Outputs::Internal
@@ -0,0 +1,23 @@
1
+ require "logstash/outputs/base"
2
+
3
+ class LogStash::Outputs::Tcp < LogStash::Outputs::Base
4
+ def initialize(url, config={}, &block)
5
+ super
6
+ end
7
+
8
+ def register
9
+ # TODO(sissel): Write generic validation methods
10
+ if !@url.host or !@url.port
11
+ @logger.fatal("No host or port given in #{self.class}: #{@url}")
12
+ # TODO(sissel): Make this an actual exception class
13
+ raise "configuration error"
14
+ end
15
+
16
+ @connection = EventMachine::connect(@url.host, @url.port)
17
+ end # def register
18
+
19
+ def receive(event)
20
+ @connection.send_data(event.to_hash.to_json)
21
+ @connection.send_data("\n")
22
+ end # def receive
23
+ end # class LogStash::Outputs::Tcp
@@ -0,0 +1,12 @@
1
+
2
+
3
+ # Ruby 1.8.7 added String#start_with? - monkeypatch the
4
+ # String class if it isn't supported (<= ruby 1.8.6)
5
+ if !String.instance_methods.include?("start_with?")
6
+ class String
7
+ def start_with?(str)
8
+ return self[0 .. (str.length-1)] == str
9
+ end
10
+ end
11
+ end
12
+
data/lib/logstash/time.rb CHANGED
@@ -1,3 +1,4 @@
1
+ require "logstash/namespace"
1
2
 
2
3
  # Provide our own Time wrapper for ISO8601 support
3
4
  # Example:
@@ -6,7 +7,7 @@
6
7
  #
7
8
  # >> LogStash::Time.now.utc.to_iso8601
8
9
  # => "2010-10-17 07:25:26.788704Z"
9
- module LogStash; class Time < ::Time
10
+ class LogStash::Time < ::Time
10
11
  ISO8601 = "%Y-%m-%dT%H:%M:%S"
11
12
 
12
13
  # Return a string that is this time in ISO8601 format.
@@ -24,4 +25,4 @@ module LogStash; class Time < ::Time
24
25
  raise "Can't convert object of type #{obj.class} (#{obj}) to iso8601."
25
26
  end
26
27
  end
27
- end; end # class LogStash::Time
28
+ end # class LogStash::Time
@@ -75,7 +75,7 @@ class LogStash::Web::ElasticSearch
75
75
  end
76
76
  req.errback do
77
77
  @logger.warn(["Query failed", params, req.response])
78
- yield :failure
78
+ yield({ "error" => req.response })
79
79
  end
80
80
  end # def search
81
81
  end
@@ -0,0 +1,73 @@
1
+ <!DOCTYPE html>
2
+ <html>
3
+ <head>
4
+ <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.3/jquery.min.js"></script>
5
+ <script src="../js/jquery.tmpl.min.js"></script>
6
+ <style type="text/css">
7
+ #radiator {
8
+ margin-top: 1em;
9
+ padding-left:10px;
10
+ padding-right:10px;
11
+ font-size: 120%;
12
+ border-top: 1px solid black;
13
+ width: 100%;
14
+ }
15
+
16
+ #radiator .entry:nth-child(2n) {
17
+ background-color: #C4FFC6;
18
+ }
19
+
20
+ #radiator .timestamp {
21
+ white-space: nowrap;
22
+ text-align: left;
23
+ vertical-align: top;
24
+ font-family: monospace;
25
+ font-size: 75%;
26
+ padding-right: 1em;
27
+ width: 14em;
28
+ border-right: 1px solid grey;
29
+ }
30
+
31
+ #radiator .message {
32
+ font-family: monospace;
33
+ //text-indent: -2em;
34
+ //padding-left: 3em;
35
+ }
36
+ </style>
37
+ </head>
38
+ <body>
39
+ <h1> logstash information radiator </h1>
40
+
41
+ New log data will be streamed here in real-time as logstash
42
+ receives it.
43
+
44
+ <table id="radiator"></table>
45
+
46
+ <script id="message-template" type="text/x-jquery-tmpl"><![CDATA[
47
+ <tr>
48
+ <td class="timestamp">${received_timestamp}</td>
49
+ <td class="message">${message}</td>
50
+ </tr>
51
+ ]]></script>
52
+
53
+ <script>
54
+ $(document).ready(function() {
55
+ var ws = new WebSocket("ws://snack.home:3000");
56
+ ws.onmessage = function(event) {
57
+ eval("var data = " + event.data);
58
+ //var el = $("<div></div>")
59
+ //el.html(data.message)
60
+ var el = $("#message-template").tmpl(data);
61
+ el.addClass("message")
62
+ .css("display", "none")
63
+ .appendTo($("#radiator"))
64
+ .fadeIn()
65
+ .delay(10000)
66
+ .fadeOut(2000, function() {
67
+ $(this).remove();
68
+ });
69
+ }
70
+ });
71
+ </script>
72
+ </body>
73
+ </html>
@@ -49,11 +49,21 @@ class LogStash::Web::Server < Sinatra::Base
49
49
  count = params["count"] = (params["count"] or 50).to_i
50
50
  offset = params["offset"] = (params["offset"] or 0).to_i
51
51
  elasticsearch.search(params) do |@results|
52
+ #p instance_variables
53
+ if @results.include?("error")
54
+ body haml :"search/error", :layout => !request.xhr?
55
+ next
56
+ end
57
+
52
58
  @hits = (@results["hits"]["hits"] rescue [])
53
59
  @total = (@results["hits"]["total"] rescue 0)
54
60
  @graphpoints = []
55
- @results["facets"]["by_hour"]["entries"].each do |entry|
56
- @graphpoints << [entry["key"], entry["count"]]
61
+ begin
62
+ @results["facets"]["by_hour"]["entries"].each do |entry|
63
+ @graphpoints << [entry["key"], entry["count"]]
64
+ end
65
+ rescue => e
66
+ puts e
57
67
  end
58
68
 
59
69
  if count and offset
@@ -69,12 +79,21 @@ class LogStash::Web::Server < Sinatra::Base
69
79
  next_params = params.clone
70
80
  next_params["offset"] = [offset + count, @total - count].min
71
81
  @next_href = "?" + next_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
82
+ last_params = next_params.clone
83
+ last_params["offset"] = @total - offset
84
+ @last_href = "?" + last_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
72
85
  end
73
86
 
74
87
  if offset > 0
75
88
  prev_params = params.clone
76
89
  prev_params["offset"] = [offset - count, 0].max
77
90
  @prev_href = "?" + prev_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
91
+
92
+ if prev_params["offset"] > 0
93
+ first_params = prev_params.clone
94
+ first_params["offset"] = 0
95
+ @first_href = "?" + first_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
96
+ end
78
97
  end
79
98
 
80
99
  body haml :"search/ajax", :layout => !request.xhr?
@@ -14,6 +14,9 @@
14
14
  %strong
15
15
  Results #{@result_start} - #{@result_end} of #{@total}
16
16
  |
17
+ - if @first_href
18
+ %a.pager{ :href => @first_href } first
19
+ |
17
20
  - if @prev_href
18
21
  %a.pager{ :href => @prev_href }
19
22
  prev
@@ -22,6 +25,10 @@
22
25
  - if @next_href
23
26
  %a.pager{ :href => @next_href }
24
27
  next
28
+ - if @last_href
29
+ |
30
+ %a.pager{ :href => @last_href }
31
+ last
25
32
  - if @hits.length == 0
26
33
  - if !params[:q]
27
34
  No query given. How about <a href="?q=*" class="querychanger">this?</a>
@@ -0,0 +1,3 @@
1
+ #error
2
+ %h4 The query '#{params["q"]}' resulted the following error:
3
+ %pre&= @results["error"]
@@ -46,6 +46,8 @@ body
46
46
  background-color: pink
47
47
  border: 1px solid red
48
48
  padding: 3px
49
+ pre
50
+ white-space: pre-wrap
49
51
  #error h1
50
52
  font-size: 130%
51
53
  padding: 0
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-lite
3
3
  version: !ruby/object:Gem::Version
4
- hash: 40202240049533
4
+ hash: 40202246267461
5
5
  prerelease: false
6
6
  segments:
7
7
  - 0
8
8
  - 2
9
- - 20101120024757
10
- version: 0.2.20101120024757
9
+ - 20101123133737
10
+ version: 0.2.20101123133737
11
11
  platform: ruby
12
12
  authors:
13
13
  - Jordan Sissel
@@ -15,7 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2010-11-20 00:00:00 -08:00
18
+ date: 2010-11-23 00:00:00 -08:00
19
19
  default_executable:
20
20
  dependencies:
21
21
  - !ruby/object:Gem::Dependency
@@ -59,6 +59,8 @@ files:
59
59
  - lib/logstash.rb
60
60
  - lib/logstash/agent.rb
61
61
  - lib/logstash/inputs.rb
62
+ - lib/logstash/ruby_fixes.rb
63
+ - lib/logstash/inputs/internal.rb
62
64
  - lib/logstash/inputs/syslog.rb
63
65
  - lib/logstash/inputs/file.rb
64
66
  - lib/logstash/inputs/base.rb
@@ -66,11 +68,13 @@ files:
66
68
  - lib/logstash/inputs/tcp.rb
67
69
  - lib/logstash/outputs/gelf.rb
68
70
  - lib/logstash/outputs/elasticsearch.rb
71
+ - lib/logstash/outputs/internal.rb
69
72
  - lib/logstash/outputs/mongodb.rb
70
73
  - lib/logstash/outputs/stdout.rb
71
74
  - lib/logstash/outputs/websocket.rb
72
75
  - lib/logstash/outputs/base.rb
73
76
  - lib/logstash/outputs/amqp.rb
77
+ - lib/logstash/outputs/tcp.rb
74
78
  - lib/logstash/namespace.rb
75
79
  - lib/logstash/time.rb
76
80
  - lib/logstash/filters.rb
@@ -98,6 +102,7 @@ files:
98
102
  - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_55_fbf9ee_1x400.png
99
103
  - lib/logstash/web/public/css/smoothness/images/ui-bg_flat_75_ffffff_40x100.png
100
104
  - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_95_fef1ec_1x400.png
105
+ - lib/logstash/web/public/ws/index.html
101
106
  - lib/logstash/web/public/js/jquery.livequery.js
102
107
  - lib/logstash/web/public/js/logstash.js
103
108
  - lib/logstash/web/public/js/jquery-hashchange-1.0.0.js
@@ -167,8 +172,10 @@ files:
167
172
  - lib/logstash/web/views/layout.haml
168
173
  - lib/logstash/web/views/search/ajax.haml
169
174
  - lib/logstash/web/views/search/results.haml
175
+ - lib/logstash/web/views/search/error.haml
170
176
  - lib/logstash/web/views/header.haml
171
177
  - examples/test.rb
178
+ - examples/sample-agent-in-ruby.rb
172
179
  - etc/tograylog.yaml
173
180
  - etc/logstash-elasticsearch-rabbitmq-river.yaml
174
181
  - etc/logstash-reader.yaml