logstash-lite 0.2.20101118134500

Sign up to get free protection for your applications and to get access to all the features.
Files changed (134) hide show
  1. data/bin/logstash +56 -0
  2. data/bin/logstash-web +6 -0
  3. data/etc/logstash-elasticsearch-rabbitmq-river.yaml +41 -0
  4. data/etc/logstash-mongodb-storage.yaml +5 -0
  5. data/etc/logstash-parser.yaml +20 -0
  6. data/etc/logstash-reader.yaml +8 -0
  7. data/etc/logstash-shipper.yaml +18 -0
  8. data/etc/logstash-standalone.yaml +47 -0
  9. data/etc/prod.yaml +38 -0
  10. data/etc/redhat/logstash +92 -0
  11. data/etc/redhat/logstash-agent +83 -0
  12. data/etc/redhat/logstash-agent.sysconfig +7 -0
  13. data/etc/redhat/logstash.spec +171 -0
  14. data/etc/redhat/logstash.sysconfig +18 -0
  15. data/etc/tograylog.yaml +37 -0
  16. data/examples/test.rb +38 -0
  17. data/lib/logstash.rb +3 -0
  18. data/lib/logstash/agent.rb +116 -0
  19. data/lib/logstash/event.rb +70 -0
  20. data/lib/logstash/filters.rb +17 -0
  21. data/lib/logstash/filters/base.rb +17 -0
  22. data/lib/logstash/filters/date.rb +59 -0
  23. data/lib/logstash/filters/field.rb +29 -0
  24. data/lib/logstash/filters/grok.rb +74 -0
  25. data/lib/logstash/filters/grokdiscovery.rb +60 -0
  26. data/lib/logstash/inputs.rb +18 -0
  27. data/lib/logstash/inputs/amqp.rb +48 -0
  28. data/lib/logstash/inputs/base.rb +32 -0
  29. data/lib/logstash/inputs/file.rb +47 -0
  30. data/lib/logstash/inputs/syslog.rb +123 -0
  31. data/lib/logstash/inputs/tcp.rb +51 -0
  32. data/lib/logstash/logging.rb +82 -0
  33. data/lib/logstash/namespace.rb +6 -0
  34. data/lib/logstash/outputs.rb +15 -0
  35. data/lib/logstash/outputs/amqp.rb +48 -0
  36. data/lib/logstash/outputs/base.rb +29 -0
  37. data/lib/logstash/outputs/elasticsearch.rb +71 -0
  38. data/lib/logstash/outputs/gelf.rb +35 -0
  39. data/lib/logstash/outputs/mongodb.rb +19 -0
  40. data/lib/logstash/outputs/stdout.rb +15 -0
  41. data/lib/logstash/outputs/websocket.rb +35 -0
  42. data/lib/logstash/time.rb +27 -0
  43. data/lib/logstash/web/lib/elasticsearch.rb +79 -0
  44. data/lib/logstash/web/public/css/smoothness/images/ui-bg_flat_0_aaaaaa_40x100.png +0 -0
  45. data/lib/logstash/web/public/css/smoothness/images/ui-bg_flat_75_ffffff_40x100.png +0 -0
  46. data/lib/logstash/web/public/css/smoothness/images/ui-bg_glass_55_fbf9ee_1x400.png +0 -0
  47. data/lib/logstash/web/public/css/smoothness/images/ui-bg_glass_65_ffffff_1x400.png +0 -0
  48. data/lib/logstash/web/public/css/smoothness/images/ui-bg_glass_75_dadada_1x400.png +0 -0
  49. data/lib/logstash/web/public/css/smoothness/images/ui-bg_glass_75_e6e6e6_1x400.png +0 -0
  50. data/lib/logstash/web/public/css/smoothness/images/ui-bg_glass_95_fef1ec_1x400.png +0 -0
  51. data/lib/logstash/web/public/css/smoothness/images/ui-bg_highlight-soft_75_cccccc_1x100.png +0 -0
  52. data/lib/logstash/web/public/css/smoothness/images/ui-icons_222222_256x240.png +0 -0
  53. data/lib/logstash/web/public/css/smoothness/images/ui-icons_2e83ff_256x240.png +0 -0
  54. data/lib/logstash/web/public/css/smoothness/images/ui-icons_454545_256x240.png +0 -0
  55. data/lib/logstash/web/public/css/smoothness/images/ui-icons_888888_256x240.png +0 -0
  56. data/lib/logstash/web/public/css/smoothness/images/ui-icons_cd0a0a_256x240.png +0 -0
  57. data/lib/logstash/web/public/css/smoothness/jquery-ui-1.8.5.custom.css +572 -0
  58. data/lib/logstash/web/public/js/flot/API.txt +1024 -0
  59. data/lib/logstash/web/public/js/flot/FAQ.txt +71 -0
  60. data/lib/logstash/web/public/js/flot/LICENSE.txt +22 -0
  61. data/lib/logstash/web/public/js/flot/Makefile +15 -0
  62. data/lib/logstash/web/public/js/flot/NEWS.txt +340 -0
  63. data/lib/logstash/web/public/js/flot/PLUGINS.txt +105 -0
  64. data/lib/logstash/web/public/js/flot/README.txt +81 -0
  65. data/lib/logstash/web/public/js/flot/examples/ajax.html +143 -0
  66. data/lib/logstash/web/public/js/flot/examples/annotating.html +75 -0
  67. data/lib/logstash/web/public/js/flot/examples/arrow-down.gif +0 -0
  68. data/lib/logstash/web/public/js/flot/examples/arrow-left.gif +0 -0
  69. data/lib/logstash/web/public/js/flot/examples/arrow-right.gif +0 -0
  70. data/lib/logstash/web/public/js/flot/examples/arrow-up.gif +0 -0
  71. data/lib/logstash/web/public/js/flot/examples/basic.html +38 -0
  72. data/lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-1.json +4 -0
  73. data/lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-2.json +4 -0
  74. data/lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-3.json +4 -0
  75. data/lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-4.json +4 -0
  76. data/lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-5.json +4 -0
  77. data/lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth.json +4 -0
  78. data/lib/logstash/web/public/js/flot/examples/data-japan-gdp-growth.json +4 -0
  79. data/lib/logstash/web/public/js/flot/examples/data-usa-gdp-growth.json +4 -0
  80. data/lib/logstash/web/public/js/flot/examples/dual-axis.html +39 -0
  81. data/lib/logstash/web/public/js/flot/examples/graph-types.html +75 -0
  82. data/lib/logstash/web/public/js/flot/examples/hs-2004-27-a-large_web.jpg +0 -0
  83. data/lib/logstash/web/public/js/flot/examples/image.html +45 -0
  84. data/lib/logstash/web/public/js/flot/examples/index.html +43 -0
  85. data/lib/logstash/web/public/js/flot/examples/interacting.html +93 -0
  86. data/lib/logstash/web/public/js/flot/examples/layout.css +6 -0
  87. data/lib/logstash/web/public/js/flot/examples/navigate.html +118 -0
  88. data/lib/logstash/web/public/js/flot/examples/selection.html +114 -0
  89. data/lib/logstash/web/public/js/flot/examples/setting-options.html +65 -0
  90. data/lib/logstash/web/public/js/flot/examples/stacking.html +77 -0
  91. data/lib/logstash/web/public/js/flot/examples/thresholding.html +54 -0
  92. data/lib/logstash/web/public/js/flot/examples/time.html +71 -0
  93. data/lib/logstash/web/public/js/flot/examples/tracking.html +95 -0
  94. data/lib/logstash/web/public/js/flot/examples/turning-series.html +98 -0
  95. data/lib/logstash/web/public/js/flot/examples/visitors.html +90 -0
  96. data/lib/logstash/web/public/js/flot/examples/zooming.html +98 -0
  97. data/lib/logstash/web/public/js/flot/excanvas.js +1427 -0
  98. data/lib/logstash/web/public/js/flot/excanvas.min.js +1 -0
  99. data/lib/logstash/web/public/js/flot/jquery.colorhelpers.js +174 -0
  100. data/lib/logstash/web/public/js/flot/jquery.colorhelpers.min.js +1 -0
  101. data/lib/logstash/web/public/js/flot/jquery.flot.crosshair.js +156 -0
  102. data/lib/logstash/web/public/js/flot/jquery.flot.crosshair.min.js +1 -0
  103. data/lib/logstash/web/public/js/flot/jquery.flot.image.js +237 -0
  104. data/lib/logstash/web/public/js/flot/jquery.flot.image.min.js +1 -0
  105. data/lib/logstash/web/public/js/flot/jquery.flot.js +2119 -0
  106. data/lib/logstash/web/public/js/flot/jquery.flot.min.js +1 -0
  107. data/lib/logstash/web/public/js/flot/jquery.flot.navigate.js +272 -0
  108. data/lib/logstash/web/public/js/flot/jquery.flot.navigate.min.js +1 -0
  109. data/lib/logstash/web/public/js/flot/jquery.flot.selection.js +299 -0
  110. data/lib/logstash/web/public/js/flot/jquery.flot.selection.min.js +1 -0
  111. data/lib/logstash/web/public/js/flot/jquery.flot.stack.js +152 -0
  112. data/lib/logstash/web/public/js/flot/jquery.flot.stack.min.js +1 -0
  113. data/lib/logstash/web/public/js/flot/jquery.flot.threshold.js +103 -0
  114. data/lib/logstash/web/public/js/flot/jquery.flot.threshold.min.js +1 -0
  115. data/lib/logstash/web/public/js/flot/jquery.js +4376 -0
  116. data/lib/logstash/web/public/js/flot/jquery.min.js +19 -0
  117. data/lib/logstash/web/public/js/jquery-hashchange-1.0.0.js +121 -0
  118. data/lib/logstash/web/public/js/jquery.livequery.js +250 -0
  119. data/lib/logstash/web/public/js/jquery.tmpl.min.js +1 -0
  120. data/lib/logstash/web/public/js/logstash.js +202 -0
  121. data/lib/logstash/web/server.rb +90 -0
  122. data/lib/logstash/web/views/header.haml +8 -0
  123. data/lib/logstash/web/views/layout.haml +21 -0
  124. data/lib/logstash/web/views/main/index.haml +5 -0
  125. data/lib/logstash/web/views/search/ajax.haml +32 -0
  126. data/lib/logstash/web/views/search/results.haml +17 -0
  127. data/lib/logstash/web/views/style.sass +50 -0
  128. data/patterns/firewalls +2 -0
  129. data/patterns/grok-patterns +90 -0
  130. data/patterns/haproxy +5 -0
  131. data/patterns/linux-syslog +7 -0
  132. data/patterns/nagios +7 -0
  133. data/patterns/ruby +2 -0
  134. metadata +228 -0
@@ -0,0 +1,90 @@
1
+ #!/usr/bin/env ruby
2
+ ##rackup -Ilib:../lib -s thin
3
+
4
+ $:.unshift("%s/../lib" % File.dirname(__FILE__))
5
+ $:.unshift(File.dirname(__FILE__))
6
+
7
+ require "rubygems"
8
+ require "json"
9
+ require "eventmachine"
10
+ require "rack"
11
+ require "sinatra/async"
12
+ require "lib/elasticsearch"
13
+ require "logstash/namespace"
14
+
15
+ class EventMachine::ConnectionError < RuntimeError; end
16
+
17
+ class LogStash::Web::Server < Sinatra::Base
18
+ register Sinatra::Async
19
+ set :haml, :format => :html5
20
+ set :logging, true
21
+ set :public, "#{File.dirname(__FILE__)}/public"
22
+ set :views, "#{File.dirname(__FILE__)}/views"
23
+ elasticsearch = LogStash::Web::ElasticSearch.new
24
+
25
+ aget '/style.css' do
26
+ headers "Content-Type" => "text/css; charset=utf8"
27
+ body sass :style
28
+ end
29
+
30
+ aget '/' do
31
+ redirect "/search"
32
+ end # '/'
33
+
34
+ aget '/search' do
35
+ headers({"Content-Type" => "text/html" })
36
+ if params[:q] and params[:q] != ""
37
+ elasticsearch.search(params) do |@results|
38
+ @hits = (@results["hits"]["hits"] rescue [])
39
+ body haml :"search/results", :layout => !request.xhr?
40
+ end
41
+ else
42
+ @hits = []
43
+ body haml :"search/results", :layout => !request.xhr?
44
+ end
45
+ end
46
+
47
+ apost '/search/ajax' do
48
+ headers({"Content-Type" => "text/html" })
49
+ count = params["count"] = (params["count"] or 50).to_i
50
+ offset = params["offset"] = (params["offset"] or 0).to_i
51
+ elasticsearch.search(params) do |@results|
52
+ @hits = (@results["hits"]["hits"] rescue [])
53
+ @total = (@results["hits"]["total"] rescue 0)
54
+ @graphpoints = []
55
+ @results["facets"]["by_hour"]["entries"].each do |entry|
56
+ @graphpoints << [entry["key"], entry["count"]]
57
+ end
58
+
59
+ if count and offset
60
+ if @total > (count + offset)
61
+ @result_end = (count + offset)
62
+ else
63
+ @result_end = @total
64
+ end
65
+ @result_start = offset
66
+ end
67
+
68
+ if count + offset < @total
69
+ next_params = params.clone
70
+ next_params["offset"] = [offset + count, @total - count].min
71
+ @next_href = "?" + next_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
72
+ end
73
+
74
+ if offset > 0
75
+ prev_params = params.clone
76
+ prev_params["offset"] = [offset - count, 0].max
77
+ @prev_href = "?" + prev_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
78
+ end
79
+
80
+ body haml :"search/ajax", :layout => !request.xhr?
81
+ end # elasticsearch.search
82
+ end # apost '/search/ajax'
83
+
84
+ end # class LogStashWeb
85
+
86
+ Rack::Handler::Thin.run(
87
+ Rack::CommonLogger.new( \
88
+ Rack::ShowExceptions.new( \
89
+ LogStash::Web::Server.new)),
90
+ :Port => 9292)
@@ -0,0 +1,8 @@
1
+ .logo
2
+ logstash.
3
+ .search
4
+ %form.search{ :action => "/search" }
5
+ %label{ :for => "q" } Query:
6
+ %input.query{ :id => "query", :type => "text", :name => "q", :value => params[:q],
7
+ :size => 60 }
8
+ %input{ :id => "searchbutton", :type => "submit" }
@@ -0,0 +1,21 @@
1
+ !!! 5
2
+ %html
3
+ %head
4
+ %title= @title || "logstash"
5
+ %link{ :rel => "stylesheet", :href => "/style.css", :type => "text/css" }
6
+ %link{ :rel => "stylesheet", :href => "/css/smoothness/jquery-ui-1.8.5.custom.css", :type => "text/css" }
7
+ %script{ :src => "https://ajax.googleapis.com/ajax/libs/jquery/1.4.3/jquery.min.js",
8
+ :type => "text/javascript" }
9
+ %body
10
+ #header
11
+ =haml :header, :layout => false
12
+ #content
13
+ =yield
14
+ #footer
15
+
16
+ %script{ :src => "https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.5/jquery-ui.min.js",
17
+ :type => "text/javascript" }
18
+ %script{ :src => "js/jquery.tmpl.min.js", :type => "text/javascript" }
19
+ %script{ :src => "js/jquery-hashchange-1.0.0.js", :type => "text/javascript" }
20
+ %script{ :src => "js/flot/jquery.flot.js", :type => "text/javascript" }
21
+ %script{ :src => "js/logstash.js", :type => "text/javascript" }
@@ -0,0 +1,5 @@
1
+ %form.search{ :action => "/search" }
2
+ %label{ :for => "q" }
3
+ %input.query{ :type => "text", :name => "q" }
4
+ %input{ :type => "submit" }
5
+
@@ -0,0 +1,32 @@
1
+ #results
2
+ - if (params[:q].strip.length > 0 rescue false)
3
+ %h1
4
+ Search results for '#{params[:q]}'
5
+ - if @graphpoints
6
+ #visual
7
+ :javascript
8
+ $(function() {
9
+ var graphdata = #{@graphpoints.to_json};
10
+ window.logstash.plot(graphdata);
11
+ });
12
+ - if @total and @result_start and @result_end
13
+ %small
14
+ %strong
15
+ Results #{@result_start} - #{@result_end} of #{@total}
16
+ |
17
+ - if @prev_href
18
+ %a.pager{ :href => @prev_href }
19
+ prev
20
+ - if @next_href
21
+ |
22
+ - if @next_href
23
+ %a.pager{ :href => @next_href }
24
+ next
25
+ %ul.results
26
+ - @hits.reverse.each do |hit|
27
+ %li.event{ :"data-full" => hit.to_json }&= hit["_source"]["@message"]
28
+ - if @hits.length == 0
29
+ - if !params[:q]
30
+ No query given. How about <a href="?q=*" class="querychanger">this?</a>
31
+ - else
32
+ No results for query '#{params[:q]}'
@@ -0,0 +1,17 @@
1
+ - if @error
2
+ #error
3
+ %strong A search error occurred:
4
+ =@error
5
+ #ssquery{ :style => "display: none;", :"data-query" => params[:q] }
6
+ #inspector{ :style => "display: none;" }
7
+ The following fields are known for the log you selected. Click on any link to
8
+ append it to your search. If you shift+click, the field will be added to the
9
+ search as an exclude rather than include.
10
+ %ul
11
+
12
+ %i
13
+ You can click on any search result to see what kind of fields we know about
14
+ for that event. You can also click on the graph to zoom to that time period.
15
+ The query language is that of Lucene's string query (<a href="http://lucene.apache.org/java/2_4_0/queryparsersyntax.html">docs</a>).
16
+
17
+ =haml :"search/ajax", :layout => false
@@ -0,0 +1,50 @@
1
+ $lightgrey: #d8d8d8
2
+ $darkgrey: #adadad
3
+ body
4
+ margin: 0
5
+ padding: 0
6
+ #header
7
+ border-top: 4px solid black
8
+ border-bottom: 1px solid black
9
+ background-color: lightgreen
10
+ padding-left: 1em
11
+
12
+ .search
13
+ display: inline
14
+ .logo
15
+ font-size: 130%
16
+ font-weight: bold
17
+ float: right
18
+ padding-right: 20px
19
+ #content
20
+ margin-left: 2em
21
+ margin-right: 2em
22
+ margin-top: 1em
23
+ #content ul.results
24
+ font-family: monospace
25
+ #content li.event
26
+ padding-bottom: 3px
27
+ white-space: pre-wrap
28
+ #content li.selected
29
+ background-color: #FCE69D !important
30
+ #content li.event:nth-child(2n)
31
+ background-color: #E3F6CE
32
+ #content li.event:nth-child(2n+1)
33
+ background-color: #F5FBEF
34
+ #content li.event:hover
35
+ background-color: lightgreen
36
+ #error
37
+ background-color: pink
38
+ border: 1px solid red
39
+ padding: 3px
40
+ #error h1
41
+ font-size: 130%
42
+ padding: 0
43
+ margin: 0
44
+ #inspector
45
+ font-size: 70%
46
+ #results #visual
47
+ width: 850px
48
+ height: 200px
49
+ #results h1
50
+ font-size: 100%
@@ -0,0 +1,2 @@
1
+ # NetScreen firewall logs
2
+ NETSCREENSESSIONLOG %{SYSLOGDATE:date} %{IPORHOST:device} %{IPORHOST}: NetScreen device_id=%{WORD:device_id}%{DATA}: start_time=%{QUOTEDSTRING:start_time} duration=%{INT:duration} policy_id=%{INT:policy_id} service=%{DATA:service} proto=%{INT:proto} src zone=%{WORD:src_zone} dst zone=%{WORD:dst_zone} action=%{WORD:action} sent=%{INT:sent} rcvd=%{INT:rcvd} src=%{IPORHOST:src_ip} dst=%{IPORHOST:dst_ip} src_port=%{INT:src_port} dst_port=%{INT:dst_port} src-xlated ip=%{IPORHOST:src_xlated_ip} port=%{INT:src_xlated_port} dst-xlated ip=%{IPORHOST:dst_xlated_ip} port=%{INT:dst_xlated_port} session_id=%{INT:session_id} reason=%{GREEDYDATA:reason}
@@ -0,0 +1,90 @@
1
+ USERNAME [a-zA-Z0-9_-]+
2
+ USER %{USERNAME}
3
+ INT (?:[+-]?(?:[0-9]+))
4
+ BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
5
+ NUMBER (?:%{BASE10NUM})
6
+ BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
7
+ BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
8
+
9
+ POSINT \b(?:[0-9]+)\b
10
+ WORD \b\w+\b
11
+ NOTSPACE \S+
12
+ DATA .*?
13
+ GREEDYDATA .*
14
+ QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
15
+
16
+ # Networking
17
+ MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
18
+ CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
19
+ WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
20
+ COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
21
+ IP (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
22
+ HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
23
+ HOST %{HOSTNAME}
24
+ IPORHOST (?:%{HOSTNAME}|%{IP})
25
+ HOSTPORT (?:%{IPORHOST=~/\./}:%{POSINT})
26
+
27
+ # paths
28
+ PATH (?:%{UNIXPATH}|%{WINPATH})
29
+ UNIXPATH (?<![\w\\/])(?:/(?:[\w_%!$@:.,-]+|\\.)*)+
30
+ #UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+
31
+ LINUXTTY (?:/dev/pts/%{POSINT})
32
+ BSDTTY (?:/dev/tty[pq][a-z0-9])
33
+ TTY (?:%{BSDTTY}|%{LINUXTTY})
34
+ WINPATH (?:[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
35
+ URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
36
+ URIHOST %{IPORHOST}(?::%{POSINT})?
37
+ # uripath comes loosely from RFC1738, but mostly from what Firefox
38
+ # doesn't turn into %XX
39
+ URIPATH (?:/[A-Za-z0-9$.+!*'(),~:#%_-]*)+
40
+ #URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
41
+ URIPARAM \?[A-Za-z0-9$.+!*'(),~#%&/=:;_-]*
42
+ URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
43
+ URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:$|%{URIPATHPARAM})
44
+
45
+ # Months: January, Feb, 3, 03, 12, December
46
+ MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
47
+ MONTHNUM (?:0?[1-9]|1[0-2])
48
+ MONTHDAY (?:3[01]|[1-2]?[0-9]|0?[1-9])
49
+
50
+ # Days: Monday, Tue, Thu, etc...
51
+ DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
52
+
53
+ # Years?
54
+ YEAR [0-9]+
55
+ # Time: HH:MM:SS
56
+ #TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
57
+ # I'm still on the fence about using grok to perform the time match,
58
+ # since it's probably slower.
59
+ # TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)?
60
+ HOUR (?:2[0123]|[01][0-9])
61
+ MINUTE (?:[0-5][0-9])
62
+ # '60' is a leap second in most time standards and thus is valid.
63
+ SECOND (?:(?:[0-5][0-9]|60)(?:[.,][0-9]+)?)
64
+ TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
65
+ # datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
66
+ DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
67
+ DATE_EU %{YEAR}[/-]%{MONTHNUM}[/-]%{MONTHDAY}
68
+ ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
69
+ ISO8601_SECOND (?:%{SECOND}|60)
70
+ TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
71
+ DATE %{DATE_US}|%{DATE_EU}
72
+ DATESTAMP %{DATE}[- ]%{TIME}
73
+ TZ (?:[PMCE][SD]T)
74
+ DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
75
+ DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} (?:%{TZ} )?%{YEAR}
76
+
77
+ # Syslog Dates: Month Day HH:MM:SS
78
+ SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
79
+ PROG (?:[\w._/-]+)
80
+ SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
81
+ SYSLOGHOST %{IPORHOST}
82
+ SYSLOGFACILITY <%{POSINT:facility}.%{POSINT:priority}>
83
+ HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT:ZONE}
84
+
85
+ # Shortcuts
86
+ QS %{QUOTEDSTRING}
87
+
88
+ # Log formats
89
+ SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
90
+ COMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) "(?:%{URI:referrer}|-)" %{QS:agent}
data/patterns/haproxy ADDED
@@ -0,0 +1,5 @@
1
+ HAPROXYDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME}.%{INT:milliseconds}
2
+ HAPROXYTERMINATIONSTATE [CAPRIcs-][RQCHDLT-][NIDV-][NIPRD-]
3
+
4
+ # parse an haproxy 'httplog' line
5
+ HAPROXYHTTP %{SYSLOGDATE:date} %{IPORHOST:server} %{SYSLOGPROG}: %{IP:clientip}:%{INT:clientport} \[%{HAPROXYDATE:haproxydate}\] %{NOTSPACE:proxyname} %{NOTSPACE}/%{IPORHOST:backend} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{INT:time_duration} %{INT:response} %{INT:bytes} - - %{HAPROXYTERMINATIONSTATE:terminationstate} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn} %{INT:srv_queue}/%{INT:backend_queue} "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:version}"
@@ -0,0 +1,7 @@
1
+ SYSLOGBASE2 (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
2
+ SYSLOGPAMSESSION %{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:pam_module}\(%{DATA:pam_caller}\): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})?
3
+
4
+ CRON_ACTION [A-Z ]+
5
+ CRONLOG %{SYSLOGBASE} \(%{USER:user}\) %{CRON_ACTION:action} \(%{DATA:message}\)
6
+
7
+ SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message}
data/patterns/nagios ADDED
@@ -0,0 +1,7 @@
1
+ NAGIOSTIME \[%{NUMBER:epochtime}\]
2
+
3
+ NAGIOS_SERVICE_ALERT SERVICE ALERT: %{IPORHOST:hostname};%{DATA:checkname};%{DATA:state};%{DATA:statelevel};%{NUMBER:attempt};%{GREEDYDATA:message}
4
+ NAGIOS_SERVICE_FLAPPING_ALERT SERVICE FLAPPING ALERT: %{IPORHOST:hostname};%{DATA:checkname};%{DATA:state};%{GREEDYDATA:message}
5
+ NAGIOS_SERVICE_NOTIFICATION SERVICE NOTIFICATION: %{DATA:notifyname};%{IPORHOST:hostname};%{DATA:checkname};%{DATA:state};%{DATA:contact};%{GREEDYDATA:message}
6
+
7
+ NAGIOSLOGLINE %{NAGIOSTIME} (?:%{NAGIOS_SERVICE_ALERT}|%{NAGIOS_SERVICE_FLAPPING_ALERT}|%{NAGIOS_SERVICE_NOTIFICATION})
data/patterns/ruby ADDED
@@ -0,0 +1,2 @@
1
+ RUBY_LOGLEVEL (?:DEBUG|FATAL|ERROR|WARN|INFO)
2
+ RUBY_LOGGER [DFEWI], \[%{TIMESTAMP_ISO8601} #{POSINT:pid}\] *%{RUBY_LOGLEVEL} -- %{DATA:progname}: %{DATA:message}
metadata ADDED
@@ -0,0 +1,228 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-lite
3
+ version: !ruby/object:Gem::Version
4
+ hash: 40202236269023
5
+ prerelease: false
6
+ segments:
7
+ - 0
8
+ - 2
9
+ - 20101118134500
10
+ version: 0.2.20101118134500
11
+ platform: ruby
12
+ authors:
13
+ - Jordan Sissel
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2010-11-18 00:00:00 -08:00
19
+ default_executable:
20
+ dependencies:
21
+ - !ruby/object:Gem::Dependency
22
+ name: eventmachine-tail
23
+ prerelease: false
24
+ requirement: &id001 !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ hash: 3
30
+ segments:
31
+ - 0
32
+ version: "0"
33
+ type: :runtime
34
+ version_requirements: *id001
35
+ - !ruby/object:Gem::Dependency
36
+ name: json
37
+ prerelease: false
38
+ requirement: &id002 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ hash: 3
44
+ segments:
45
+ - 0
46
+ version: "0"
47
+ type: :runtime
48
+ version_requirements: *id002
49
+ description: scalable log and event management (search, archive, pipeline)
50
+ email: jls@semicomplete.com
51
+ executables:
52
+ - logstash
53
+ - logstash-web
54
+ extensions: []
55
+
56
+ extra_rdoc_files: []
57
+
58
+ files:
59
+ - lib/logstash.rb
60
+ - lib/logstash/agent.rb
61
+ - lib/logstash/inputs.rb
62
+ - lib/logstash/inputs/syslog.rb
63
+ - lib/logstash/inputs/file.rb
64
+ - lib/logstash/inputs/base.rb
65
+ - lib/logstash/inputs/amqp.rb
66
+ - lib/logstash/inputs/tcp.rb
67
+ - lib/logstash/outputs/gelf.rb
68
+ - lib/logstash/outputs/elasticsearch.rb
69
+ - lib/logstash/outputs/mongodb.rb
70
+ - lib/logstash/outputs/stdout.rb
71
+ - lib/logstash/outputs/websocket.rb
72
+ - lib/logstash/outputs/base.rb
73
+ - lib/logstash/outputs/amqp.rb
74
+ - lib/logstash/namespace.rb
75
+ - lib/logstash/time.rb
76
+ - lib/logstash/filters.rb
77
+ - lib/logstash/outputs.rb
78
+ - lib/logstash/filters/grokdiscovery.rb
79
+ - lib/logstash/filters/grok.rb
80
+ - lib/logstash/filters/base.rb
81
+ - lib/logstash/filters/field.rb
82
+ - lib/logstash/filters/date.rb
83
+ - lib/logstash/logging.rb
84
+ - lib/logstash/event.rb
85
+ - lib/logstash/web/lib/elasticsearch.rb
86
+ - lib/logstash/web/public/css/smoothness/jquery-ui-1.8.5.custom.css
87
+ - lib/logstash/web/public/css/smoothness/images/ui-icons_2e83ff_256x240.png
88
+ - lib/logstash/web/public/css/smoothness/images/ui-icons_cd0a0a_256x240.png
89
+ - lib/logstash/web/public/css/smoothness/images/ui-icons_222222_256x240.png
90
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_highlight-soft_75_cccccc_1x100.png
91
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_75_e6e6e6_1x400.png
92
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_65_ffffff_1x400.png
93
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_flat_0_aaaaaa_40x100.png
94
+ - lib/logstash/web/public/css/smoothness/images/ui-icons_454545_256x240.png
95
+ - lib/logstash/web/public/css/smoothness/images/ui-icons_888888_256x240.png
96
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_75_dadada_1x400.png
97
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_55_fbf9ee_1x400.png
98
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_flat_75_ffffff_40x100.png
99
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_95_fef1ec_1x400.png
100
+ - lib/logstash/web/public/js/jquery.livequery.js
101
+ - lib/logstash/web/public/js/logstash.js
102
+ - lib/logstash/web/public/js/jquery-hashchange-1.0.0.js
103
+ - lib/logstash/web/public/js/jquery.tmpl.min.js
104
+ - lib/logstash/web/public/js/flot/README.txt
105
+ - lib/logstash/web/public/js/flot/examples/tracking.html
106
+ - lib/logstash/web/public/js/flot/examples/layout.css
107
+ - lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-1.json
108
+ - lib/logstash/web/public/js/flot/examples/data-japan-gdp-growth.json
109
+ - lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-2.json
110
+ - lib/logstash/web/public/js/flot/examples/setting-options.html
111
+ - lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth.json
112
+ - lib/logstash/web/public/js/flot/examples/data-usa-gdp-growth.json
113
+ - lib/logstash/web/public/js/flot/examples/stacking.html
114
+ - lib/logstash/web/public/js/flot/examples/navigate.html
115
+ - lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-4.json
116
+ - lib/logstash/web/public/js/flot/examples/graph-types.html
117
+ - lib/logstash/web/public/js/flot/examples/dual-axis.html
118
+ - lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-5.json
119
+ - lib/logstash/web/public/js/flot/examples/arrow-down.gif
120
+ - lib/logstash/web/public/js/flot/examples/annotating.html
121
+ - lib/logstash/web/public/js/flot/examples/zooming.html
122
+ - lib/logstash/web/public/js/flot/examples/selection.html
123
+ - lib/logstash/web/public/js/flot/examples/basic.html
124
+ - lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-3.json
125
+ - lib/logstash/web/public/js/flot/examples/arrow-right.gif
126
+ - lib/logstash/web/public/js/flot/examples/turning-series.html
127
+ - lib/logstash/web/public/js/flot/examples/visitors.html
128
+ - lib/logstash/web/public/js/flot/examples/image.html
129
+ - lib/logstash/web/public/js/flot/examples/arrow-up.gif
130
+ - lib/logstash/web/public/js/flot/examples/hs-2004-27-a-large_web.jpg
131
+ - lib/logstash/web/public/js/flot/examples/thresholding.html
132
+ - lib/logstash/web/public/js/flot/examples/time.html
133
+ - lib/logstash/web/public/js/flot/examples/interacting.html
134
+ - lib/logstash/web/public/js/flot/examples/index.html
135
+ - lib/logstash/web/public/js/flot/examples/arrow-left.gif
136
+ - lib/logstash/web/public/js/flot/examples/ajax.html
137
+ - lib/logstash/web/public/js/flot/jquery.flot.threshold.js
138
+ - lib/logstash/web/public/js/flot/jquery.flot.selection.js
139
+ - lib/logstash/web/public/js/flot/Makefile
140
+ - lib/logstash/web/public/js/flot/FAQ.txt
141
+ - lib/logstash/web/public/js/flot/jquery.flot.crosshair.min.js
142
+ - lib/logstash/web/public/js/flot/jquery.flot.selection.min.js
143
+ - lib/logstash/web/public/js/flot/jquery.colorhelpers.js
144
+ - lib/logstash/web/public/js/flot/jquery.colorhelpers.min.js
145
+ - lib/logstash/web/public/js/flot/NEWS.txt
146
+ - lib/logstash/web/public/js/flot/jquery.js
147
+ - lib/logstash/web/public/js/flot/jquery.flot.threshold.min.js
148
+ - lib/logstash/web/public/js/flot/excanvas.min.js
149
+ - lib/logstash/web/public/js/flot/API.txt
150
+ - lib/logstash/web/public/js/flot/jquery.flot.stack.min.js
151
+ - lib/logstash/web/public/js/flot/jquery.flot.image.min.js
152
+ - lib/logstash/web/public/js/flot/jquery.flot.min.js
153
+ - lib/logstash/web/public/js/flot/jquery.flot.crosshair.js
154
+ - lib/logstash/web/public/js/flot/jquery.flot.js
155
+ - lib/logstash/web/public/js/flot/jquery.flot.image.js
156
+ - lib/logstash/web/public/js/flot/jquery.flot.stack.js
157
+ - lib/logstash/web/public/js/flot/excanvas.js
158
+ - lib/logstash/web/public/js/flot/LICENSE.txt
159
+ - lib/logstash/web/public/js/flot/jquery.flot.navigate.min.js
160
+ - lib/logstash/web/public/js/flot/jquery.flot.navigate.js
161
+ - lib/logstash/web/public/js/flot/jquery.min.js
162
+ - lib/logstash/web/public/js/flot/PLUGINS.txt
163
+ - lib/logstash/web/server.rb
164
+ - lib/logstash/web/views/style.sass
165
+ - lib/logstash/web/views/main/index.haml
166
+ - lib/logstash/web/views/layout.haml
167
+ - lib/logstash/web/views/search/ajax.haml
168
+ - lib/logstash/web/views/search/results.haml
169
+ - lib/logstash/web/views/header.haml
170
+ - examples/test.rb
171
+ - etc/tograylog.yaml
172
+ - etc/logstash-elasticsearch-rabbitmq-river.yaml
173
+ - etc/logstash-reader.yaml
174
+ - etc/logstash-parser.yaml
175
+ - etc/logstash-mongodb-storage.yaml
176
+ - etc/logstash-standalone.yaml
177
+ - etc/redhat/logstash.sysconfig
178
+ - etc/redhat/logstash-agent.sysconfig
179
+ - etc/redhat/logstash.spec
180
+ - etc/redhat/logstash
181
+ - etc/redhat/logstash-agent
182
+ - etc/prod.yaml
183
+ - etc/logstash-shipper.yaml
184
+ - patterns/linux-syslog
185
+ - patterns/haproxy
186
+ - patterns/grok-patterns
187
+ - patterns/ruby
188
+ - patterns/firewalls
189
+ - patterns/nagios
190
+ - bin/logstash
191
+ - bin/logstash-web
192
+ has_rdoc: true
193
+ homepage: http://code.google.com/p/logstash/
194
+ licenses: []
195
+
196
+ post_install_message:
197
+ rdoc_options: []
198
+
199
+ require_paths:
200
+ - lib
201
+ - lib
202
+ required_ruby_version: !ruby/object:Gem::Requirement
203
+ none: false
204
+ requirements:
205
+ - - ">="
206
+ - !ruby/object:Gem::Version
207
+ hash: 3
208
+ segments:
209
+ - 0
210
+ version: "0"
211
+ required_rubygems_version: !ruby/object:Gem::Requirement
212
+ none: false
213
+ requirements:
214
+ - - ">="
215
+ - !ruby/object:Gem::Version
216
+ hash: 3
217
+ segments:
218
+ - 0
219
+ version: "0"
220
+ requirements: []
221
+
222
+ rubyforge_project:
223
+ rubygems_version: 1.3.7
224
+ signing_key:
225
+ specification_version: 3
226
+ summary: logstash - log and event management (lite install, no dependencies)
227
+ test_files: []
228
+