logstash-lite 0.2.20101118134500

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. data/bin/logstash +56 -0
  2. data/bin/logstash-web +6 -0
  3. data/etc/logstash-elasticsearch-rabbitmq-river.yaml +41 -0
  4. data/etc/logstash-mongodb-storage.yaml +5 -0
  5. data/etc/logstash-parser.yaml +20 -0
  6. data/etc/logstash-reader.yaml +8 -0
  7. data/etc/logstash-shipper.yaml +18 -0
  8. data/etc/logstash-standalone.yaml +47 -0
  9. data/etc/prod.yaml +38 -0
  10. data/etc/redhat/logstash +92 -0
  11. data/etc/redhat/logstash-agent +83 -0
  12. data/etc/redhat/logstash-agent.sysconfig +7 -0
  13. data/etc/redhat/logstash.spec +171 -0
  14. data/etc/redhat/logstash.sysconfig +18 -0
  15. data/etc/tograylog.yaml +37 -0
  16. data/examples/test.rb +38 -0
  17. data/lib/logstash.rb +3 -0
  18. data/lib/logstash/agent.rb +116 -0
  19. data/lib/logstash/event.rb +70 -0
  20. data/lib/logstash/filters.rb +17 -0
  21. data/lib/logstash/filters/base.rb +17 -0
  22. data/lib/logstash/filters/date.rb +59 -0
  23. data/lib/logstash/filters/field.rb +29 -0
  24. data/lib/logstash/filters/grok.rb +74 -0
  25. data/lib/logstash/filters/grokdiscovery.rb +60 -0
  26. data/lib/logstash/inputs.rb +18 -0
  27. data/lib/logstash/inputs/amqp.rb +48 -0
  28. data/lib/logstash/inputs/base.rb +32 -0
  29. data/lib/logstash/inputs/file.rb +47 -0
  30. data/lib/logstash/inputs/syslog.rb +123 -0
  31. data/lib/logstash/inputs/tcp.rb +51 -0
  32. data/lib/logstash/logging.rb +82 -0
  33. data/lib/logstash/namespace.rb +6 -0
  34. data/lib/logstash/outputs.rb +15 -0
  35. data/lib/logstash/outputs/amqp.rb +48 -0
  36. data/lib/logstash/outputs/base.rb +29 -0
  37. data/lib/logstash/outputs/elasticsearch.rb +71 -0
  38. data/lib/logstash/outputs/gelf.rb +35 -0
  39. data/lib/logstash/outputs/mongodb.rb +19 -0
  40. data/lib/logstash/outputs/stdout.rb +15 -0
  41. data/lib/logstash/outputs/websocket.rb +35 -0
  42. data/lib/logstash/time.rb +27 -0
  43. data/lib/logstash/web/lib/elasticsearch.rb +79 -0
  44. data/lib/logstash/web/public/css/smoothness/images/ui-bg_flat_0_aaaaaa_40x100.png +0 -0
  45. data/lib/logstash/web/public/css/smoothness/images/ui-bg_flat_75_ffffff_40x100.png +0 -0
  46. data/lib/logstash/web/public/css/smoothness/images/ui-bg_glass_55_fbf9ee_1x400.png +0 -0
  47. data/lib/logstash/web/public/css/smoothness/images/ui-bg_glass_65_ffffff_1x400.png +0 -0
  48. data/lib/logstash/web/public/css/smoothness/images/ui-bg_glass_75_dadada_1x400.png +0 -0
  49. data/lib/logstash/web/public/css/smoothness/images/ui-bg_glass_75_e6e6e6_1x400.png +0 -0
  50. data/lib/logstash/web/public/css/smoothness/images/ui-bg_glass_95_fef1ec_1x400.png +0 -0
  51. data/lib/logstash/web/public/css/smoothness/images/ui-bg_highlight-soft_75_cccccc_1x100.png +0 -0
  52. data/lib/logstash/web/public/css/smoothness/images/ui-icons_222222_256x240.png +0 -0
  53. data/lib/logstash/web/public/css/smoothness/images/ui-icons_2e83ff_256x240.png +0 -0
  54. data/lib/logstash/web/public/css/smoothness/images/ui-icons_454545_256x240.png +0 -0
  55. data/lib/logstash/web/public/css/smoothness/images/ui-icons_888888_256x240.png +0 -0
  56. data/lib/logstash/web/public/css/smoothness/images/ui-icons_cd0a0a_256x240.png +0 -0
  57. data/lib/logstash/web/public/css/smoothness/jquery-ui-1.8.5.custom.css +572 -0
  58. data/lib/logstash/web/public/js/flot/API.txt +1024 -0
  59. data/lib/logstash/web/public/js/flot/FAQ.txt +71 -0
  60. data/lib/logstash/web/public/js/flot/LICENSE.txt +22 -0
  61. data/lib/logstash/web/public/js/flot/Makefile +15 -0
  62. data/lib/logstash/web/public/js/flot/NEWS.txt +340 -0
  63. data/lib/logstash/web/public/js/flot/PLUGINS.txt +105 -0
  64. data/lib/logstash/web/public/js/flot/README.txt +81 -0
  65. data/lib/logstash/web/public/js/flot/examples/ajax.html +143 -0
  66. data/lib/logstash/web/public/js/flot/examples/annotating.html +75 -0
  67. data/lib/logstash/web/public/js/flot/examples/arrow-down.gif +0 -0
  68. data/lib/logstash/web/public/js/flot/examples/arrow-left.gif +0 -0
  69. data/lib/logstash/web/public/js/flot/examples/arrow-right.gif +0 -0
  70. data/lib/logstash/web/public/js/flot/examples/arrow-up.gif +0 -0
  71. data/lib/logstash/web/public/js/flot/examples/basic.html +38 -0
  72. data/lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-1.json +4 -0
  73. data/lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-2.json +4 -0
  74. data/lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-3.json +4 -0
  75. data/lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-4.json +4 -0
  76. data/lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-5.json +4 -0
  77. data/lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth.json +4 -0
  78. data/lib/logstash/web/public/js/flot/examples/data-japan-gdp-growth.json +4 -0
  79. data/lib/logstash/web/public/js/flot/examples/data-usa-gdp-growth.json +4 -0
  80. data/lib/logstash/web/public/js/flot/examples/dual-axis.html +39 -0
  81. data/lib/logstash/web/public/js/flot/examples/graph-types.html +75 -0
  82. data/lib/logstash/web/public/js/flot/examples/hs-2004-27-a-large_web.jpg +0 -0
  83. data/lib/logstash/web/public/js/flot/examples/image.html +45 -0
  84. data/lib/logstash/web/public/js/flot/examples/index.html +43 -0
  85. data/lib/logstash/web/public/js/flot/examples/interacting.html +93 -0
  86. data/lib/logstash/web/public/js/flot/examples/layout.css +6 -0
  87. data/lib/logstash/web/public/js/flot/examples/navigate.html +118 -0
  88. data/lib/logstash/web/public/js/flot/examples/selection.html +114 -0
  89. data/lib/logstash/web/public/js/flot/examples/setting-options.html +65 -0
  90. data/lib/logstash/web/public/js/flot/examples/stacking.html +77 -0
  91. data/lib/logstash/web/public/js/flot/examples/thresholding.html +54 -0
  92. data/lib/logstash/web/public/js/flot/examples/time.html +71 -0
  93. data/lib/logstash/web/public/js/flot/examples/tracking.html +95 -0
  94. data/lib/logstash/web/public/js/flot/examples/turning-series.html +98 -0
  95. data/lib/logstash/web/public/js/flot/examples/visitors.html +90 -0
  96. data/lib/logstash/web/public/js/flot/examples/zooming.html +98 -0
  97. data/lib/logstash/web/public/js/flot/excanvas.js +1427 -0
  98. data/lib/logstash/web/public/js/flot/excanvas.min.js +1 -0
  99. data/lib/logstash/web/public/js/flot/jquery.colorhelpers.js +174 -0
  100. data/lib/logstash/web/public/js/flot/jquery.colorhelpers.min.js +1 -0
  101. data/lib/logstash/web/public/js/flot/jquery.flot.crosshair.js +156 -0
  102. data/lib/logstash/web/public/js/flot/jquery.flot.crosshair.min.js +1 -0
  103. data/lib/logstash/web/public/js/flot/jquery.flot.image.js +237 -0
  104. data/lib/logstash/web/public/js/flot/jquery.flot.image.min.js +1 -0
  105. data/lib/logstash/web/public/js/flot/jquery.flot.js +2119 -0
  106. data/lib/logstash/web/public/js/flot/jquery.flot.min.js +1 -0
  107. data/lib/logstash/web/public/js/flot/jquery.flot.navigate.js +272 -0
  108. data/lib/logstash/web/public/js/flot/jquery.flot.navigate.min.js +1 -0
  109. data/lib/logstash/web/public/js/flot/jquery.flot.selection.js +299 -0
  110. data/lib/logstash/web/public/js/flot/jquery.flot.selection.min.js +1 -0
  111. data/lib/logstash/web/public/js/flot/jquery.flot.stack.js +152 -0
  112. data/lib/logstash/web/public/js/flot/jquery.flot.stack.min.js +1 -0
  113. data/lib/logstash/web/public/js/flot/jquery.flot.threshold.js +103 -0
  114. data/lib/logstash/web/public/js/flot/jquery.flot.threshold.min.js +1 -0
  115. data/lib/logstash/web/public/js/flot/jquery.js +4376 -0
  116. data/lib/logstash/web/public/js/flot/jquery.min.js +19 -0
  117. data/lib/logstash/web/public/js/jquery-hashchange-1.0.0.js +121 -0
  118. data/lib/logstash/web/public/js/jquery.livequery.js +250 -0
  119. data/lib/logstash/web/public/js/jquery.tmpl.min.js +1 -0
  120. data/lib/logstash/web/public/js/logstash.js +202 -0
  121. data/lib/logstash/web/server.rb +90 -0
  122. data/lib/logstash/web/views/header.haml +8 -0
  123. data/lib/logstash/web/views/layout.haml +21 -0
  124. data/lib/logstash/web/views/main/index.haml +5 -0
  125. data/lib/logstash/web/views/search/ajax.haml +32 -0
  126. data/lib/logstash/web/views/search/results.haml +17 -0
  127. data/lib/logstash/web/views/style.sass +50 -0
  128. data/patterns/firewalls +2 -0
  129. data/patterns/grok-patterns +90 -0
  130. data/patterns/haproxy +5 -0
  131. data/patterns/linux-syslog +7 -0
  132. data/patterns/nagios +7 -0
  133. data/patterns/ruby +2 -0
  134. metadata +228 -0
@@ -0,0 +1,90 @@
1
+ #!/usr/bin/env ruby
2
+ ##rackup -Ilib:../lib -s thin
3
+
4
+ $:.unshift("%s/../lib" % File.dirname(__FILE__))
5
+ $:.unshift(File.dirname(__FILE__))
6
+
7
+ require "rubygems"
8
+ require "json"
9
+ require "eventmachine"
10
+ require "rack"
11
+ require "sinatra/async"
12
+ require "lib/elasticsearch"
13
+ require "logstash/namespace"
14
+
15
+ class EventMachine::ConnectionError < RuntimeError; end
16
+
17
+ class LogStash::Web::Server < Sinatra::Base
18
+ register Sinatra::Async
19
+ set :haml, :format => :html5
20
+ set :logging, true
21
+ set :public, "#{File.dirname(__FILE__)}/public"
22
+ set :views, "#{File.dirname(__FILE__)}/views"
23
+ elasticsearch = LogStash::Web::ElasticSearch.new
24
+
25
+ aget '/style.css' do
26
+ headers "Content-Type" => "text/css; charset=utf8"
27
+ body sass :style
28
+ end
29
+
30
+ aget '/' do
31
+ redirect "/search"
32
+ end # '/'
33
+
34
+ aget '/search' do
35
+ headers({"Content-Type" => "text/html" })
36
+ if params[:q] and params[:q] != ""
37
+ elasticsearch.search(params) do |@results|
38
+ @hits = (@results["hits"]["hits"] rescue [])
39
+ body haml :"search/results", :layout => !request.xhr?
40
+ end
41
+ else
42
+ @hits = []
43
+ body haml :"search/results", :layout => !request.xhr?
44
+ end
45
+ end
46
+
47
+ apost '/search/ajax' do
48
+ headers({"Content-Type" => "text/html" })
49
+ count = params["count"] = (params["count"] or 50).to_i
50
+ offset = params["offset"] = (params["offset"] or 0).to_i
51
+ elasticsearch.search(params) do |@results|
52
+ @hits = (@results["hits"]["hits"] rescue [])
53
+ @total = (@results["hits"]["total"] rescue 0)
54
+ @graphpoints = []
55
+ @results["facets"]["by_hour"]["entries"].each do |entry|
56
+ @graphpoints << [entry["key"], entry["count"]]
57
+ end
58
+
59
+ if count and offset
60
+ if @total > (count + offset)
61
+ @result_end = (count + offset)
62
+ else
63
+ @result_end = @total
64
+ end
65
+ @result_start = offset
66
+ end
67
+
68
+ if count + offset < @total
69
+ next_params = params.clone
70
+ next_params["offset"] = [offset + count, @total - count].min
71
+ @next_href = "?" + next_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
72
+ end
73
+
74
+ if offset > 0
75
+ prev_params = params.clone
76
+ prev_params["offset"] = [offset - count, 0].max
77
+ @prev_href = "?" + prev_params.collect { |k,v| [URI.escape(k.to_s), URI.escape(v.to_s)].join("=") }.join("&")
78
+ end
79
+
80
+ body haml :"search/ajax", :layout => !request.xhr?
81
+ end # elasticsearch.search
82
+ end # apost '/search/ajax'
83
+
84
+ end # class LogStashWeb
85
+
86
+ Rack::Handler::Thin.run(
87
+ Rack::CommonLogger.new( \
88
+ Rack::ShowExceptions.new( \
89
+ LogStash::Web::Server.new)),
90
+ :Port => 9292)
@@ -0,0 +1,8 @@
1
+ .logo
2
+ logstash.
3
+ .search
4
+ %form.search{ :action => "/search" }
5
+ %label{ :for => "q" } Query:
6
+ %input.query{ :id => "query", :type => "text", :name => "q", :value => params[:q],
7
+ :size => 60 }
8
+ %input{ :id => "searchbutton", :type => "submit" }
@@ -0,0 +1,21 @@
1
+ !!! 5
2
+ %html
3
+ %head
4
+ %title= @title || "logstash"
5
+ %link{ :rel => "stylesheet", :href => "/style.css", :type => "text/css" }
6
+ %link{ :rel => "stylesheet", :href => "/css/smoothness/jquery-ui-1.8.5.custom.css", :type => "text/css" }
7
+ %script{ :src => "https://ajax.googleapis.com/ajax/libs/jquery/1.4.3/jquery.min.js",
8
+ :type => "text/javascript" }
9
+ %body
10
+ #header
11
+ =haml :header, :layout => false
12
+ #content
13
+ =yield
14
+ #footer
15
+
16
+ %script{ :src => "https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.5/jquery-ui.min.js",
17
+ :type => "text/javascript" }
18
+ %script{ :src => "js/jquery.tmpl.min.js", :type => "text/javascript" }
19
+ %script{ :src => "js/jquery-hashchange-1.0.0.js", :type => "text/javascript" }
20
+ %script{ :src => "js/flot/jquery.flot.js", :type => "text/javascript" }
21
+ %script{ :src => "js/logstash.js", :type => "text/javascript" }
@@ -0,0 +1,5 @@
1
+ %form.search{ :action => "/search" }
2
+ %label{ :for => "q" }
3
+ %input.query{ :type => "text", :name => "q" }
4
+ %input{ :type => "submit" }
5
+
@@ -0,0 +1,32 @@
1
+ #results
2
+ - if (params[:q].strip.length > 0 rescue false)
3
+ %h1
4
+ Search results for '#{params[:q]}'
5
+ - if @graphpoints
6
+ #visual
7
+ :javascript
8
+ $(function() {
9
+ var graphdata = #{@graphpoints.to_json};
10
+ window.logstash.plot(graphdata);
11
+ });
12
+ - if @total and @result_start and @result_end
13
+ %small
14
+ %strong
15
+ Results #{@result_start} - #{@result_end} of #{@total}
16
+ |
17
+ - if @prev_href
18
+ %a.pager{ :href => @prev_href }
19
+ prev
20
+ - if @next_href
21
+ |
22
+ - if @next_href
23
+ %a.pager{ :href => @next_href }
24
+ next
25
+ %ul.results
26
+ - @hits.reverse.each do |hit|
27
+ %li.event{ :"data-full" => hit.to_json }&= hit["_source"]["@message"]
28
+ - if @hits.length == 0
29
+ - if !params[:q]
30
+ No query given. How about <a href="?q=*" class="querychanger">this?</a>
31
+ - else
32
+ No results for query '#{params[:q]}'
@@ -0,0 +1,17 @@
1
+ - if @error
2
+ #error
3
+ %strong A search error occurred:
4
+ =@error
5
+ #ssquery{ :style => "display: none;", :"data-query" => params[:q] }
6
+ #inspector{ :style => "display: none;" }
7
+ The following fields are known for the log you selected. Click on any link to
8
+ append it to your search. If you shift+click, the field will be added to the
9
+ search as an exclude rather than include.
10
+ %ul
11
+
12
+ %i
13
+ You can click on any search result to see what kind of fields we know about
14
+ for that event. You can also click on the graph to zoom to that time period.
15
+ The query language is that of Lucene's string query (<a href="http://lucene.apache.org/java/2_4_0/queryparsersyntax.html">docs</a>).
16
+
17
+ =haml :"search/ajax", :layout => false
@@ -0,0 +1,50 @@
1
+ $lightgrey: #d8d8d8
2
+ $darkgrey: #adadad
3
+ body
4
+ margin: 0
5
+ padding: 0
6
+ #header
7
+ border-top: 4px solid black
8
+ border-bottom: 1px solid black
9
+ background-color: lightgreen
10
+ padding-left: 1em
11
+
12
+ .search
13
+ display: inline
14
+ .logo
15
+ font-size: 130%
16
+ font-weight: bold
17
+ float: right
18
+ padding-right: 20px
19
+ #content
20
+ margin-left: 2em
21
+ margin-right: 2em
22
+ margin-top: 1em
23
+ #content ul.results
24
+ font-family: monospace
25
+ #content li.event
26
+ padding-bottom: 3px
27
+ white-space: pre-wrap
28
+ #content li.selected
29
+ background-color: #FCE69D !important
30
+ #content li.event:nth-child(2n)
31
+ background-color: #E3F6CE
32
+ #content li.event:nth-child(2n+1)
33
+ background-color: #F5FBEF
34
+ #content li.event:hover
35
+ background-color: lightgreen
36
+ #error
37
+ background-color: pink
38
+ border: 1px solid red
39
+ padding: 3px
40
+ #error h1
41
+ font-size: 130%
42
+ padding: 0
43
+ margin: 0
44
+ #inspector
45
+ font-size: 70%
46
+ #results #visual
47
+ width: 850px
48
+ height: 200px
49
+ #results h1
50
+ font-size: 100%
@@ -0,0 +1,2 @@
1
+ # NetScreen firewall logs
2
+ NETSCREENSESSIONLOG %{SYSLOGDATE:date} %{IPORHOST:device} %{IPORHOST}: NetScreen device_id=%{WORD:device_id}%{DATA}: start_time=%{QUOTEDSTRING:start_time} duration=%{INT:duration} policy_id=%{INT:policy_id} service=%{DATA:service} proto=%{INT:proto} src zone=%{WORD:src_zone} dst zone=%{WORD:dst_zone} action=%{WORD:action} sent=%{INT:sent} rcvd=%{INT:rcvd} src=%{IPORHOST:src_ip} dst=%{IPORHOST:dst_ip} src_port=%{INT:src_port} dst_port=%{INT:dst_port} src-xlated ip=%{IPORHOST:src_xlated_ip} port=%{INT:src_xlated_port} dst-xlated ip=%{IPORHOST:dst_xlated_ip} port=%{INT:dst_xlated_port} session_id=%{INT:session_id} reason=%{GREEDYDATA:reason}
@@ -0,0 +1,90 @@
1
+ USERNAME [a-zA-Z0-9_-]+
2
+ USER %{USERNAME}
3
+ INT (?:[+-]?(?:[0-9]+))
4
+ BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
5
+ NUMBER (?:%{BASE10NUM})
6
+ BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
7
+ BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
8
+
9
+ POSINT \b(?:[0-9]+)\b
10
+ WORD \b\w+\b
11
+ NOTSPACE \S+
12
+ DATA .*?
13
+ GREEDYDATA .*
14
+ QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
15
+
16
+ # Networking
17
+ MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
18
+ CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
19
+ WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
20
+ COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
21
+ IP (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
22
+ HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
23
+ HOST %{HOSTNAME}
24
+ IPORHOST (?:%{HOSTNAME}|%{IP})
25
+ HOSTPORT (?:%{IPORHOST=~/\./}:%{POSINT})
26
+
27
+ # paths
28
+ PATH (?:%{UNIXPATH}|%{WINPATH})
29
+ UNIXPATH (?<![\w\\/])(?:/(?:[\w_%!$@:.,-]+|\\.)*)+
30
+ #UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+
31
+ LINUXTTY (?:/dev/pts/%{POSINT})
32
+ BSDTTY (?:/dev/tty[pq][a-z0-9])
33
+ TTY (?:%{BSDTTY}|%{LINUXTTY})
34
+ WINPATH (?:[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
35
+ URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
36
+ URIHOST %{IPORHOST}(?::%{POSINT})?
37
+ # uripath comes loosely from RFC1738, but mostly from what Firefox
38
+ # doesn't turn into %XX
39
+ URIPATH (?:/[A-Za-z0-9$.+!*'(),~:#%_-]*)+
40
+ #URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
41
+ URIPARAM \?[A-Za-z0-9$.+!*'(),~#%&/=:;_-]*
42
+ URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
43
+ URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:$|%{URIPATHPARAM})
44
+
45
+ # Months: January, Feb, 3, 03, 12, December
46
+ MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
47
+ MONTHNUM (?:0?[1-9]|1[0-2])
48
+ MONTHDAY (?:3[01]|[1-2]?[0-9]|0?[1-9])
49
+
50
+ # Days: Monday, Tue, Thu, etc...
51
+ DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
52
+
53
+ # Years?
54
+ YEAR [0-9]+
55
+ # Time: HH:MM:SS
56
+ #TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
57
+ # I'm still on the fence about using grok to perform the time match,
58
+ # since it's probably slower.
59
+ # TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)?
60
+ HOUR (?:2[0123]|[01][0-9])
61
+ MINUTE (?:[0-5][0-9])
62
+ # '60' is a leap second in most time standards and thus is valid.
63
+ SECOND (?:(?:[0-5][0-9]|60)(?:[.,][0-9]+)?)
64
+ TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
65
+ # datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
66
+ DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
67
+ DATE_EU %{YEAR}[/-]%{MONTHNUM}[/-]%{MONTHDAY}
68
+ ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
69
+ ISO8601_SECOND (?:%{SECOND}|60)
70
+ TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
71
+ DATE %{DATE_US}|%{DATE_EU}
72
+ DATESTAMP %{DATE}[- ]%{TIME}
73
+ TZ (?:[PMCE][SD]T)
74
+ DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
75
+ DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} (?:%{TZ} )?%{YEAR}
76
+
77
+ # Syslog Dates: Month Day HH:MM:SS
78
+ SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
79
+ PROG (?:[\w._/-]+)
80
+ SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
81
+ SYSLOGHOST %{IPORHOST}
82
+ SYSLOGFACILITY <%{POSINT:facility}.%{POSINT:priority}>
83
+ HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT:ZONE}
84
+
85
+ # Shortcuts
86
+ QS %{QUOTEDSTRING}
87
+
88
+ # Log formats
89
+ SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
90
+ COMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) "(?:%{URI:referrer}|-)" %{QS:agent}
data/patterns/haproxy ADDED
@@ -0,0 +1,5 @@
1
+ HAPROXYDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME}.%{INT:milliseconds}
2
+ HAPROXYTERMINATIONSTATE [CAPRIcs-][RQCHDLT-][NIDV-][NIPRD-]
3
+
4
+ # parse an haproxy 'httplog' line
5
+ HAPROXYHTTP %{SYSLOGDATE:date} %{IPORHOST:server} %{SYSLOGPROG}: %{IP:clientip}:%{INT:clientport} \[%{HAPROXYDATE:haproxydate}\] %{NOTSPACE:proxyname} %{NOTSPACE}/%{IPORHOST:backend} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{INT:time_duration} %{INT:response} %{INT:bytes} - - %{HAPROXYTERMINATIONSTATE:terminationstate} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn} %{INT:srv_queue}/%{INT:backend_queue} "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:version}"
@@ -0,0 +1,7 @@
1
+ SYSLOGBASE2 (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
2
+ SYSLOGPAMSESSION %{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:pam_module}\(%{DATA:pam_caller}\): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})?
3
+
4
+ CRON_ACTION [A-Z ]+
5
+ CRONLOG %{SYSLOGBASE} \(%{USER:user}\) %{CRON_ACTION:action} \(%{DATA:message}\)
6
+
7
+ SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message}
data/patterns/nagios ADDED
@@ -0,0 +1,7 @@
1
+ NAGIOSTIME \[%{NUMBER:epochtime}\]
2
+
3
+ NAGIOS_SERVICE_ALERT SERVICE ALERT: %{IPORHOST:hostname};%{DATA:checkname};%{DATA:state};%{DATA:statelevel};%{NUMBER:attempt};%{GREEDYDATA:message}
4
+ NAGIOS_SERVICE_FLAPPING_ALERT SERVICE FLAPPING ALERT: %{IPORHOST:hostname};%{DATA:checkname};%{DATA:state};%{GREEDYDATA:message}
5
+ NAGIOS_SERVICE_NOTIFICATION SERVICE NOTIFICATION: %{DATA:notifyname};%{IPORHOST:hostname};%{DATA:checkname};%{DATA:state};%{DATA:contact};%{GREEDYDATA:message}
6
+
7
+ NAGIOSLOGLINE %{NAGIOSTIME} (?:%{NAGIOS_SERVICE_ALERT}|%{NAGIOS_SERVICE_FLAPPING_ALERT}|%{NAGIOS_SERVICE_NOTIFICATION})
data/patterns/ruby ADDED
@@ -0,0 +1,2 @@
1
+ RUBY_LOGLEVEL (?:DEBUG|FATAL|ERROR|WARN|INFO)
2
+ RUBY_LOGGER [DFEWI], \[%{TIMESTAMP_ISO8601} #{POSINT:pid}\] *%{RUBY_LOGLEVEL} -- %{DATA:progname}: %{DATA:message}
metadata ADDED
@@ -0,0 +1,228 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-lite
3
+ version: !ruby/object:Gem::Version
4
+ hash: 40202236269023
5
+ prerelease: false
6
+ segments:
7
+ - 0
8
+ - 2
9
+ - 20101118134500
10
+ version: 0.2.20101118134500
11
+ platform: ruby
12
+ authors:
13
+ - Jordan Sissel
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2010-11-18 00:00:00 -08:00
19
+ default_executable:
20
+ dependencies:
21
+ - !ruby/object:Gem::Dependency
22
+ name: eventmachine-tail
23
+ prerelease: false
24
+ requirement: &id001 !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ hash: 3
30
+ segments:
31
+ - 0
32
+ version: "0"
33
+ type: :runtime
34
+ version_requirements: *id001
35
+ - !ruby/object:Gem::Dependency
36
+ name: json
37
+ prerelease: false
38
+ requirement: &id002 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ hash: 3
44
+ segments:
45
+ - 0
46
+ version: "0"
47
+ type: :runtime
48
+ version_requirements: *id002
49
+ description: scalable log and event management (search, archive, pipeline)
50
+ email: jls@semicomplete.com
51
+ executables:
52
+ - logstash
53
+ - logstash-web
54
+ extensions: []
55
+
56
+ extra_rdoc_files: []
57
+
58
+ files:
59
+ - lib/logstash.rb
60
+ - lib/logstash/agent.rb
61
+ - lib/logstash/inputs.rb
62
+ - lib/logstash/inputs/syslog.rb
63
+ - lib/logstash/inputs/file.rb
64
+ - lib/logstash/inputs/base.rb
65
+ - lib/logstash/inputs/amqp.rb
66
+ - lib/logstash/inputs/tcp.rb
67
+ - lib/logstash/outputs/gelf.rb
68
+ - lib/logstash/outputs/elasticsearch.rb
69
+ - lib/logstash/outputs/mongodb.rb
70
+ - lib/logstash/outputs/stdout.rb
71
+ - lib/logstash/outputs/websocket.rb
72
+ - lib/logstash/outputs/base.rb
73
+ - lib/logstash/outputs/amqp.rb
74
+ - lib/logstash/namespace.rb
75
+ - lib/logstash/time.rb
76
+ - lib/logstash/filters.rb
77
+ - lib/logstash/outputs.rb
78
+ - lib/logstash/filters/grokdiscovery.rb
79
+ - lib/logstash/filters/grok.rb
80
+ - lib/logstash/filters/base.rb
81
+ - lib/logstash/filters/field.rb
82
+ - lib/logstash/filters/date.rb
83
+ - lib/logstash/logging.rb
84
+ - lib/logstash/event.rb
85
+ - lib/logstash/web/lib/elasticsearch.rb
86
+ - lib/logstash/web/public/css/smoothness/jquery-ui-1.8.5.custom.css
87
+ - lib/logstash/web/public/css/smoothness/images/ui-icons_2e83ff_256x240.png
88
+ - lib/logstash/web/public/css/smoothness/images/ui-icons_cd0a0a_256x240.png
89
+ - lib/logstash/web/public/css/smoothness/images/ui-icons_222222_256x240.png
90
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_highlight-soft_75_cccccc_1x100.png
91
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_75_e6e6e6_1x400.png
92
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_65_ffffff_1x400.png
93
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_flat_0_aaaaaa_40x100.png
94
+ - lib/logstash/web/public/css/smoothness/images/ui-icons_454545_256x240.png
95
+ - lib/logstash/web/public/css/smoothness/images/ui-icons_888888_256x240.png
96
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_75_dadada_1x400.png
97
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_55_fbf9ee_1x400.png
98
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_flat_75_ffffff_40x100.png
99
+ - lib/logstash/web/public/css/smoothness/images/ui-bg_glass_95_fef1ec_1x400.png
100
+ - lib/logstash/web/public/js/jquery.livequery.js
101
+ - lib/logstash/web/public/js/logstash.js
102
+ - lib/logstash/web/public/js/jquery-hashchange-1.0.0.js
103
+ - lib/logstash/web/public/js/jquery.tmpl.min.js
104
+ - lib/logstash/web/public/js/flot/README.txt
105
+ - lib/logstash/web/public/js/flot/examples/tracking.html
106
+ - lib/logstash/web/public/js/flot/examples/layout.css
107
+ - lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-1.json
108
+ - lib/logstash/web/public/js/flot/examples/data-japan-gdp-growth.json
109
+ - lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-2.json
110
+ - lib/logstash/web/public/js/flot/examples/setting-options.html
111
+ - lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth.json
112
+ - lib/logstash/web/public/js/flot/examples/data-usa-gdp-growth.json
113
+ - lib/logstash/web/public/js/flot/examples/stacking.html
114
+ - lib/logstash/web/public/js/flot/examples/navigate.html
115
+ - lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-4.json
116
+ - lib/logstash/web/public/js/flot/examples/graph-types.html
117
+ - lib/logstash/web/public/js/flot/examples/dual-axis.html
118
+ - lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-5.json
119
+ - lib/logstash/web/public/js/flot/examples/arrow-down.gif
120
+ - lib/logstash/web/public/js/flot/examples/annotating.html
121
+ - lib/logstash/web/public/js/flot/examples/zooming.html
122
+ - lib/logstash/web/public/js/flot/examples/selection.html
123
+ - lib/logstash/web/public/js/flot/examples/basic.html
124
+ - lib/logstash/web/public/js/flot/examples/data-eu-gdp-growth-3.json
125
+ - lib/logstash/web/public/js/flot/examples/arrow-right.gif
126
+ - lib/logstash/web/public/js/flot/examples/turning-series.html
127
+ - lib/logstash/web/public/js/flot/examples/visitors.html
128
+ - lib/logstash/web/public/js/flot/examples/image.html
129
+ - lib/logstash/web/public/js/flot/examples/arrow-up.gif
130
+ - lib/logstash/web/public/js/flot/examples/hs-2004-27-a-large_web.jpg
131
+ - lib/logstash/web/public/js/flot/examples/thresholding.html
132
+ - lib/logstash/web/public/js/flot/examples/time.html
133
+ - lib/logstash/web/public/js/flot/examples/interacting.html
134
+ - lib/logstash/web/public/js/flot/examples/index.html
135
+ - lib/logstash/web/public/js/flot/examples/arrow-left.gif
136
+ - lib/logstash/web/public/js/flot/examples/ajax.html
137
+ - lib/logstash/web/public/js/flot/jquery.flot.threshold.js
138
+ - lib/logstash/web/public/js/flot/jquery.flot.selection.js
139
+ - lib/logstash/web/public/js/flot/Makefile
140
+ - lib/logstash/web/public/js/flot/FAQ.txt
141
+ - lib/logstash/web/public/js/flot/jquery.flot.crosshair.min.js
142
+ - lib/logstash/web/public/js/flot/jquery.flot.selection.min.js
143
+ - lib/logstash/web/public/js/flot/jquery.colorhelpers.js
144
+ - lib/logstash/web/public/js/flot/jquery.colorhelpers.min.js
145
+ - lib/logstash/web/public/js/flot/NEWS.txt
146
+ - lib/logstash/web/public/js/flot/jquery.js
147
+ - lib/logstash/web/public/js/flot/jquery.flot.threshold.min.js
148
+ - lib/logstash/web/public/js/flot/excanvas.min.js
149
+ - lib/logstash/web/public/js/flot/API.txt
150
+ - lib/logstash/web/public/js/flot/jquery.flot.stack.min.js
151
+ - lib/logstash/web/public/js/flot/jquery.flot.image.min.js
152
+ - lib/logstash/web/public/js/flot/jquery.flot.min.js
153
+ - lib/logstash/web/public/js/flot/jquery.flot.crosshair.js
154
+ - lib/logstash/web/public/js/flot/jquery.flot.js
155
+ - lib/logstash/web/public/js/flot/jquery.flot.image.js
156
+ - lib/logstash/web/public/js/flot/jquery.flot.stack.js
157
+ - lib/logstash/web/public/js/flot/excanvas.js
158
+ - lib/logstash/web/public/js/flot/LICENSE.txt
159
+ - lib/logstash/web/public/js/flot/jquery.flot.navigate.min.js
160
+ - lib/logstash/web/public/js/flot/jquery.flot.navigate.js
161
+ - lib/logstash/web/public/js/flot/jquery.min.js
162
+ - lib/logstash/web/public/js/flot/PLUGINS.txt
163
+ - lib/logstash/web/server.rb
164
+ - lib/logstash/web/views/style.sass
165
+ - lib/logstash/web/views/main/index.haml
166
+ - lib/logstash/web/views/layout.haml
167
+ - lib/logstash/web/views/search/ajax.haml
168
+ - lib/logstash/web/views/search/results.haml
169
+ - lib/logstash/web/views/header.haml
170
+ - examples/test.rb
171
+ - etc/tograylog.yaml
172
+ - etc/logstash-elasticsearch-rabbitmq-river.yaml
173
+ - etc/logstash-reader.yaml
174
+ - etc/logstash-parser.yaml
175
+ - etc/logstash-mongodb-storage.yaml
176
+ - etc/logstash-standalone.yaml
177
+ - etc/redhat/logstash.sysconfig
178
+ - etc/redhat/logstash-agent.sysconfig
179
+ - etc/redhat/logstash.spec
180
+ - etc/redhat/logstash
181
+ - etc/redhat/logstash-agent
182
+ - etc/prod.yaml
183
+ - etc/logstash-shipper.yaml
184
+ - patterns/linux-syslog
185
+ - patterns/haproxy
186
+ - patterns/grok-patterns
187
+ - patterns/ruby
188
+ - patterns/firewalls
189
+ - patterns/nagios
190
+ - bin/logstash
191
+ - bin/logstash-web
192
+ has_rdoc: true
193
+ homepage: http://code.google.com/p/logstash/
194
+ licenses: []
195
+
196
+ post_install_message:
197
+ rdoc_options: []
198
+
199
+ require_paths:
200
+ - lib
201
+ - lib
202
+ required_ruby_version: !ruby/object:Gem::Requirement
203
+ none: false
204
+ requirements:
205
+ - - ">="
206
+ - !ruby/object:Gem::Version
207
+ hash: 3
208
+ segments:
209
+ - 0
210
+ version: "0"
211
+ required_rubygems_version: !ruby/object:Gem::Requirement
212
+ none: false
213
+ requirements:
214
+ - - ">="
215
+ - !ruby/object:Gem::Version
216
+ hash: 3
217
+ segments:
218
+ - 0
219
+ version: "0"
220
+ requirements: []
221
+
222
+ rubyforge_project:
223
+ rubygems_version: 1.3.7
224
+ signing_key:
225
+ specification_version: 3
226
+ summary: logstash - log and event management (lite install, no dependencies)
227
+ test_files: []
228
+