logstash-lite 0.2.20101118134500

data/bin/logstash

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname($0) + "/../lib"
+
+require "rubygems"
+require "eventmachine"
+require "logstash/agent"
+require "optparse"
+require "yaml"
+
+Settings = Struct.new(:config_file, :daemonize)
+
+settings = Settings.new
+settings.daemonize = false
+settings.config_file = nil
+progname = File.basename($0)
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: #{progname} [options]"
+
+  opts.on("-f CONFIGFILE", "--config CONFIGFILE",
+          "Load the logstash config from a specific file") do |arg|
+    settings.config_file = arg
+  end
+
+  #opts.on("-d", "--daemonize", "Daemonize (default is run in foreground)") do 
+    #settings.daemonize = true
+  #end
+end
+
+begin
+  opts.parse!
+  if settings.config_file == "" or settings.config_file == nil
+    raise "No config file given. (missing -f or --config flag?)"
+  end
+
+rescue
+  $stderr.puts "#{progname}: #{$!}"
+  $stderr.puts opts
+  exit(1)
+end
+
+# TODO(sissel): put the config management stuff in LogStash::Agent
+if !File.exist?(settings.config_file)
+  $stderr.puts "Config file '#{settings.config_file}' does not exist."
+end
+
+begin
+  config = YAML::load_file(settings.config_file)
+rescue => e
+  $stderr.puts "Loading config file '#{settings.config_file}' failed: #{e}"
+  exit 1
+end
+
+agent = LogStash::Agent.new(config)
+agent.run

data/bin/logstash-web

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname($0) + "/../lib"
+
+require "rubygems"
+require "logstash/web/server"

data/etc/logstash-elasticsearch-rabbitmq-river.yaml

@@ -0,0 +1,41 @@
+--- 
+# this is a sample logstash config (code is still highly in change, so
+# this could change later)
+#
+#
+inputs:
+# Give a list of inputs. Tag them for easy query/filter later.
+  linux-syslog: # this is the 'linux-syslog' type
+  - /var/log/messages # watch /var/log/messages (uses eventmachine-tail)
+  - /var/log/kern.log
+  - /var/log/auth.log
+  - /var/log/user.log
+  apache-access: # similar, different type.
+  - /var/log/apache2/access.log
+  - /b/access
+  apache-error:
+  - /var/log/apache2/error.log
+filters:
+- grok:
+    linux-syslog: # for logs of type 'linux-syslog'
+      patterns:
+      - %{SYSLOGLINE}
+    apache-access: # for logs of type 'apache-error'
+      patterns:
+      - %{COMBINEDAPACHELOG}
+- date:
+    linux-syslog:  # for logs of type 'linux-syslog'
+      # Look for a field 'timestamp' with this format, parse and it for the timestamp
+      # This field comes from the SYSLOGLINE pattern
+      timestamp: "%b %e %H:%M:%S"
+    apache-access:
+      timestamp: "%d/%b/%Y:%H:%M:%S %Z"
+outputs:
+- stdout:///
+- "elasticsearch://localhost:9200/logs/all?method=river&type=rabbitmq&host=127.0.0.1&user=guest&pass=guest&vhost=/&queue=es"
+# But we could write to mongodb, too.
+#  - mongodb://localhost/parsedlogs
+# And also write to an AMQP topic
+#  - amqp://localhost/topic/parsedlogs
+# Write to stdout ... etc.
+# - stdout:///

data/etc/logstash-mongodb-storage.yaml

@@ -0,0 +1,5 @@
+# Example config that reads parsed logs from AMQP and dumps results into mongodb
+inputs:
+  - amqp://localhost/topic/parsedlogs
+outputs:
+  - mongodb://localhost/parsedlogs

data/etc/logstash-parser.yaml

@@ -0,0 +1,20 @@
+# Example config that parses rawlogs with grok and puts them on another AMQP topic
+inputs:
+  - amqp://localhost/topic/rawlogs
+outputs:
+  - amqp://localhost/topic/parsedlogs
+  - stdout:///
+filters:
+  grok:
+    linux-syslog: # for logs tagged 'linux-syslog'
+      timestamp: 
+        key: date
+        format: %b %e %H:%M:%S
+      patterns:
+      - %{SYSLOGLINE}
+    apache-access: # for logs tagged 'apache-error'
+      timestamp:
+        key: timestamp
+        format: %d/%b/%Y:%H:%M:%S %Z
+      patterns:
+      - %{COMBINEDAPACHELOG}

data/etc/logstash-reader.yaml

@@ -0,0 +1,8 @@
+# Example config that reads parsed logs from AMQP and prints to stdout
+inputs:
+  - amqp://localhost/topic/parsedlogs
+#filters:
+  #field:
+  #- progname.include?("tester")
+outputs:
+  - stdout:///

data/etc/logstash-shipper.yaml

@@ -0,0 +1,18 @@
+--- 
+configname: shipper
+# Example config that only ships log data from files to an AMQP topic
+inputs:
+  linux-syslog:
+  - /var/log/messages
+  - /var/log/kern.log
+  - /var/log/auth.log
+  - /var/log/user.log
+  apache-access:
+  - /var/log/apache2/access.log
+  - /b/access
+  apache-error:
+  - /var/log/apache2/access.log
+  unknown:
+  - /b/randomdata
+outputs:
+- amqp://localhost/topic/rawlogs

data/etc/logstash-standalone.yaml

@@ -0,0 +1,47 @@
+--- 
+# this is a sample logstash config (code is still highly in change, so
+# this could change later)
+#
+#
+inputs:
+# Give a list of inputs. Tag them for easy query/filter later.
+  linux-syslog: # this is the 'linux-syslog' type
+  - /var/log/messages # watch /var/log/messages (uses eventmachine-tail)
+  - /var/log/kern.log
+  - /var/log/auth.log
+  - /var/log/user.log
+  apache-access: # similar, different type.
+  - /var/log/apache2/access.log
+  - /b/access
+  apache-error:
+  - /var/log/apache2/error.log
+filters:
+- grok:
+    linux-syslog: # for logs of type 'linux-syslog'
+      patterns:
+      - %{SYSLOGLINE}
+    apache-access: # for logs of type 'apache-error'
+      patterns:
+      - %{COMBINEDAPACHELOG}
+- grokdiscovery:
+    linux-syslog:
+    - message
+    - DATA
+    - GREEDYDATA
+- date:
+    linux-syslog:  # for logs of type 'linux-syslog'
+      # Look for a field 'timestamp' with this format, parse and it for the timestamp
+      # This field comes from the SYSLOGLINE pattern
+      timestamp: "%b %e %H:%M:%S"
+      timestamp8601: ISO8601
+    apache-access:
+      timestamp: "%d/%b/%Y:%H:%M:%S %Z"
+outputs:
+- stdout:///
+- elasticsearch://localhost:9200/logs/all
+# But we could write to mongodb, too.
+#  - mongodb://localhost/parsedlogs
+# And also write to an AMQP topic
+#  - amqp://localhost/topic/parsedlogs
+# Write to stdout ... etc.
+# - stdout:///

data/etc/prod.yaml

@@ -0,0 +1,38 @@
+--- 
+inputs:
+  all:
+  - amqp://activemq/topic/logstash-events
+  linux-syslog:
+  - /var/log/messages
+  - /var/log/kern.log
+  - /var/log/auth.log
+  - /var/log/user.log
+  apache-access:
+  - /var/log/apache2/access.log
+  apache-error:
+  - /var/log/apache2/error.log
+filters:
+- grok:
+    linux-syslog: # for logs of type 'linux-syslog'
+      patterns:
+      - %{SYSLOGLINE}
+    apache-access: # for logs of type 'apache-error'
+      patterns:
+      - %{COMBINEDAPACHELOG}
+    nagios:
+      patterns:
+      - %{NAGIOSLOGLINE}
+- date:
+    linux-syslog:  # for logs of type 'linux-syslog'
+      # Look for a field 'timestamp' with this format, parse and it for the timestamp
+      # This field comes from the SYSLOGLINE pattern
+      timestamp: "%b %e %H:%M:%S"
+      timestamp8601: ISO8601
+    apache-access:
+      timestamp: "%d/%b/%Y:%H:%M:%S %Z"
+    nagios:
+      epochtime: %s
+outputs:
+- stdout:///
+#- elasticsearch://localhost:9200/logstash/all
+- "elasticsearch://localhost:9200/logstash/all_river?method=river&type=rabbitmq&host=activemq&user=guest&pass=guest&vhost=/&queue=es"

data/etc/redhat/logstash

@@ -0,0 +1,92 @@
+#!/bin/sh
+#
+# chkconfig: - 70 45
+# description: The LogStash daemon processes, indexes, and searches logs centrally
+# processname: logstashd
+# config: /etc/sysconfig/logstash
+
+# Standard LSB functions
+#. /lib/lsb/init-functions
+
+# Source function library.
+. /etc/init.d/functions
+
+# Defaults. Override in /etc/sysconfig/logstash
+USER=logstash
+OPTIONS=""
+INDEX=true
+PARSERS=1
+DEBUG=false
+DIR=/opt/logstash
+CONFIG=$DIR/etc/logstashd.yaml
+LOGFILE=/var/logstash/logstashd.log
+
+if [ -f /etc/sysconfig/logstash ]; then 
+	. /etc/sysconfig/logstash
+fi
+
+RETVAL=0
+prog="logstashd"
+
+start () {
+	echo -n $"Starting $prog: "
+        mkdir -p /var/run/logstash
+        chown $USER /var/run/logstash
+
+	if $DEBUG; then
+		OPTIONS="$OPTIONS -d"
+	fi
+	if ! $INDEX; then
+		OPTIONS="$OPTIONS -I"
+	fi
+	OPTIONS="$OPTIONS -p $PARSERS"
+        OPTIONS="$OPTIONS --pidfile /var/run/logstash/logstashd.pid"
+        OPTIONS="$OPTIONS --logfile $LOGFILE"
+        OPTIONS="$OPTIONS $CONFIG"
+
+	daemon --user $USER --pidfile /var/run/logstash/logstashd.pid \
+	       $DIR/bin/logstashd $OPTIONS
+	RETVAL=$?
+	echo
+	[ $RETVAL -eq 0 ] && touch /var/lock/subsys/logstashd
+}
+
+stop () {
+	echo -n $"Stopping $prog: "
+	killproc -p /var/run/logstash/logstashd.pid $DIR/bin/logstashd
+	RETVAL=$?
+	echo
+	if [ $RETVAL -eq 0 ] ; then
+	    rm -f /var/lock/subsys/logstashd
+	    rm -f /var/run/logstash/logstashd.pid
+	fi
+}
+
+restart () {
+        stop
+        start
+}
+
+# See how we were called.
+case "$1" in
+  start)
+	start
+	;;
+  stop)
+	stop
+	;;
+  status)
+	status -p /var/run/logstash/logstashd.pid logstashd
+	;;
+  restart|reload|force-reload)
+	restart
+	;;
+  condrestart)
+	[ -f /var/lock/subsys/logstashd ] && restart || :
+	;;
+  *)
+	echo $"Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
+	exit 1
+esac
+
+exit $?

data/etc/redhat/logstash-agent

@@ -0,0 +1,83 @@
+#!/bin/sh
+#
+# chkconfig: - 70 45
+# description: The LogStash collection agent
+# processname: logstash-agent
+# config: /etc/sysconfig/logstash
+
+# Standard LSB functions
+#. /lib/lsb/init-functions
+
+# Source function library.
+. /etc/init.d/functions
+
+# Defaults. Override in /etc/sysconfig/logstash-agent
+OPTIONS=""
+DEBUG=false
+DIR=/opt/logstash
+CONFIG=$DIR/etc/logstash-agent.yaml
+LOGFILE=/var/log/logstash-agent.log
+
+if [ -f /etc/sysconfig/logstash-agent ]; then 
+	. /etc/sysconfig/logstash-agent
+fi
+
+RETVAL=0
+prog="logstash-agent"
+
+start () {
+	echo -n $"Starting $prog: "
+
+	if $DEBUG; then
+		OPTIONS="$OPTIONS -d"
+	fi
+        OPTIONS="$OPTIONS --pidfile /var/run/logstash-agent.pid"
+        OPTIONS="$OPTIONS --logfile $LOGFILE"
+        OPTIONS="$OPTIONS $CONFIG"
+
+	daemon --pidfile /var/run/logstash-agent.pid \
+	       $DIR/bin/logstash-agent $OPTIONS
+	RETVAL=$?
+	echo
+	[ $RETVAL -eq 0 ] && touch /var/lock/subsys/logstash-agent
+}
+
+stop () {
+	echo -n $"Stopping $prog: "
+	killproc -p /var/run/logstash-agent.pid logstash-agent
+	RETVAL=$?
+	echo
+	if [ $RETVAL -eq 0 ] ; then
+	    rm -f /var/lock/subsys/logstash-agent
+	    rm -f /var/run/logstash-agent.pid
+	fi
+}
+
+restart () {
+        stop
+        start
+}
+
+# See how we were called.
+case "$1" in
+  start)
+	start
+	;;
+  stop)
+	stop
+	;;
+  status)
+	status -p /var/run/logstash-agent.pid logstash-agent
+	;;
+  restart|reload|force-reload)
+	restart
+	;;
+  condrestart)
+	[ -f /var/lock/subsys/logstash-agent ] && restart || :
+	;;
+  *)
+	echo $"Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
+	exit 1
+esac
+
+exit $?

data/etc/redhat/logstash-agent.sysconfig

@@ -0,0 +1,7 @@
+# LogStash agent configuration
+
+# Path to config file
+CONFIG=/opt/logstash/etc/logstash-agent.yaml
+
+# Set to true for very verbose debugging output
+DEBUG=false

data/etc/redhat/logstash.spec

@@ -0,0 +1,171 @@
+%define logstash_dir /opt/logstash
+
+Name:		logstash
+Version:	0
+Release:	1
+License:	BSD
+URL:		http://code.google.com/p/logstash
+BuildRoot:	%{_tmppath}/%{name}-%{version}-%{release}-root-%(id -u)
+Source0:	http://code.google.com/p/logstash/downloads/%{name}-%{version}.tar.gz
+Summary:	Centralized log indexing and searching
+Group:		Applications/System
+BuildArch:      noarch
+
+%description
+LogStash
+
+%package server
+Summary:        LogStash parsing, indexing, and searching server
+Requires:	%{name}-libs = %{version}-%{release}
+Requires:       grok-ruby
+Requires(pre):  shadow-utils
+Requires(post): chkconfig
+Requires(preun): chkconfig
+Requires(preun): initscripts
+Requires(postun): initscripts
+Group:		Applications/System
+
+%description server
+LogStash daemon.
+
+%package libs
+Summary:	LogStash common Ruby libraries
+Requires:	ruby >= 1.8.5
+Group:		Applications/System
+
+%description libs
+LogStash Ruby libraries.
+
+%package agent
+Summary:	LogStash log file collection agent
+Requires:	%{name}-libs = %{version}-%{release}
+Requires(post): chkconfig
+Requires(preun): chkconfig
+Requires(preun): initscripts
+Requires(postun): initscripts
+Group:		Applications/System
+
+%description agent
+LogStash file collection agent.
+
+%package web
+Summary:	LogStash web query interface
+Requires:	%{name}-libs = %{version}-%{release}
+Group:		Applications/System
+
+%description web
+LogStash web query interface.
+
+%prep
+%setup
+
+%install
+rm -rf %{buildroot}
+%{__mkdir_p} %{buildroot}%{logstash_dir}
+%{__mkdir_p} %{buildroot}%{logstash_dir}/docs
+%{__mkdir_p} %{buildroot}/etc/init.d
+%{__mkdir_p} %{buildroot}/etc/sysconfig
+%{__mkdir_p} %{buildroot}/var/logstash
+tar cf - bin etc/*.yaml lib patterns web \
+  | (cd %{buildroot}%{logstash_dir} && tar xf -)
+install -c etc/redhat/logstash %{buildroot}/etc/init.d/logstash
+install -c etc/redhat/logstash-agent %{buildroot}/etc/init.d/logstash-agent
+install -c etc/redhat/logstash.sysconfig %{buildroot}/etc/sysconfig/logstash
+install -c etc/redhat/logstash-agent.sysconfig \
+        %{buildroot}/etc/sysconfig/logstash-agent
+install -c docs/AgentConfiguration.wiki %{buildroot}%{logstash_dir}/docs/
+install -c docs/Design.wiki %{buildroot}%{logstash_dir}/docs/
+install -c docs/NetworkDesign.wiki %{buildroot}%{logstash_dir}/docs/
+install -c docs/Scaling.wiki %{buildroot}%{logstash_dir}/docs/
+install -c docs/ServerConfiguration.wiki %{buildroot}%{logstash_dir}/docs/
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files libs
+%defattr(-, root, root, 0755)
+%{logstash_dir}/lib/config
+%{logstash_dir}/lib/log.rb
+%{logstash_dir}/lib/log/json.rb
+%{logstash_dir}/lib/log/text.rb
+%{logstash_dir}/lib/logs.rb
+%{logstash_dir}/lib/net/clients
+%{logstash_dir}/lib/net/common.rb
+%{logstash_dir}/lib/net/messages
+%{logstash_dir}/lib/net.rb
+%{logstash_dir}/lib/program.rb
+%{logstash_dir}/lib/util.rb
+%{logstash_dir}/lib/db/index.rb
+%{logstash_dir}/lib/db/indexreader.rb
+%{logstash_dir}/lib/programs/agent.rb
+
+%files server
+%defattr(-, root, root, 0755)
+/etc/init.d/logstash
+%config(noreplace) /etc/sysconfig/logstash
+%{logstash_dir}/bin/list_log_keys.rb
+%{logstash_dir}/bin/list_log_types.rb
+%{logstash_dir}/bin/logstashd
+%{logstash_dir}/bin/search.rb
+%{logstash_dir}/docs/Design.wiki
+%{logstash_dir}/docs/NetworkDesign.wiki
+%{logstash_dir}/docs/Scaling.wiki
+%{logstash_dir}/docs/ServerConfiguration.wiki
+%{logstash_dir}/lib/log
+%{logstash_dir}/lib/net/servers
+%{logstash_dir}/patterns
+%config(noreplace) %{logstash_dir}/etc/logstashd.yaml
+%defattr(-, logstash, logstash, 0775)
+/var/logstash
+
+%files agent
+%defattr(-, root, root, 0755)
+/etc/init.d/logstash-agent
+%config(noreplace) /etc/sysconfig/logstash-agent
+%{logstash_dir}/bin/logstash-agent
+%{logstash_dir}/docs/AgentConfiguration.wiki
+%{logstash_dir}/lib/net/clients/agent.rb
+%{logstash_dir}/lib/file/tail/since.rb
+%config(noreplace) %{logstash_dir}/etc/logstash-agent.yaml
+
+%files web
+%defattr(-, root, root, 0755)
+%{logstash_dir}/web
+
+%pre server
+getent group logstash &>/dev/null || groupadd -r logstash
+getent passwd logstash &>/dev/null || \
+  useradd -r -g logstash -d %{logstash_dir} -s /sbin/nologin \
+    -c "LogStash Daemon" logstash
+
+%post server
+/sbin/chkconfig --add logstash
+
+%post agent
+/sbin/chkconfig --add logstash-agent
+
+%preun server
+if [ "$1" = 0 ] ; then
+  /sbin/service logstash stop &>/dev/null
+  /sbin/chkconfig --del logstash &>/dev/null || true
+fi
+
+%preun agent
+if [ "$1" = 0 ] ; then
+  /sbin/service logstash-agent stop &>/dev/null
+  /sbin/chkconfig --del logstash-agent &>/dev/null || true
+fi
+
+%postun server
+if [ "$1" -ge 1 ]; then
+  /sbin/service logstash condrestart &>/dev/null || true
+fi
+
+%postun agent
+if [ "$1" -ge 1 ]; then
+  /sbin/service logstash-agent condrestart &>/dev/null || true
+fi
+
+%changelog
+* Mon Oct 19 2009 Pete Fritchman <petef@databits.net> - 0.3.0-1
+- Initial packaging