remote_syslog-gitlab 0.0.1

data/examples/log_files.yml.example

@@ -0,0 +1,9 @@
+files: 
+  - /var/log/httpd/access_log
+  - /var/log/httpd/error_log
+  - /opt/misc/*.log
+  - /var/log/mysqld.log
+  - /var/run/mysqld/mysqld-slow.log
+destination:
+  host: logs.papertrailapp.com
+  port: 12345   # NOTE: change to your Papertrail port

data/examples/log_files.yml.example.advanced

@@ -0,0 +1,21 @@
+# see README - demonstrates all optional arguments and more glob formats
+files: 
+  - /var/log/httpd/access_log
+  - /var/log/httpd/error_log
+  - /opt/misc/*.log
+  - /home/**/*.log
+  - /var/log/mysqld.log
+  - /var/run/mysqld/mysqld-slow.log
+exclude_files:
+  - old
+  - 200\d
+hostname: www42  # override OS hostname
+parse_fields: syslog   # predefined regex name or double-quoted regex
+prepend: '0xDEADBEEF: '  # prepend this before every log message
+exclude_patterns:
+  - exclude this
+  - \d+ things
+destination:
+  host: logs.papertrailapp.com
+  port: 12345   # NOTE: change this to YOUR papertrail port!
+new_file_check_interval: 5 # Check every 5 seconds

data/examples/remote_syslog.init.d

@@ -0,0 +1,110 @@
+#!/bin/bash
+
+### BEGIN INIT INFO
+# Provides: remote_syslog
+# Required-Start: $remote_fs $syslog
+# Required-Stop: $remote_fs $syslog
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: Start and Stop
+# Description: Runs remote_syslog
+### END INIT INFO
+
+#       /etc/init.d/remote_syslog
+#
+# Starts the remote_syslog daemon
+#
+# chkconfig: 345 90 5
+# description: Runs remote_syslog
+#
+# processname: remote_syslog
+
+prog="remote_syslog"
+config="/etc/log_files.yml"
+pid_dir="/var/run"
+
+EXTRAOPTIONS=""
+
+pid_file="$pid_dir/$prog.pid"
+
+PATH=/sbin:/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:$PATH
+
+RETVAL=0
+
+is_running(){
+  [ -e $pid_file ]
+}
+
+start(){
+    echo -n $"Starting $prog: "
+
+    unset HOME MAIL USER USERNAME
+    $prog -c $config --pid-file=$pid_file $EXTRAOPTIONS
+    RETVAL=$?
+    echo
+    return $RETVAL
+}
+
+stop(){
+    echo -n $"Stopping $prog: "
+    if (is_running); then
+      kill `cat $pid_file`
+      RETVAL=$?
+      echo
+      return $RETVAL
+    else
+      echo "$pid_file not found"
+    fi
+}
+
+status(){
+    echo -n $"Checking for $pid_file: "
+
+    if (is_running); then
+      echo "found"
+    else
+      echo "not found"
+    fi
+}
+
+reload(){
+    restart
+}
+
+restart(){
+    stop
+    start
+}
+
+condrestart(){
+    is_running && restart
+    return 0
+}
+
+
+# See how we were called.
+case "$1" in
+    start)
+	start
+	;;
+    stop)
+	stop
+	;;
+    status)
+	status
+	;;
+    restart)
+	restart
+	;;
+    reload)
+	reload
+	;;
+    condrestart)
+	condrestart
+	;;
+    *)
+	echo $"Usage: $0 {start|stop|status|restart|condrestart|reload}"
+	RETVAL=1
+esac
+
+exit $RETVAL

data/examples/remote_syslog.supervisor.conf

@@ -0,0 +1,9 @@
+# Supervisor conf file for stock Ubuntu 11.04 install
+[program:remote_syslog]
+
+command=/var/lib/gems/1.8/bin/remote_syslog --no-detach
+user=root
+group=root
+autostart=true
+autorestart=true
+redirect_stderr=true

data/examples/remote_syslog.upstart.conf

@@ -0,0 +1,9 @@
+description "Monitor files and send to remote syslog"
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+pre-start exec /usr/bin/test -e /etc/log_files.yml
+
+exec /var/lib/gems/1.8/bin/remote_syslog -D

data/lib/remote_syslog/agent.rb

@@ -0,0 +1,144 @@
+require 'eventmachine'
+require 'servolux'
+
+require 'remote_syslog/eventmachine_reader'
+require 'remote_syslog/file_tail_reader'
+require 'remote_syslog/glob_watch'
+require 'remote_syslog/message_generator'
+require 'remote_syslog/udp_endpoint'
+require 'remote_syslog/tls_endpoint'
+require 'remote_syslog/tcp_endpoint'
+
+module RemoteSyslog
+  class Agent < Servolux::Server
+    # Who should we connect to?
+    attr_accessor :destination_host, :destination_port
+
+    # Should use TCP?
+    attr_accessor :tcp
+
+    # Should use TLS?
+    attr_accessor :tls
+
+    # TLS settings
+    attr_accessor :client_cert_chain, :client_private_key, :server_cert
+
+    # syslog defaults
+    attr_accessor :facility, :severity, :hostname
+
+    # Other settings
+    attr_accessor :strip_color, :parse_fields, :prepend
+
+    # Exclude messages matching pattern
+    attr_accessor :exclude_pattern
+
+    # Files (can be globs)
+    attr_reader :files
+
+    # Exclude files matching pattern
+    attr_accessor :exclude_file_pattern
+
+    # How often should we check for new files?
+    attr_accessor :glob_check_interval
+
+    # Should we use eventmachine to tail?
+    attr_accessor :eventmachine_tail
+
+    def initialize(options = {})
+      @files = []
+      @glob_check_interval = 10
+      @eventmachine_tail = options.fetch(:eventmachine_tail, true)
+
+      unless logger = options[:logger]
+        logger = Logger.new(STDERR)
+        logger.level = Logger::ERROR
+      end
+
+      super('remote_syslog', :logger => logger, :pid_file => options[:pid_file])
+    end
+
+    def log_file=(file)
+      @log_file = File.expand_path(file)
+
+      level = self.logger.level
+      self.logger = Logger.new(file)
+      self.logger.level = level
+    end
+
+    def redirect_io!
+      if @log_file
+        STDOUT.reopen(@log_file, 'a')
+        STDERR.reopen(@log_file, 'a')
+        STDERR.sync = STDOUT.sync = true
+      end
+    end
+
+    def files=(files)
+      @files = [ @files, files ].flatten.compact.uniq
+    end
+
+    def watch_file(file)
+      if eventmachine_tail
+        RemoteSyslog::EventMachineReader.new(file,
+          :callback => @message_generator.method(:transmit),
+          :logger => logger)
+      else
+        RemoteSyslog::FileTailReader.new(file,
+          :callback => @message_generator.method(:transmit),
+          :logger => logger)
+      end
+    end
+
+    def run
+      EventMachine.run do
+        EM.error_handler do |e|
+          logger.error "Unhandled EventMachine Exception: #{e.class}: #{e.message}:\n\t#{e.backtrace.join("\n\t")}"
+        end
+
+        if @tls
+          max_message_size = 10240
+
+          connection = TlsEndpoint.new(@destination_host, @destination_port,
+            :client_cert_chain => @client_cert_chain,
+            :client_private_key => @client_private_key,
+            :server_cert => @server_cert,
+            :logger => logger)
+        elsif @tcp
+          max_message_size = 20480
+
+          connection = TcpEndpoint.new(@destination_host, @destination_port,
+            :logger => logger)
+        else
+          max_message_size = 1024
+          connection = UdpEndpoint.new(@destination_host, @destination_port,
+            :logger => logger)
+        end
+
+        @message_generator = RemoteSyslog::MessageGenerator.new(connection, 
+          :facility => @facility, :severity => @severity, 
+          :strip_color => @strip_color, :hostname => @hostname, 
+          :parse_fields => @parse_fields, :exclude_pattern => @exclude_pattern,
+          :prepend => @prepend, :max_message_size => max_message_size)
+
+        files.each do |file|
+          RemoteSyslog::GlobWatch.new(file, @glob_check_interval, 
+            @exclude_file_pattern, method(:watch_file))
+        end
+      end
+    end
+
+    def endpoint_mode
+      @endpoint_mode ||= if @tls
+        'TCP/TLS'
+      elsif @tcp
+        'TCP'
+      else
+        'UDP'
+      end
+    end
+
+    def before_stopping
+      EM.stop
+    end
+  end
+end

data/lib/remote_syslog/cli.rb

@@ -0,0 +1,292 @@
+require 'optparse'
+require 'yaml'
+require 'pathname'
+require 'servolux'
+
+require 'remote_syslog/agent'
+
+module RemoteSyslog
+  class Cli
+    FIELD_REGEXES = {
+      'syslog' => /^(\w+ +\d+ \S+) (\S+) ([^: ]+):? (.*)$/,
+      'rfc3339' => /^(\S+) (\S+) ([^: ]+):? (.*)$/
+    }
+
+    DEFAULT_PID_FILES = [
+      "/var/run/remote_syslog.pid",
+      "#{ENV['HOME']}/run/remote_syslog.pid",
+      "#{ENV['HOME']}/tmp/remote_syslog.pid",
+      "#{ENV['HOME']}/remote_syslog.pid",
+      "#{ENV['TMPDIR']}/remote_syslog.pid",
+      "/tmp/remote_syslog.pid"
+    ]
+
+    DEFAULT_CONFIG_FILE = '/etc/log_files.yml'
+
+    def self.process!(argv)
+      c = new(argv)
+      c.parse
+      c.run
+    end
+
+    attr_reader :program_name
+
+    def initialize(argv)
+      @argv = argv
+      @program_name = File.basename($0)
+
+      @strip_color = false
+      @exclude_pattern = nil
+
+      @daemonize_options = {
+        :ARGV         => %w(start),
+        :dir_mode     => :system,
+        :backtrace    => false,
+        :monitor      => false,
+      }
+
+      @agent = RemoteSyslog::Agent.new(:pid_file => default_pid_file)
+    end
+
+    def is_file_writable?(file)
+      directory = File.dirname(file)
+
+      (File.directory?(directory) && File.writable?(directory) && !File.exist?(file)) || File.writable?(file)
+    end
+
+    def default_pid_file
+      DEFAULT_PID_FILES.each do |file|
+        return file if is_file_writable?(file)
+      end
+    end
+
+    def parse
+      op = OptionParser.new do |opts|
+        opts.banner = "Usage: #{program_name} [OPTION]... <FILE>..."
+        opts.separator ''
+
+        opts.separator "Options:"
+
+        opts.on("-c", "--configfile PATH", "Path to config (/etc/log_files.yml)") do |v|
+          @configfile = v
+        end
+        opts.on("-d", "--dest-host HOSTNAME", "Destination syslog hostname or IP (logs.papertrailapp.com)") do |v|
+          @agent.destination_host = v
+        end
+        opts.on("-p", "--dest-port PORT", "Destination syslog port (514)") do |v|
+          @agent.destination_port = v
+        end
+        opts.on("-D", "--no-detach", "Don't daemonize and detach from the terminal") do
+          @no_detach = true
+        end
+        opts.on("-f", "--facility FACILITY", "Facility (user)") do |v|
+          @agent.facility = v
+        end
+        opts.on("--hostname HOST", "Local hostname to send from") do |v|
+          @agent.hostname = v
+        end
+        opts.on("-P", "--pid-dir DIRECTORY", "DEPRECATED: Directory to write .pid file in") do |v|
+          puts "Warning: --pid-dir is deprecated. Please use --pid-file FILENAME instead"
+          @pid_directory = v
+        end
+        opts.on("--pid-file FILENAME", "Location of the PID file (default #{@agent.pid_file})") do |v|
+          @agent.pid_file = v
+        end
+        opts.on("--parse-syslog", "Parse file as syslog-formatted file") do
+          @agent.parse_fields = FIELD_REGEXES['syslog']
+        end
+        opts.on("-s", "--severity SEVERITY", "Severity (notice)") do |v|
+          @agent.severity = v
+        end
+        opts.on("--strip-color", "Strip color codes") do
+          @agent.strip_color = true
+        end
+        opts.on("--tcp", "Connect via TCP (no TLS)") do
+          @agent.tcp = true
+        end
+        opts.on("--tls", "Connect via TCP with TLS") do
+          @agent.tls = true
+        end
+
+
+        opts.on("--new-file-check-interval INTERVAL", OptionParser::DecimalInteger,
+          "Time between checks for new files") do |v|
+          @agent.glob_check_interval = v
+        end
+
+        opts.separator ''
+        opts.separator 'Advanced options:'
+
+        opts.on("--[no-]eventmachine-tail", "Enable or disable using eventmachine-tail") do |v|
+          @agent.eventmachine_tail = v
+        end
+        opts.on("--debug-log FILE", "Log internal debug messages") do |v|
+          @agent.log_file = v
+        end
+
+        severities = Logger::Severity.constants + Logger::Severity.constants.map { |s| s.downcase }
+        opts.on("--debug-level LEVEL", severities, "Log internal debug messages at level") do |v|
+          @agent.logger.level = Logger::Severity.const_get(v.upcase)
+        end
+
+        opts.separator ""
+        opts.separator "Common options:"
+
+        opts.on("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+
+        opts.on("--version", "Show version") do
+          puts RemoteSyslog::VERSION
+          exit(0)
+        end
+
+        opts.separator ''
+        opts.separator "Example:"
+        opts.separator "    $ #{program_name} -c configs/logs.yml -p 12345 /var/log/mysqld.log"
+      end
+
+      op.parse!(@argv)
+
+      @files = @argv.dup.delete_if { |a| a.empty? }
+
+      if @configfile
+        if File.exist?(@configfile)
+          parse_config(@configfile)
+        else
+          error "The config file specified could not be found: #{@configfile}"
+        end
+      elsif File.exist?(DEFAULT_CONFIG_FILE)
+        parse_config(DEFAULT_CONFIG_FILE)
+      end
+
+      if @files.empty?
+        error "You must specify at least one file to watch"
+      end
+
+      @agent.destination_host ||= 'logs.papertrailapp.com'
+      @agent.destination_port ||= 514
+
+      # handle relative paths before Daemonize changes the wd to / and expand wildcards
+      @files = @files.flatten.map { |f| File.expand_path(f) }.uniq
+
+      @agent.files = @files
+
+      if @pid_directory
+        if @agent.pid_file
+          @agent.pid_file = File.expand_path("#{@pid_directory}/#{File.basename(@agent.pid_file)}")
+        else
+          @agent.pid_file = File.expand_path("#{@pid_directory}/remote_syslog.pid")
+        end
+      end
+
+      # We're dealing with an old-style pid_file
+      if @agent.pid_file && File.basename(@agent.pid_file) == @agent.pid_file
+        default_pid_dir = File.dirname(default_pid_file)
+
+        @agent.pid_file = File.join(default_pid_dir, @agent.pid_file)
+
+        if File.extname(@agent.pid_file) == ''
+          @agent.pid_file << '.pid'
+        end
+      end
+
+      @agent.pid_file ||= default_pid_file
+
+      if !@no_detach && !::Servolux.fork?
+        @no_detach = true
+
+        puts "Fork is not supported in this Ruby environment. Running in foreground."
+      end
+    rescue OptionParser::ParseError => e
+      error e.message, true
+    end
+
+    def parse_config(file)
+      config = YAML.load_file(file)
+
+      @files += Array(config['files'])
+
+      if config['destination'] && config['destination']['host']
+        @agent.destination_host ||= config['destination']['host']
+      end
+
+      if config['destination'] && config['destination']['port']
+        @agent.destination_port ||= config['destination']['port']
+      end
+
+      if config['hostname']
+        @agent.hostname = config['hostname']
+      end
+
+      @agent.server_cert        = config['ssl_server_cert']
+      @agent.client_cert_chain  = config['ssl_client_cert_chain']
+      @agent.client_private_key = config['ssl_client_private_key']
+
+      if config['parse_fields']
+        @agent.parse_fields = FIELD_REGEXES[config['parse_fields']] || Regexp.new(config['parse_fields'])
+      end
+
+      if config['exclude_patterns']
+        @agent.exclude_pattern = Regexp.new(config['exclude_patterns'].map { |r| "(#{r})" }.join('|'))
+      end
+
+      if config['exclude_files']
+        @agent.exclude_file_pattern = Regexp.new(config['exclude_files'].map { |r| "(#{r})" }.join('|'))
+      end
+
+      if config['new_file_check_interval']
+        @agent.glob_check_interval = config['new_file_check_interval']
+      end
+
+      if config['prepend']
+        @agent.prepend = config['prepend']
+      end
+    end
+
+    def run
+      Thread.abort_on_exception = true
+
+      if @agent.tls && !EventMachine.ssl?
+        error "TLS is not supported by eventmachine installed on this system.\nThe openssl-devel/openssl-dev package must be installed before installing eventmachine."
+      end
+
+      if @no_detach
+        puts "Watching #{@agent.files.length} files/globs. Sending to #{@agent.destination_host}:#{@agent.destination_port} (#{@agent.endpoint_mode})."
+        @agent.run
+      else
+        daemon = Servolux::Daemon.new(:server => @agent, :after_fork => method(:redirect_io))
+
+        if daemon.alive?
+          error "Already running at #{@agent.pid_file}. To run another instance, specify a different `--pid-file`.", true
+        end
+
+        puts "Watching #{@agent.files.length} files/globs. Sending to #{@agent.destination_host}:#{@agent.destination_port} (#{@agent.endpoint_mode})."
+        daemon.startup
+      end
+    rescue Servolux::Daemon::StartupError => e
+      case message = e.message[/^(Child raised error: )?(.*)$/, 2]
+      when /#<Errno::EACCES: (.*)>$/
+        error $1
+      else
+        error message
+      end
+    rescue Interrupt
+      exit(0)
+    end
+
+    def redirect_io
+      @agent.redirect_io!
+    end
+
+
+    def error(message, try_help = false)
+      puts "#{program_name}: #{message}"
+      if try_help
+        puts "Try `#{program_name} --help' for more information."
+      end
+      exit(1)
+    end
+  end
+end

data/lib/remote_syslog/eventmachine_reader.rb

@@ -0,0 +1,43 @@
+require 'eventmachine'
+require 'eventmachine-tail'
+require 'em-dns-resolver'
+
+# Force eventmachine-tail not to change the encoding
+# This will allow ruby 1.9 to deal with any file data
+old_verbose, $VERBOSE = $VERBOSE, nil
+EventMachine::FileTail::FORCE_ENCODING = false
+$VERBOSE = old_verbose
+
+module RemoteSyslog
+  class EventMachineReader < EventMachine::FileTail
+    def initialize(path, options = {}, &block)
+      @callback = options[:callback] || block
+      @buffer = BufferedTokenizer.new
+      @logger = options[:logger] || Logger.new(STDERR)
+
+      @tag = options[:program] || File.basename(path)
+
+      # Remove characters that can't be in a tag
+      @tag = @tag.gsub(%r{[: \]\[\\]+}, '-')
+
+      # Make sure the tag isn't too long
+      if @tag.length > 32
+        @tag = @tag[0..31]
+      end
+
+      @logger.debug "Watching #{path} with EventMachineReader"
+
+      super(path, -1)
+    end
+
+    def receive_data(data)
+      @buffer.extract(data).each do |line|
+        @callback.call(@tag, line)
+      end
+    end
+
+    def on_exception(exception)
+      @logger.error "Exception: #{exception.class}: #{exception.message}\n\t#{exception.backtrace.join("\n\t")}"
+    end
+  end
+end

data/lib/remote_syslog/file_tail_reader.rb

@@ -0,0 +1,40 @@
+require 'file/tail'
+
+module RemoteSyslog
+  class FileTailReader
+    def initialize(path, options = {}, &block)
+      @path = path
+      @callback = options[:callback] || block
+      @logger   = options[:logger] || Logger.new(STDERR)
+      @tag      = options[:program] || File.basename(path)
+
+      # Remove characters that can't be in a tag
+      @tag = @tag.gsub(%r{[: \]\[\\]+}, '-')
+
+      # Make sure the tag isn't too long
+      if @tag.length > 32
+        @tag = @tag[0..31]
+      end
+
+      @logger.debug "Watching #{path} with FileTailReader"
+
+      start
+    end
+
+    def start
+      @thread = Thread.new do
+        run
+      end
+    end
+
+    def run
+      File::Tail::Logfile.tail(@path) do |line|
+        EventMachine.schedule do
+          @callback.call(@tag, line)
+        end
+      end
+    rescue => e
+      @logger.error "Unhandled FileTailReader Exception: #{e.class}: #{e.message}:\n\t#{e.backtrace.join("\n\t")}"
+    end
+  end
+end

data/lib/remote_syslog/glob_watch.rb

@@ -0,0 +1,25 @@
+require 'eventmachine-tail'
+
+module RemoteSyslog
+  class GlobWatch < EventMachine::FileGlobWatch
+    def initialize(path, interval, exclude_files, callback)
+      @exclude_files = exclude_files
+      @callback = callback
+
+      super(path, interval)
+    end
+
+    def file_found(path)
+      # Check if we should exclude this file
+      if @exclude_files && @exclude_files =~ path
+        return
+      end
+
+      @callback.call(path)
+    end
+
+    def file_deleted(path)
+      # Nothing to do
+    end
+  end
+end