remote_syslog 1.1.1 → 1.2.0

data/README.md

@@ -90,6 +90,41 @@ Only 1 destination server is supported; the command-line argument wins.
       port: 12345
 
 
+### Optional: Parse fields from messages written by syslogd
+
+This is not needed for most configurations.
+
+In cases where logs from multiple programs are in the same file (for example, 
+``/var/log/messages``), the log line may include text that is not part of the 
+log message, like a timestamp, hostname, or program name. remote_syslog can 
+parse the program, hostname, and/or message text so that the message has 
+accurate metadata.
+
+To do that, add an optional top-level configuration option `parse_fields` 
+with the name of a predefined regex (by remote_syslog) or a regex string. To 
+use the predefined regex for standard syslog messages, provide:
+
+    parse_fields: syslog
+
+The `syslog` regex is `(\w+ \d+ \S+) (\S+) ([^:]+): (.*)`. It parses this:
+
+    Jul 18 08:25:08 hostname programname[1234]: The log message
+
+Or provide your own regex that includes these 4 backreferences, in order: 
+timestamp, system name, program name, message. Match and return empty 
+strings for any empty positions where the log value should be ignored.
+For example, in the log:
+
+    something-meaningless The log message
+
+You could ignore the first word, returning 3 empty values then the log 
+message with:
+
+    parse_fields: "something-meaningless ()()()(.*)"
+
+Per-file parsing is not supported. Run multiple instances.
+
+
 ## Reporting bugs
 
 1. See whether the issue has already been reported: <https://github.com/papertrail/remote_syslog/issues/>

data/bin/remote_syslog

@@ -1,121 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'optparse'
-require 'yaml'
-require 'pathname'
-require 'socket'
-
-require 'daemons'
-require 'eventmachine'
-require 'eventmachine-tail'
-
 require 'remote_syslog'
+require 'remote_syslog/cli'
 
-def remote_syslog_daemon(args)
-  options = {
-    :configfile => '/etc/log_files.yml',
-    :dest_host  => 'logs.papertrailapp.com',
-    :dest_port  => 514
-  }
-  daemonize_options = {
-    :app_name    => File.basename($0) || "remote_syslog",
-    :ARGV        => ['start'],
-    :dir_mode    => :system,
-    :multiple    => false,
-    :ontop       => false,
-    :mode        => :exec,
-    :backtrace   => false,
-    :monitor     => false,
-    :strip_color => false
-  }
-
-  op = OptionParser.new do |opts|
-    opts.banner = "Usage:   remote_syslog [options] <path to add'l log 1> .. <path to add'l log n>"
-    opts.separator ''
-    opts.separator "Example: remote_syslog -c configs/logs.yml -p 12345 /var/log/mysqld.log"
-    opts.separator ''
-    opts.separator "Options:"
-    
-    opts.on("-c", "--configfile PATH", "Path to config (/etc/log_files.yml)") do |v|
-      options[:configfile] = File.expand_path(v)
-    end
-    opts.on("-d", "--dest-host HOSTNAME", "Destination syslog hostname or IP (logs.papertrailapp.com)") do |v|
-      options[:dest_host] = v
-    end
-    opts.on("-D", "--no-detach", "Don't daemonize and detach from the terminal") do
-      daemonize_options[:ontop] = true
-      daemonize_options[:ARGV]  = ['run']
-      # write PID file in . because /var/run is sometimes only writable by root
-      daemonize_options[:dir_mode] = :script
-      daemonize_options[:dir] = '.'
-    end
-    opts.on("-f", "--facility FACILITY", "Facility (user)") do |v|
-      options[:facility] = v.upcase
-    end
-    opts.on("-p", "--dest-port PORT", "Destination syslog port (514)") do |v|
-      options[:dest_port] = v
-    end
-    opts.on("-P", "--pid-dir DIRECTORY", "Directory to write .pid file in (/var/run/)") do |v|
-      daemonize_options[:dir_mode] = :script
-      daemonize_options[:dir] = v
-    end
-    opts.on("-s", "--severity SEVERITY", "Severity (notice)") do |v|
-      options[:severity] = v.upcase
-    end
-    opts.on("--strip-color", "Strip color codes") do
-      options[:strip_color] = true
-    end
-    opts.on_tail("-h", "--help", "Show this message") do
-      puts opts
-      exit
-    end
-  end
-  
-  op.parse!
-  
-  files = ARGV
-  if File.exist?(options[:configfile])
-    config = open(options[:configfile]) do |f|
-      YAML.load(f)
-    end
-    
-    files += config['files']
-    if config['destination']
-      options[:dest_host] = config['destination']['host'] if config['destination']['host']
-      options[:dest_port] = config['destination']['port'] if config['destination']['port']
-    end
-    if config['hostname']
-      options[:hostname] = config['hostname']
-    end
-  elsif files.empty?
-    puts "No filenames provided and #{options[:configfile]} not found."
-    puts ''
-    puts op
-    exit
-  end
-
-  # handle relative paths before Daemonize changes the wd to / and expand wildcards
-  files = files.map { |f| Dir.glob(f) }.flatten.map { |f| File.expand_path(f) }.uniq
-
-  Daemons.run_proc(daemonize_options[:app_name], daemonize_options) do
-    EventMachine.run do
-      socket = EventMachine.open_datagram_socket('0.0.0.0', 0)
-      
-      files.each do |path|
-        begin
-          EventMachine::file_tail(File.expand_path(path), RemoteSyslog::Reader, 
-            options[:dest_host], options[:dest_port],
-            :socket => socket, :facility => options[:facility],
-            :severity => options[:severity], :strip_color => options[:strip_color],
-            :hostname => options[:hostname])
-                                  
-        rescue Errno::ENOENT => e
-          puts "#{File.expand_path(path)} not found, continuing. (#{e.message})"
-        end
-      end
-    end
-  end
-  
-end
-
-remote_syslog_daemon(ARGV)
+RemoteSyslog::Cli.process!(ARGV)

data/examples/log_files.yml.example.syslog

@@ -0,0 +1,5 @@
+files: [/var/log/messages]
+parse_fields: syslog # predefined regex name or double-quoted regex
+destination:
+  host: logs.papertrailapp.com
+  port: 12345   # optional, defaults to 514

data/lib/remote_syslog.rb

@@ -1,6 +1,5 @@
 module RemoteSyslog
- VERSION = "1.1.1"
+ VERSION = "1.2.0"
 end
 
-require 'remote_syslog/levels'
 require 'remote_syslog/reader'

data/lib/remote_syslog/cli.rb

@@ -0,0 +1,168 @@
+require 'optparse'
+require 'yaml'
+require 'pathname'
+require 'daemons'
+
+
+module RemoteSyslog
+  class Cli
+    FIELD_REGEXES = {
+      'syslog' => /^(\w+ \d+ \S+) (\w+) ([^: ]+):? (.*)$/,
+    }
+
+    def self.process!(argv)
+      c = new(argv)
+      c.parse
+      c.run
+    end
+
+    def initialize(argv)
+      @argv = argv
+
+      @app_name = File.basename($0) || 'remote_syslog'
+
+      @configfile  = '/etc/log_files.yml'
+      @strip_color = false
+
+      @daemonize_options = {
+        :dir_mode     => :system,
+        :backtrace    => false,
+        :monitor      => false,
+      }
+    end
+
+    def pid_file=(v)
+      m = v.match(%r{^(.+/)?([^/]+?)(\.pid)?$})
+      if m[1]
+        @daemonize_options[:dir_mode] = :normal
+        @daemonize_options[:dir] = m[1]
+      end
+
+      @app_name = m[2]
+    end
+
+    def parse
+      op = OptionParser.new do |opts|
+        opts.banner = "Usage:   remote_syslog [options] <path to add'l log 1> .. <path to add'l log n>"
+        opts.separator ''
+        opts.separator "Example: remote_syslog -c configs/logs.yml -p 12345 /var/log/mysqld.log"
+        opts.separator ''
+        opts.separator "Options:"
+
+        opts.on("-c", "--configfile PATH", "Path to config (/etc/log_files.yml)") do |v|
+          @configfile = File.expand_path(v)
+        end
+        opts.on("-d", "--dest-host HOSTNAME", "Destination syslog hostname or IP (logs.papertrailapp.com)") do |v|
+          @dest_host = v
+        end
+        opts.on("-p", "--dest-port PORT", "Destination syslog port (514)") do |v|
+          @dest_port = v
+        end
+        opts.on("-D", "--no-detach", "Don't daemonize and detach from the terminal") do
+          @no_detach = true
+        end
+        opts.on("-f", "--facility FACILITY", "Facility (user)") do |v|
+          @facility = v
+        end
+        opts.on("--hostname HOST", "Local hostname to send from") do |v|
+          @hostname = v
+        end
+        opts.on("-P", "--pid-dir DIRECTORY", "Directory to write .pid file in (/var/run/)") do |v|
+          @daemonize_options[:dir_mode] = :normal
+          @daemonize_options[:dir] = v
+        end
+        opts.on("--pid-file FILENAME", "PID filename (<program name>.pid)") do |v|
+          self.pid_file = v
+        end
+        opts.on("--parse-syslog", "Parse file as syslog-formatted file") do
+          @parse_fields = FIELD_REGEXES['syslog']
+        end
+        opts.on("-s", "--severity SEVERITY", "Severity (notice)") do |v|
+          @severity = v
+        end
+        opts.on("--strip-color", "Strip color codes") do
+          @strip_color = true
+        end
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+      end
+
+      op.parse!(@argv)
+
+      @files = @argv
+
+      parse_config
+
+      @dest_host ||= 'logs.papertrailapp.com'
+      @dest_port ||= 514
+
+      if @files.empty?
+        puts "No filenames provided and #{@configfile} not found."
+        puts ''
+        puts op
+        exit
+      end
+
+      # handle relative paths before Daemonize changes the wd to / and expand wildcards
+      @files = @files.map { |f| Dir.glob(f) }.flatten.map { |f| File.expand_path(f) }.uniq
+
+    end
+
+    def parse_config
+      if File.exist?(@configfile)
+        config = open(@configfile) do |f|
+          YAML.load(f)
+        end
+
+        @files += Array(config['files'])
+
+        if config['destination'] && config['destination']['host']
+          @dest_host ||= config['destination']['host']
+        end
+
+        if config['destination'] && config['destination']['port']
+          @dest_port ||= config['destination']['port']
+        end
+
+        if config['hostname']
+          @hostname = config['hostname']
+        end
+
+        if config['parse_fields']
+          @parse_fields = FIELD_REGEXES[config['parse_fields']] || Regexp.new(config['parse_fields'])
+        end
+      end
+    end
+
+    def run
+      if @no_detach
+        start
+      else
+        Daemons.run_proc(@app_name, @daemonize_options) do
+          start
+        end
+      end
+    end
+
+    def start
+      EventMachine.run do
+        socket = EventMachine.open_datagram_socket('0.0.0.0', 0)
+
+        @files.each do |path|
+          begin
+            EventMachine::file_tail(path, RemoteSyslog::Reader,
+              @dest_host, @dest_port,
+              :socket => socket, :facility => @facility,
+              :severity => @severity, :strip_color => @strip_color,
+              :hostname => @hostname, :parse_fields => @parse_fields)
+
+          rescue Errno::ENOENT => e
+            puts "#{path} not found, continuing. (#{e.message})"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/remote_syslog/reader.rb

@@ -1,3 +1,4 @@
+require 'socket'
 require 'eventmachine'
 require 'eventmachine-tail'
 require 'em-dns-resolver'
@@ -13,6 +14,7 @@ module RemoteSyslog
       @destination_address = destination_address
       @destination_port    = destination_port.to_i
 
+      @parse_fields = options[:parse_fields]
       @strip_color = options[:strip_color]
 
       @socket = options[:socket] || EventMachine.open_datagram_socket('0.0.0.0', 0)
@@ -31,6 +33,11 @@ module RemoteSyslog
       @packet.severity = options[:severity] || 'notice'
       @packet.tag      = options[:program]  || File.basename(path) || File.basename($0)
 
+      # Make sure the tag isn't too long
+      if @packet.tag.length > 32
+        @packet.tag = @packet.tag[0..31]
+      end
+
       # Try to resolve the destination address
       resolve_destination_address
 
@@ -63,6 +70,14 @@ module RemoteSyslog
       packet = @packet.dup
       packet.content = message
 
+      if @parse_fields
+        if message =~ @parse_fields
+          packet.hostname = $2 if $2 && $2 != ''
+          packet.tag      = $3 if $3 && $2 != ''
+          packet.content  = $4 if $4 && $4 != ''
+        end
+      end
+
       @socket.send_datagram(packet.assemble, destination_address, @destination_port)
     end
   end

data/remote_syslog.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |s|
   ## If your rubyforge_project name is different, then edit it and comment out
   ## the sub! line in the Rakefile
   s.name              = 'remote_syslog'
-  s.version           = '1.1.1'
-  s.date              = '2011-07-18'
+  s.version           = '1.2.0'
+  s.date              = '2011-07-25'
   s.rubyforge_project = 'remote_syslog'
 
   ## Make sure your summary is short. The description may be as long
@@ -61,10 +61,11 @@ Gem::Specification.new do |s|
     Rakefile
     bin/remote_syslog
     examples/log_files.yml.example
+    examples/log_files.yml.example.syslog
     examples/remote_syslog.init.d
     examples/remote_syslog.supervisor.conf
     lib/remote_syslog.rb
-    lib/remote_syslog/levels.rb
+    lib/remote_syslog/cli.rb
     lib/remote_syslog/reader.rb
     remote_syslog.gemspec
   ]

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: remote_syslog
 version: !ruby/object:Gem::Version 
-  hash: 17
-  prerelease: 
+  prerelease: false
   segments: 
   - 1
-  - 1
-  - 1
-  version: 1.1.1
+  - 2
+  - 0
+  version: 1.2.0
 platform: ruby
 authors: 
 - Troy Davis
@@ -16,18 +15,16 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-18 00:00:00 -07:00
+date: 2011-07-25 00:00:00 -07:00
 default_executable: remote_syslog
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: daemons
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -37,11 +34,9 @@ dependencies:
   name: eventmachine
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 59
         segments: 
         - 0
         - 12
@@ -53,11 +48,9 @@ dependencies:
   name: eventmachine-tail
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -67,11 +60,9 @@ dependencies:
   name: syslog_protocol
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -81,11 +72,9 @@ dependencies:
   name: em-resolv-replace
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -109,10 +98,11 @@ files:
 - Rakefile
 - bin/remote_syslog
 - examples/log_files.yml.example
+- examples/log_files.yml.example.syslog
 - examples/remote_syslog.init.d
 - examples/remote_syslog.supervisor.conf
 - lib/remote_syslog.rb
-- lib/remote_syslog/levels.rb
+- lib/remote_syslog/cli.rb
 - lib/remote_syslog/reader.rb
 - remote_syslog.gemspec
 has_rdoc: true
@@ -125,27 +115,23 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: remote_syslog
-rubygems_version: 1.4.2
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 2
 summary: Monitor plain text log file(s) for new entries and send to remote syslog collector

data/lib/remote_syslog/levels.rb

@@ -1,37 +0,0 @@
-module RemoteSyslog
-  class Levels
-    SEVERITIES = {
-      :emerg => 0,
-      :alert => 1,
-      :crit  => 2,
-      :err   => 3,
-      :warning => 4,
-      :notice  => 5,
-      :info  => 6,
-      :debug => 7
-    }.freeze
-    
-    FACILITIES = {
-      :kern => (0<<3),
-      :user => (1<<3),
-      :mail => (2<<3),
-      :daemon => (3<<3),
-      :auth => (4<<3),
-      :syslog => (5<<3),
-      :lpr  => (6<<3),
-      :news => (7<<3),
-      :uucp => (8<<3),
-      :cron => (9<<3),
-      :authpriv => (10<<3),
-      :ftp  => (11<<3),
-      :local0 => (16<<3),
-      :local1 => (17<<3),
-      :local2 => (18<<3),
-      :local3 => (19<<3),
-      :local4 => (20<<3),
-      :local5 => (21<<3),
-      :local6 => (22<<3),
-      :local7 => (23<<3)
-    }.freeze
-  end
-end