log2json 0.1.11 → 0.1.12

data/bin/redis2disk

@@ -0,0 +1,96 @@
+#!/bin/bash
+#
+# Check every INTERVAL seconds the size of the log2json log queue in Redis,
+# and reduce the queue size if its average rate of change over the INTERVAL
+# is greater than MAX_RATE for CROSS_COUNT times within EVAL_PERIOD seconds.
+#
+# The logs will be saved on disk and can be pushed back to the queue by
+# another script at a slower rate when the queue is empty most of times.
+#
+#
+set -e
+
+# name of the queue in Redis. Must be a list in Redis.
+QUEUE=${QUEUE:-jsonlogs}
+
+# average number of log records per second over INTERVAL seconds
+MAX_RATE=${MAX_RATE:-25000}
+
+# how many times the average rate of change greater than MAX_RATE should happen
+# with in EVAL_PERIOD seconds before we reduce the queue size.
+CROSS_COUNT=${CROSS_COUNT:-2}
+
+# the length of time(in seconds) within which a "cross-over"
+# is considered valid and so counts toward CROSS_COUNT.
+# It should be a multiple of INTERVAL.
+EVAL_PERIOD=${EVAL_PERIOD:-15}
+
+# Samples the queue size roughly every INTERVAL seconds.
+# The smaller the more accurate and less chance to miss sudden burst of traffic.
+INTERVAL=${INTERVAL:-5}
+
+# LUA script for Redis to lpop N log records off the queue.
+LUA_LPOP_KEY_N='
+    local i = tonumber(ARGV[1])
+    local res = {}
+    local length = redis.call("llen",KEYS[1])
+    if length < i then i = length end
+    while (i > 0) do
+    local item = redis.call("lpop", KEYS[1])
+    if (not item) then
+      break
+    end
+    table.insert(res, item)
+    i = i-1
+    end
+    return res
+'
+
+# folder to store the log records off loaded from the queue
+BATCH_DIR=${BATCH_DIR:-/mnt/redis}
+
+log() { echo "$(date): $*"; }
+
+while true
+do
+    if [ $(( $(date +%s) - period_start )) -ge "$EVAL_PERIOD" ]; then
+        period_start=$(date +%s)
+        count=0 # how many times have we crossed MAX_RATE within the period so far.
+                # reset to 0 at the start of a new period or when it reaches CROSS_COUNT.
+        log "New period begins---"
+    fi
+    s2=$(redis-cli llen $QUEUE) # current queue size
+    log "queue size=$s2"
+
+    if [ "$s1" ]; then
+        delta=$(( s2 - s1 ))
+        rate=$(( delta / INTERVAL ))
+        log "delta=$delta rate=$rate"
+
+        # if within the period, we have crossed MAX_RATE CROSS_COUNT times
+        # then move the records from the queue to a batch file.
+        if [ "$rate" -gt "$MAX_RATE" ]; then
+            count=$((count + 1))
+            log "MAX_RATE($MAX_RATE/s) crossed! count=$count"
+
+            if [ "$count" -ge "$CROSS_COUNT" ]; then
+                batch_file=$BATCH_DIR/${QUEUE}_$(date +%Y-%m-%dT%T.%N%z)
+                n_records=$((delta * count))
+                log "CROSS_COUNT($CROSS_COUNT) reached! Off loading $n_records log records to $batch_file ..."
+
+                redis-cli --eval <(echo "$LUA_LPOP_KEY_N") $QUEUE , $n_records > $batch_file &
+                count=0
+                off_loaded=1
+            fi
+        fi
+    fi
+
+    if [ "$off_loaded" ]; then
+        s1=$(redis-cli llen $QUEUE)
+        off_loaded=
+    else
+        s1=$s2
+    fi
+    sleep "$INTERVAL"
+done
+

data/bin/tail-log.sh

@@ -4,9 +4,17 @@ set -e
 
 # Find out the absolute path to the tail utility.
 # This is a patched version of the tail utility in GNU coreutils-8.13 compiled for Ubuntu 12.04 LTS.
-# The difference is that if header will be shown(ie, with -v or when multiple files are specified),
-# it will also print "==> file.name <== [event]" to stdout whenever a file truncation or a new file is
-# detected. [event] will be one of "[new_file]" or "[truncated]".
+# With the following differences:
+#
+#   - if header will be shown(ie, with -v or when multiple files are specified),
+#     it will also print "==> file.name <== [event]" to stdout whenever a file truncation or a new file is
+#     detected. [event] will be one of "[new_file]" or "[truncated]".
+#
+#   - It allows the use of multiple -n options. Each -n will apply to the files specified on the
+#     command line in order.(ie, first -n N corresponds to the first file, etc.)
+#     If there are more files listed than the number of -n options, then the last -n applies to the
+#     rest of the files.
+#
 TAIL=$(
 ruby -- - <<'EOF'
   require 'log2json'
@@ -37,12 +45,17 @@ build_tail_args() {
   for fpath in "$@"
   do
     sincedb_path=$SINCEDB_DIR/$fpath.since
+    nlines=$(wc -l "$fpath" | cut -d' ' -f1)
+    nlines=${nlines:-0}
+
     if [ -r "$sincedb_path" ]; then
       read line < "$sincedb_path"
       t=($line)
       # if inode number is unchanged and the current file size is not smaller
       # then we start tailing from 1 + the line number recorded in the sincedb.
-      if [[ ${t[0]} == $(stat -c "%i" "$fpath") && ${t[1]} -le $(stat -c "%s" "$fpath") ]]; then
+      if [[ ${t[0]} == $(stat -c "%i" "$fpath") &&
+            ${t[1]} -le $(stat -c "%s" "$fpath") &&
+            ${t[2]} -le "$nlines" ]]; then
         TAIL_ARGS[$((i++))]="-n+$((t[2] + 1))"
         # tail -n+N means start tailing from the N-th line of the file
         # and we're even allowed to specify different -n+N for different files!
@@ -54,7 +67,7 @@ build_tail_args() {
     # at this point, no last position was recorded in the SINCEDB for fpath,
     # in this case we'd tail from the end of file.
 
-    TAIL_ARGS[$((i++))]="-n+$(($(wc -l "$fpath" | cut -d' ' -f1) + 1))"
+    TAIL_ARGS[$((i++))]="-n+$(( nlines + 1 ))"
     # Note: we can't just ask tail to seek to the end here(ie, with -n0) since
     #       then we'd lose track of the line count.
     # Note: if fpath doesn't exist yet, then the above evaluates to "-n+1", which

data/lib/log2json/railslogger.rb

@@ -15,6 +15,16 @@ require 'logger'
 
 module Log2Json
 
+  LEVELS = {
+    :debug => Logger::DEBUG,
+    :info => Logger::INFO,
+    :warn => Logger::WARN,
+    :error => Logger::ERROR,
+    :fatal => Logger::FATAL,
+    :unknown => 5
+  }
+  LEVELS.default = Logger::INFO
+
   def self.log_formatter
     proc do |severity, datetime, progname, msg|
       "#{datetime.strftime('%Y-%m-%dT%H:%M:%S%z')}: [#{severity}] #{$$} #{msg.gsub(/\n/, '#012')}\n"
@@ -37,6 +47,7 @@ module Log2Json
       config.active_record.colorize_logging = false 
     end
     logger = ::Logger.new(path)
+    logger.level =  LEVELS[config.log_level]
     logger.formatter = ::Log2Json::log_formatter
     if defined?(ActiveSupport::TaggedLogging)
       ActiveSupport::TaggedLogging.new(logger)
@@ -49,6 +60,7 @@ module Log2Json
   #
   def self.create_custom_unicorn_logger(config)
     logger = ::Logger.new(config.set[:stderr_path])
+    logger.level = Logger::INFO
     logger.formatter = ::Log2Json::log_formatter
     logger
   end

data/log2json-loggers.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'log2json-loggers'
-  s.version     = '0.1.9'
+  s.version     = '0.1.11'
   s.summary     = "Custom loggers for Rails and Unicorn that use log2json's single-line log format."
   s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
   s.authors     = ['Jack Kuan']

data/log2json.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'log2json'
-  s.version     = '0.1.11'
+  s.version     = '0.1.12'
   s.summary     = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
   s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
   s.authors     = ['Jack Kuan']

data/src/coreutils-8.13_tail.patch

@@ -1,9 +1,25 @@
-1049a1050,1051
->       if (print_headers)
->         printf ("==> %s <== [new_file]\n", pretty_name (f));
-1171a1174,1175
->                   if (print_headers)
->                     printf ("==> %s <== [truncated]\n", name);
-1286a1291,1292
->       if (print_headers)
->         printf ("==> %s <== [truncated]\n", name);
+83,85d82
+< /* Size of the array that stores the -n option values. */
+< #define MAX_FILE_SKIP_LINES 256
+< 
+187,189d183
+< /* Stores the number of lines to skip from the start for each file. */
+< static int n_units_argv[MAX_FILE_SKIP_LINES];
+< 
+1056,1057d1049
+<       if (print_headers)
+<         printf ("==> %s <== [new_file]\n", pretty_name (f));
+1180,1181d1171
+<                   if (print_headers)
+<                     printf ("==> %s <== [truncated]\n", name);
+1297,1298d1286
+<       if (print_headers)
+<         printf ("==> %s <== [truncated]\n", name);
+1930d1917
+<   int n_units_argc = 0;
+1962d1948
+<             n_units_argv[n_units_argc++] = *n_units;
+2177c2163
+<     ok &= tail_file (&F[i], i < MAX_FILE_SKIP_LINES ? n_units_argv[i] : n_units);
+---
+>     ok &= tail_file (&F[i], n_units);

data/src/tail.c

@@ -80,6 +80,9 @@
 /* FIXME: make Follow_name the default?  */
 #define DEFAULT_FOLLOW_MODE Follow_descriptor
 
+/* Size of the array that stores the -n option values. */
+#define MAX_FILE_SKIP_LINES 256
+
 enum Follow_mode
 {
   /* Follow the name of each file: if the file is renamed, try to reopen
@@ -181,6 +184,9 @@ static bool forever;
 /* If true, count from start of file instead of end.  */
 static bool from_start;
 
+/* Stores the number of lines to skip from the start for each file. */
+static int n_units_argv[MAX_FILE_SKIP_LINES];
+
 /* If true, print filename headers.  */
 static bool print_headers;
 
@@ -1921,6 +1927,7 @@ parse_options (int argc, char **argv,
                double *sleep_interval)
 {
   int c;
+  int n_units_argc = 0;
 
   while ((c = getopt_long (argc, argv, "c:n:fFqs:v0123456789",
                            long_options, NULL))
@@ -1952,6 +1959,7 @@ parse_options (int argc, char **argv,
                         ? _("invalid number of lines")
                         : _("invalid number of bytes")));
               }
+            n_units_argv[n_units_argc++] = *n_units;
           }
           break;
 
@@ -2166,7 +2174,7 @@ main (int argc, char **argv)
     xfreopen (NULL, "wb", stdout);
 
   for (i = 0; i < n_files; i++)
-    ok &= tail_file (&F[i], n_units);
+    ok &= tail_file (&F[i], i < MAX_FILE_SKIP_LINES ? n_units_argv[i] : n_units);
 
   if (forever && ignore_fifo_and_pipe (F, n_files))
     {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: log2json
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.12
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-15 00:00:00.000000000 Z
+date: 2013-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jls-grok
@@ -114,6 +114,7 @@ files:
 - README
 - bin/lines2redis
 - bin/nginxlog2json
+- bin/redis2disk
 - bin/redis2es
 - bin/syslog2json
 - bin/tail