log2json 0.1.12 → 0.1.13

data/bin/redis2disk

@@ -8,7 +8,6 @@
 # another script at a slower rate when the queue is empty most of times.
 #
 #
-set -e
 
 # name of the queue in Redis. Must be a list in Redis.
 QUEUE=${QUEUE:-jsonlogs}
@@ -60,11 +59,16 @@ do
         log "New period begins---"
     fi
     s2=$(redis-cli llen $QUEUE) # current queue size
+    if [ $? -ne 0 -o ! "$s2" ]; then
+        sleep 1; continue
+    fi
     log "queue size=$s2"
 
     if [ "$s1" ]; then
         delta=$(( s2 - s1 ))
         rate=$(( delta / INTERVAL ))
+        #FIXME: ideally, this should be the time elapsed rather than the fixed INTERVAL.
+
         log "delta=$delta rate=$rate"
 
         # if within the period, we have crossed MAX_RATE CROSS_COUNT times
@@ -78,7 +82,7 @@ do
                 n_records=$((delta * count))
                 log "CROSS_COUNT($CROSS_COUNT) reached! Off loading $n_records log records to $batch_file ..."
 
-                redis-cli --eval <(echo "$LUA_LPOP_KEY_N") $QUEUE , $n_records > $batch_file &
+                redis-cli --eval <(echo "$LUA_LPOP_KEY_N") $QUEUE , $n_records > $batch_file
                 count=0
                 off_loaded=1
             fi
@@ -86,7 +90,7 @@ do
     fi
 
     if [ "$off_loaded" ]; then
-        s1=$(redis-cli llen $QUEUE)
+        s1=$(redis-cli llen $QUEUE) || s1=
         off_loaded=
     else
         s1=$s2

data/log2json.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'log2json'
-  s.version     = '0.1.12'
+  s.version     = '0.1.13'
   s.summary     = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
   s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
   s.authors     = ['Jack Kuan']

data/src/coreutils-8.13_tail.patch

@@ -1,25 +1,26 @@
-83,85d82
-< /* Size of the array that stores the -n option values. */
-< #define MAX_FILE_SKIP_LINES 256
-< 
-187,189d183
-< /* Stores the number of lines to skip from the start for each file. */
-< static int n_units_argv[MAX_FILE_SKIP_LINES];
-< 
-1056,1057d1049
-<       if (print_headers)
-<         printf ("==> %s <== [new_file]\n", pretty_name (f));
-1180,1181d1171
-<                   if (print_headers)
-<                     printf ("==> %s <== [truncated]\n", name);
-1297,1298d1286
-<       if (print_headers)
-<         printf ("==> %s <== [truncated]\n", name);
-1930d1917
-<   int n_units_argc = 0;
-1962d1948
-<             n_units_argv[n_units_argc++] = *n_units;
-2177c2163
-<     ok &= tail_file (&F[i], i < MAX_FILE_SKIP_LINES ? n_units_argv[i] : n_units);
+82a83,85
+> /* Size of the array that stores the -n option values. */
+> #define MAX_FILE_SKIP_LINES 256
+> 
+183a187,189
+> /* Stores the number of lines to skip from the start for each file. */
+> static int n_units_argv[MAX_FILE_SKIP_LINES];
+> 
+1049a1056,1057
+>       if (print_headers)
+>         printf ("==> %s <== [new_file]\n", pretty_name (f));
+1171a1180,1181
+>                   if (print_headers)
+>                     printf ("==> %s <== [truncated]\n", name);
+1286a1297,1298
+>       if (print_headers)
+>         printf ("==> %s <== [truncated]\n", name);
+1917a1930
+>   int n_units_argc = 0;
+1948a1962,1963
+>             if (n_units_argc < MAX_FILE_SKIP_LINES)
+>                 n_units_argv[n_units_argc++] = *n_units;
+2163c2178
+<     ok &= tail_file (&F[i], n_units);
 ---
->     ok &= tail_file (&F[i], n_units);
+>     ok &= tail_file (&F[i], i < MAX_FILE_SKIP_LINES ? n_units_argv[i] : n_units);

data/src/tail.c

@@ -1959,7 +1959,8 @@ parse_options (int argc, char **argv,
                         ? _("invalid number of lines")
                         : _("invalid number of bytes")));
               }
-            n_units_argv[n_units_argc++] = *n_units;
+            if (n_units_argc < MAX_FILE_SKIP_LINES)
+                n_units_argv[n_units_argc++] = *n_units;
           }
           break;
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: log2json
 version: !ruby/object:Gem::Version
-  version: 0.1.12
+  version: 0.1.13
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-21 00:00:00.000000000 Z
+date: 2013-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jls-grok