log2json 0.1.11 → 0.1.12

Sign up to get free protection for your applications and to get access to all the features.
data/bin/redis2disk ADDED
@@ -0,0 +1,96 @@
1
+ #!/bin/bash
2
+ #
3
+ # Check every INTERVAL seconds the size of the log2json log queue in Redis,
4
+ # and reduce the queue size if its average rate of change over the INTERVAL
5
+ # is greater than MAX_RATE for CROSS_COUNT times within EVAL_PERIOD seconds.
6
+ #
7
+ # The logs will be saved on disk and can be pushed back to the queue by
8
+ # another script at a slower rate when the queue is empty most of times.
9
+ #
10
+ #
11
+ set -e
12
+
13
+ # name of the queue in Redis. Must be a list in Redis.
14
+ QUEUE=${QUEUE:-jsonlogs}
15
+
16
+ # average number of log records per second over INTERVAL seconds
17
+ MAX_RATE=${MAX_RATE:-25000}
18
+
19
+ # how many times the average rate of change greater than MAX_RATE should happen
20
+ # with in EVAL_PERIOD seconds before we reduce the queue size.
21
+ CROSS_COUNT=${CROSS_COUNT:-2}
22
+
23
+ # the length of time(in seconds) within which a "cross-over"
24
+ # is considered valid and so counts toward CROSS_COUNT.
25
+ # It should be a multiple of INTERVAL.
26
+ EVAL_PERIOD=${EVAL_PERIOD:-15}
27
+
28
+ # Samples the queue size roughly every INTERVAL seconds.
29
+ # The smaller the more accurate and less chance to miss sudden burst of traffic.
30
+ INTERVAL=${INTERVAL:-5}
31
+
32
+ # LUA script for Redis to lpop N log records off the queue.
33
+ LUA_LPOP_KEY_N='
34
+ local i = tonumber(ARGV[1])
35
+ local res = {}
36
+ local length = redis.call("llen",KEYS[1])
37
+ if length < i then i = length end
38
+ while (i > 0) do
39
+ local item = redis.call("lpop", KEYS[1])
40
+ if (not item) then
41
+ break
42
+ end
43
+ table.insert(res, item)
44
+ i = i-1
45
+ end
46
+ return res
47
+ '
48
+
49
+ # folder to store the log records off loaded from the queue
50
+ BATCH_DIR=${BATCH_DIR:-/mnt/redis}
51
+
52
+ log() { echo "$(date): $*"; }
53
+
54
+ while true
55
+ do
56
+ if [ $(( $(date +%s) - period_start )) -ge "$EVAL_PERIOD" ]; then
57
+ period_start=$(date +%s)
58
+ count=0 # how many times have we crossed MAX_RATE within the period so far.
59
+ # reset to 0 at the start of a new period or when it reaches CROSS_COUNT.
60
+ log "New period begins---"
61
+ fi
62
+ s2=$(redis-cli llen $QUEUE) # current queue size
63
+ log "queue size=$s2"
64
+
65
+ if [ "$s1" ]; then
66
+ delta=$(( s2 - s1 ))
67
+ rate=$(( delta / INTERVAL ))
68
+ log "delta=$delta rate=$rate"
69
+
70
+ # if within the period, we have crossed MAX_RATE CROSS_COUNT times
71
+ # then move the records from the queue to a batch file.
72
+ if [ "$rate" -gt "$MAX_RATE" ]; then
73
+ count=$((count + 1))
74
+ log "MAX_RATE($MAX_RATE/s) crossed! count=$count"
75
+
76
+ if [ "$count" -ge "$CROSS_COUNT" ]; then
77
+ batch_file=$BATCH_DIR/${QUEUE}_$(date +%Y-%m-%dT%T.%N%z)
78
+ n_records=$((delta * count))
79
+ log "CROSS_COUNT($CROSS_COUNT) reached! Off loading $n_records log records to $batch_file ..."
80
+
81
+ redis-cli --eval <(echo "$LUA_LPOP_KEY_N") $QUEUE , $n_records > $batch_file &
82
+ count=0
83
+ off_loaded=1
84
+ fi
85
+ fi
86
+ fi
87
+
88
+ if [ "$off_loaded" ]; then
89
+ s1=$(redis-cli llen $QUEUE)
90
+ off_loaded=
91
+ else
92
+ s1=$s2
93
+ fi
94
+ sleep "$INTERVAL"
95
+ done
96
+
data/bin/tail CHANGED
Binary file
data/bin/tail-log.sh CHANGED
@@ -4,9 +4,17 @@ set -e
4
4
 
5
5
  # Find out the absolute path to the tail utility.
6
6
  # This is a patched version of the tail utility in GNU coreutils-8.13 compiled for Ubuntu 12.04 LTS.
7
- # The difference is that if header will be shown(ie, with -v or when multiple files are specified),
8
- # it will also print "==> file.name <== [event]" to stdout whenever a file truncation or a new file is
9
- # detected. [event] will be one of "[new_file]" or "[truncated]".
7
+ # With the following differences:
8
+ #
9
+ # - if header will be shown(ie, with -v or when multiple files are specified),
10
+ # it will also print "==> file.name <== [event]" to stdout whenever a file truncation or a new file is
11
+ # detected. [event] will be one of "[new_file]" or "[truncated]".
12
+ #
13
+ # - It allows the use of multiple -n options. Each -n will apply to the files specified on the
14
+ # command line in order.(ie, first -n N corresponds to the first file, etc.)
15
+ # If there are more files listed than the number of -n options, then the last -n applies to the
16
+ # rest of the files.
17
+ #
10
18
  TAIL=$(
11
19
  ruby -- - <<'EOF'
12
20
  require 'log2json'
@@ -37,12 +45,17 @@ build_tail_args() {
37
45
  for fpath in "$@"
38
46
  do
39
47
  sincedb_path=$SINCEDB_DIR/$fpath.since
48
+ nlines=$(wc -l "$fpath" | cut -d' ' -f1)
49
+ nlines=${nlines:-0}
50
+
40
51
  if [ -r "$sincedb_path" ]; then
41
52
  read line < "$sincedb_path"
42
53
  t=($line)
43
54
  # if inode number is unchanged and the current file size is not smaller
44
55
  # then we start tailing from 1 + the line number recorded in the sincedb.
45
- if [[ ${t[0]} == $(stat -c "%i" "$fpath") && ${t[1]} -le $(stat -c "%s" "$fpath") ]]; then
56
+ if [[ ${t[0]} == $(stat -c "%i" "$fpath") &&
57
+ ${t[1]} -le $(stat -c "%s" "$fpath") &&
58
+ ${t[2]} -le "$nlines" ]]; then
46
59
  TAIL_ARGS[$((i++))]="-n+$((t[2] + 1))"
47
60
  # tail -n+N means start tailing from the N-th line of the file
48
61
  # and we're even allowed to specify different -n+N for different files!
@@ -54,7 +67,7 @@ build_tail_args() {
54
67
  # at this point, no last position was recorded in the SINCEDB for fpath,
55
68
  # in this case we'd tail from the end of file.
56
69
 
57
- TAIL_ARGS[$((i++))]="-n+$(($(wc -l "$fpath" | cut -d' ' -f1) + 1))"
70
+ TAIL_ARGS[$((i++))]="-n+$(( nlines + 1 ))"
58
71
  # Note: we can't just ask tail to seek to the end here(ie, with -n0) since
59
72
  # then we'd lose track of the line count.
60
73
  # Note: if fpath doesn't exist yet, then the above evaluates to "-n+1", which
@@ -15,6 +15,16 @@ require 'logger'
15
15
 
16
16
  module Log2Json
17
17
 
18
+ LEVELS = {
19
+ :debug => Logger::DEBUG,
20
+ :info => Logger::INFO,
21
+ :warn => Logger::WARN,
22
+ :error => Logger::ERROR,
23
+ :fatal => Logger::FATAL,
24
+ :unknown => 5
25
+ }
26
+ LEVELS.default = Logger::INFO
27
+
18
28
  def self.log_formatter
19
29
  proc do |severity, datetime, progname, msg|
20
30
  "#{datetime.strftime('%Y-%m-%dT%H:%M:%S%z')}: [#{severity}] #{$$} #{msg.gsub(/\n/, '#012')}\n"
@@ -37,6 +47,7 @@ module Log2Json
37
47
  config.active_record.colorize_logging = false
38
48
  end
39
49
  logger = ::Logger.new(path)
50
+ logger.level = LEVELS[config.log_level]
40
51
  logger.formatter = ::Log2Json::log_formatter
41
52
  if defined?(ActiveSupport::TaggedLogging)
42
53
  ActiveSupport::TaggedLogging.new(logger)
@@ -49,6 +60,7 @@ module Log2Json
49
60
  #
50
61
  def self.create_custom_unicorn_logger(config)
51
62
  logger = ::Logger.new(config.set[:stderr_path])
63
+ logger.level = Logger::INFO
52
64
  logger.formatter = ::Log2Json::log_formatter
53
65
  logger
54
66
  end
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'log2json-loggers'
3
- s.version = '0.1.9'
3
+ s.version = '0.1.11'
4
4
  s.summary = "Custom loggers for Rails and Unicorn that use log2json's single-line log format."
5
5
  s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
6
6
  s.authors = ['Jack Kuan']
data/log2json.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'log2json'
3
- s.version = '0.1.11'
3
+ s.version = '0.1.12'
4
4
  s.summary = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
5
5
  s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
6
6
  s.authors = ['Jack Kuan']
@@ -1,9 +1,25 @@
1
- 1049a1050,1051
2
- > if (print_headers)
3
- > printf ("==> %s <== [new_file]\n", pretty_name (f));
4
- 1171a1174,1175
5
- > if (print_headers)
6
- > printf ("==> %s <== [truncated]\n", name);
7
- 1286a1291,1292
8
- > if (print_headers)
9
- > printf ("==> %s <== [truncated]\n", name);
1
+ 83,85d82
2
+ < /* Size of the array that stores the -n option values. */
3
+ < #define MAX_FILE_SKIP_LINES 256
4
+ <
5
+ 187,189d183
6
+ < /* Stores the number of lines to skip from the start for each file. */
7
+ < static int n_units_argv[MAX_FILE_SKIP_LINES];
8
+ <
9
+ 1056,1057d1049
10
+ < if (print_headers)
11
+ < printf ("==> %s <== [new_file]\n", pretty_name (f));
12
+ 1180,1181d1171
13
+ < if (print_headers)
14
+ < printf ("==> %s <== [truncated]\n", name);
15
+ 1297,1298d1286
16
+ < if (print_headers)
17
+ < printf ("==> %s <== [truncated]\n", name);
18
+ 1930d1917
19
+ < int n_units_argc = 0;
20
+ 1962d1948
21
+ < n_units_argv[n_units_argc++] = *n_units;
22
+ 2177c2163
23
+ < ok &= tail_file (&F[i], i < MAX_FILE_SKIP_LINES ? n_units_argv[i] : n_units);
24
+ ---
25
+ > ok &= tail_file (&F[i], n_units);
data/src/tail.c CHANGED
@@ -80,6 +80,9 @@
80
80
  /* FIXME: make Follow_name the default? */
81
81
  #define DEFAULT_FOLLOW_MODE Follow_descriptor
82
82
 
83
+ /* Size of the array that stores the -n option values. */
84
+ #define MAX_FILE_SKIP_LINES 256
85
+
83
86
  enum Follow_mode
84
87
  {
85
88
  /* Follow the name of each file: if the file is renamed, try to reopen
@@ -181,6 +184,9 @@ static bool forever;
181
184
  /* If true, count from start of file instead of end. */
182
185
  static bool from_start;
183
186
 
187
+ /* Stores the number of lines to skip from the start for each file. */
188
+ static int n_units_argv[MAX_FILE_SKIP_LINES];
189
+
184
190
  /* If true, print filename headers. */
185
191
  static bool print_headers;
186
192
 
@@ -1921,6 +1927,7 @@ parse_options (int argc, char **argv,
1921
1927
  double *sleep_interval)
1922
1928
  {
1923
1929
  int c;
1930
+ int n_units_argc = 0;
1924
1931
 
1925
1932
  while ((c = getopt_long (argc, argv, "c:n:fFqs:v0123456789",
1926
1933
  long_options, NULL))
@@ -1952,6 +1959,7 @@ parse_options (int argc, char **argv,
1952
1959
  ? _("invalid number of lines")
1953
1960
  : _("invalid number of bytes")));
1954
1961
  }
1962
+ n_units_argv[n_units_argc++] = *n_units;
1955
1963
  }
1956
1964
  break;
1957
1965
 
@@ -2166,7 +2174,7 @@ main (int argc, char **argv)
2166
2174
  xfreopen (NULL, "wb", stdout);
2167
2175
 
2168
2176
  for (i = 0; i < n_files; i++)
2169
- ok &= tail_file (&F[i], n_units);
2177
+ ok &= tail_file (&F[i], i < MAX_FILE_SKIP_LINES ? n_units_argv[i] : n_units);
2170
2178
 
2171
2179
  if (forever && ignore_fifo_and_pipe (F, n_files))
2172
2180
  {
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: log2json
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.11
4
+ version: 0.1.12
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-10-15 00:00:00.000000000 Z
12
+ date: 2013-10-21 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: jls-grok
@@ -114,6 +114,7 @@ files:
114
114
  - README
115
115
  - bin/lines2redis
116
116
  - bin/nginxlog2json
117
+ - bin/redis2disk
117
118
  - bin/redis2es
118
119
  - bin/syslog2json
119
120
  - bin/tail