log2json 0.1.11 → 0.1.12
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/redis2disk +96 -0
- data/bin/tail +0 -0
- data/bin/tail-log.sh +18 -5
- data/lib/log2json/railslogger.rb +12 -0
- data/log2json-loggers.gemspec +1 -1
- data/log2json.gemspec +1 -1
- data/src/coreutils-8.13_tail.patch +25 -9
- data/src/tail.c +9 -1
- metadata +3 -2
data/bin/redis2disk
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
#
|
3
|
+
# Check every INTERVAL seconds the size of the log2json log queue in Redis,
|
4
|
+
# and reduce the queue size if its average rate of change over the INTERVAL
|
5
|
+
# is greater than MAX_RATE for CROSS_COUNT times within EVAL_PERIOD seconds.
|
6
|
+
#
|
7
|
+
# The logs will be saved on disk and can be pushed back to the queue by
|
8
|
+
# another script at a slower rate when the queue is empty most of times.
|
9
|
+
#
|
10
|
+
#
|
11
|
+
set -e
|
12
|
+
|
13
|
+
# name of the queue in Redis. Must be a list in Redis.
|
14
|
+
QUEUE=${QUEUE:-jsonlogs}
|
15
|
+
|
16
|
+
# average number of log records per second over INTERVAL seconds
|
17
|
+
MAX_RATE=${MAX_RATE:-25000}
|
18
|
+
|
19
|
+
# how many times the average rate of change greater than MAX_RATE should happen
|
20
|
+
# with in EVAL_PERIOD seconds before we reduce the queue size.
|
21
|
+
CROSS_COUNT=${CROSS_COUNT:-2}
|
22
|
+
|
23
|
+
# the length of time(in seconds) within which a "cross-over"
|
24
|
+
# is considered valid and so counts toward CROSS_COUNT.
|
25
|
+
# It should be a multiple of INTERVAL.
|
26
|
+
EVAL_PERIOD=${EVAL_PERIOD:-15}
|
27
|
+
|
28
|
+
# Samples the queue size roughly every INTERVAL seconds.
|
29
|
+
# The smaller the more accurate and less chance to miss sudden burst of traffic.
|
30
|
+
INTERVAL=${INTERVAL:-5}
|
31
|
+
|
32
|
+
# LUA script for Redis to lpop N log records off the queue.
|
33
|
+
LUA_LPOP_KEY_N='
|
34
|
+
local i = tonumber(ARGV[1])
|
35
|
+
local res = {}
|
36
|
+
local length = redis.call("llen",KEYS[1])
|
37
|
+
if length < i then i = length end
|
38
|
+
while (i > 0) do
|
39
|
+
local item = redis.call("lpop", KEYS[1])
|
40
|
+
if (not item) then
|
41
|
+
break
|
42
|
+
end
|
43
|
+
table.insert(res, item)
|
44
|
+
i = i-1
|
45
|
+
end
|
46
|
+
return res
|
47
|
+
'
|
48
|
+
|
49
|
+
# folder to store the log records off loaded from the queue
|
50
|
+
BATCH_DIR=${BATCH_DIR:-/mnt/redis}
|
51
|
+
|
52
|
+
log() { echo "$(date): $*"; }
|
53
|
+
|
54
|
+
while true
|
55
|
+
do
|
56
|
+
if [ $(( $(date +%s) - period_start )) -ge "$EVAL_PERIOD" ]; then
|
57
|
+
period_start=$(date +%s)
|
58
|
+
count=0 # how many times have we crossed MAX_RATE within the period so far.
|
59
|
+
# reset to 0 at the start of a new period or when it reaches CROSS_COUNT.
|
60
|
+
log "New period begins---"
|
61
|
+
fi
|
62
|
+
s2=$(redis-cli llen $QUEUE) # current queue size
|
63
|
+
log "queue size=$s2"
|
64
|
+
|
65
|
+
if [ "$s1" ]; then
|
66
|
+
delta=$(( s2 - s1 ))
|
67
|
+
rate=$(( delta / INTERVAL ))
|
68
|
+
log "delta=$delta rate=$rate"
|
69
|
+
|
70
|
+
# if within the period, we have crossed MAX_RATE CROSS_COUNT times
|
71
|
+
# then move the records from the queue to a batch file.
|
72
|
+
if [ "$rate" -gt "$MAX_RATE" ]; then
|
73
|
+
count=$((count + 1))
|
74
|
+
log "MAX_RATE($MAX_RATE/s) crossed! count=$count"
|
75
|
+
|
76
|
+
if [ "$count" -ge "$CROSS_COUNT" ]; then
|
77
|
+
batch_file=$BATCH_DIR/${QUEUE}_$(date +%Y-%m-%dT%T.%N%z)
|
78
|
+
n_records=$((delta * count))
|
79
|
+
log "CROSS_COUNT($CROSS_COUNT) reached! Off loading $n_records log records to $batch_file ..."
|
80
|
+
|
81
|
+
redis-cli --eval <(echo "$LUA_LPOP_KEY_N") $QUEUE , $n_records > $batch_file &
|
82
|
+
count=0
|
83
|
+
off_loaded=1
|
84
|
+
fi
|
85
|
+
fi
|
86
|
+
fi
|
87
|
+
|
88
|
+
if [ "$off_loaded" ]; then
|
89
|
+
s1=$(redis-cli llen $QUEUE)
|
90
|
+
off_loaded=
|
91
|
+
else
|
92
|
+
s1=$s2
|
93
|
+
fi
|
94
|
+
sleep "$INTERVAL"
|
95
|
+
done
|
96
|
+
|
data/bin/tail
CHANGED
Binary file
|
data/bin/tail-log.sh
CHANGED
@@ -4,9 +4,17 @@ set -e
|
|
4
4
|
|
5
5
|
# Find out the absolute path to the tail utility.
|
6
6
|
# This is a patched version of the tail utility in GNU coreutils-8.13 compiled for Ubuntu 12.04 LTS.
|
7
|
-
#
|
8
|
-
#
|
9
|
-
#
|
7
|
+
# With the following differences:
|
8
|
+
#
|
9
|
+
# - if header will be shown(ie, with -v or when multiple files are specified),
|
10
|
+
# it will also print "==> file.name <== [event]" to stdout whenever a file truncation or a new file is
|
11
|
+
# detected. [event] will be one of "[new_file]" or "[truncated]".
|
12
|
+
#
|
13
|
+
# - It allows the use of multiple -n options. Each -n will apply to the files specified on the
|
14
|
+
# command line in order.(ie, first -n N corresponds to the first file, etc.)
|
15
|
+
# If there are more files listed than the number of -n options, then the last -n applies to the
|
16
|
+
# rest of the files.
|
17
|
+
#
|
10
18
|
TAIL=$(
|
11
19
|
ruby -- - <<'EOF'
|
12
20
|
require 'log2json'
|
@@ -37,12 +45,17 @@ build_tail_args() {
|
|
37
45
|
for fpath in "$@"
|
38
46
|
do
|
39
47
|
sincedb_path=$SINCEDB_DIR/$fpath.since
|
48
|
+
nlines=$(wc -l "$fpath" | cut -d' ' -f1)
|
49
|
+
nlines=${nlines:-0}
|
50
|
+
|
40
51
|
if [ -r "$sincedb_path" ]; then
|
41
52
|
read line < "$sincedb_path"
|
42
53
|
t=($line)
|
43
54
|
# if inode number is unchanged and the current file size is not smaller
|
44
55
|
# then we start tailing from 1 + the line number recorded in the sincedb.
|
45
|
-
if [[ ${t[0]} == $(stat -c "%i" "$fpath") &&
|
56
|
+
if [[ ${t[0]} == $(stat -c "%i" "$fpath") &&
|
57
|
+
${t[1]} -le $(stat -c "%s" "$fpath") &&
|
58
|
+
${t[2]} -le "$nlines" ]]; then
|
46
59
|
TAIL_ARGS[$((i++))]="-n+$((t[2] + 1))"
|
47
60
|
# tail -n+N means start tailing from the N-th line of the file
|
48
61
|
# and we're even allowed to specify different -n+N for different files!
|
@@ -54,7 +67,7 @@ build_tail_args() {
|
|
54
67
|
# at this point, no last position was recorded in the SINCEDB for fpath,
|
55
68
|
# in this case we'd tail from the end of file.
|
56
69
|
|
57
|
-
TAIL_ARGS[$((i++))]="-n+$((
|
70
|
+
TAIL_ARGS[$((i++))]="-n+$(( nlines + 1 ))"
|
58
71
|
# Note: we can't just ask tail to seek to the end here(ie, with -n0) since
|
59
72
|
# then we'd lose track of the line count.
|
60
73
|
# Note: if fpath doesn't exist yet, then the above evaluates to "-n+1", which
|
data/lib/log2json/railslogger.rb
CHANGED
@@ -15,6 +15,16 @@ require 'logger'
|
|
15
15
|
|
16
16
|
module Log2Json
|
17
17
|
|
18
|
+
LEVELS = {
|
19
|
+
:debug => Logger::DEBUG,
|
20
|
+
:info => Logger::INFO,
|
21
|
+
:warn => Logger::WARN,
|
22
|
+
:error => Logger::ERROR,
|
23
|
+
:fatal => Logger::FATAL,
|
24
|
+
:unknown => 5
|
25
|
+
}
|
26
|
+
LEVELS.default = Logger::INFO
|
27
|
+
|
18
28
|
def self.log_formatter
|
19
29
|
proc do |severity, datetime, progname, msg|
|
20
30
|
"#{datetime.strftime('%Y-%m-%dT%H:%M:%S%z')}: [#{severity}] #{$$} #{msg.gsub(/\n/, '#012')}\n"
|
@@ -37,6 +47,7 @@ module Log2Json
|
|
37
47
|
config.active_record.colorize_logging = false
|
38
48
|
end
|
39
49
|
logger = ::Logger.new(path)
|
50
|
+
logger.level = LEVELS[config.log_level]
|
40
51
|
logger.formatter = ::Log2Json::log_formatter
|
41
52
|
if defined?(ActiveSupport::TaggedLogging)
|
42
53
|
ActiveSupport::TaggedLogging.new(logger)
|
@@ -49,6 +60,7 @@ module Log2Json
|
|
49
60
|
#
|
50
61
|
def self.create_custom_unicorn_logger(config)
|
51
62
|
logger = ::Logger.new(config.set[:stderr_path])
|
63
|
+
logger.level = Logger::INFO
|
52
64
|
logger.formatter = ::Log2Json::log_formatter
|
53
65
|
logger
|
54
66
|
end
|
data/log2json-loggers.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'log2json-loggers'
|
3
|
-
s.version = '0.1.
|
3
|
+
s.version = '0.1.11'
|
4
4
|
s.summary = "Custom loggers for Rails and Unicorn that use log2json's single-line log format."
|
5
5
|
s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
|
6
6
|
s.authors = ['Jack Kuan']
|
data/log2json.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'log2json'
|
3
|
-
s.version = '0.1.
|
3
|
+
s.version = '0.1.12'
|
4
4
|
s.summary = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
|
5
5
|
s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
|
6
6
|
s.authors = ['Jack Kuan']
|
@@ -1,9 +1,25 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
1
|
+
83,85d82
|
2
|
+
< /* Size of the array that stores the -n option values. */
|
3
|
+
< #define MAX_FILE_SKIP_LINES 256
|
4
|
+
<
|
5
|
+
187,189d183
|
6
|
+
< /* Stores the number of lines to skip from the start for each file. */
|
7
|
+
< static int n_units_argv[MAX_FILE_SKIP_LINES];
|
8
|
+
<
|
9
|
+
1056,1057d1049
|
10
|
+
< if (print_headers)
|
11
|
+
< printf ("==> %s <== [new_file]\n", pretty_name (f));
|
12
|
+
1180,1181d1171
|
13
|
+
< if (print_headers)
|
14
|
+
< printf ("==> %s <== [truncated]\n", name);
|
15
|
+
1297,1298d1286
|
16
|
+
< if (print_headers)
|
17
|
+
< printf ("==> %s <== [truncated]\n", name);
|
18
|
+
1930d1917
|
19
|
+
< int n_units_argc = 0;
|
20
|
+
1962d1948
|
21
|
+
< n_units_argv[n_units_argc++] = *n_units;
|
22
|
+
2177c2163
|
23
|
+
< ok &= tail_file (&F[i], i < MAX_FILE_SKIP_LINES ? n_units_argv[i] : n_units);
|
24
|
+
---
|
25
|
+
> ok &= tail_file (&F[i], n_units);
|
data/src/tail.c
CHANGED
@@ -80,6 +80,9 @@
|
|
80
80
|
/* FIXME: make Follow_name the default? */
|
81
81
|
#define DEFAULT_FOLLOW_MODE Follow_descriptor
|
82
82
|
|
83
|
+
/* Size of the array that stores the -n option values. */
|
84
|
+
#define MAX_FILE_SKIP_LINES 256
|
85
|
+
|
83
86
|
enum Follow_mode
|
84
87
|
{
|
85
88
|
/* Follow the name of each file: if the file is renamed, try to reopen
|
@@ -181,6 +184,9 @@ static bool forever;
|
|
181
184
|
/* If true, count from start of file instead of end. */
|
182
185
|
static bool from_start;
|
183
186
|
|
187
|
+
/* Stores the number of lines to skip from the start for each file. */
|
188
|
+
static int n_units_argv[MAX_FILE_SKIP_LINES];
|
189
|
+
|
184
190
|
/* If true, print filename headers. */
|
185
191
|
static bool print_headers;
|
186
192
|
|
@@ -1921,6 +1927,7 @@ parse_options (int argc, char **argv,
|
|
1921
1927
|
double *sleep_interval)
|
1922
1928
|
{
|
1923
1929
|
int c;
|
1930
|
+
int n_units_argc = 0;
|
1924
1931
|
|
1925
1932
|
while ((c = getopt_long (argc, argv, "c:n:fFqs:v0123456789",
|
1926
1933
|
long_options, NULL))
|
@@ -1952,6 +1959,7 @@ parse_options (int argc, char **argv,
|
|
1952
1959
|
? _("invalid number of lines")
|
1953
1960
|
: _("invalid number of bytes")));
|
1954
1961
|
}
|
1962
|
+
n_units_argv[n_units_argc++] = *n_units;
|
1955
1963
|
}
|
1956
1964
|
break;
|
1957
1965
|
|
@@ -2166,7 +2174,7 @@ main (int argc, char **argv)
|
|
2166
2174
|
xfreopen (NULL, "wb", stdout);
|
2167
2175
|
|
2168
2176
|
for (i = 0; i < n_files; i++)
|
2169
|
-
ok &= tail_file (&F[i], n_units);
|
2177
|
+
ok &= tail_file (&F[i], i < MAX_FILE_SKIP_LINES ? n_units_argv[i] : n_units);
|
2170
2178
|
|
2171
2179
|
if (forever && ignore_fifo_and_pipe (F, n_files))
|
2172
2180
|
{
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: log2json
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.12
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-10-
|
12
|
+
date: 2013-10-21 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: jls-grok
|
@@ -114,6 +114,7 @@ files:
|
|
114
114
|
- README
|
115
115
|
- bin/lines2redis
|
116
116
|
- bin/nginxlog2json
|
117
|
+
- bin/redis2disk
|
117
118
|
- bin/redis2es
|
118
119
|
- bin/syslog2json
|
119
120
|
- bin/tail
|