log2json 0.1.9 → 0.1.10
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/redis2es +10 -6
- data/lib/log2json/railslogger.rb +32 -68
- data/log2json-loggers.gemspec +2 -2
- data/log2json.gemspec +1 -1
- metadata +2 -2
data/bin/redis2es
CHANGED
@@ -41,11 +41,11 @@ const(:LOG_ENCODING, 'UTF-8')
|
|
41
41
|
const(:LOG_INDEX_NAME, 'log2json-%Y.%m.%d')
|
42
42
|
|
43
43
|
# max number of log records allowed in the queue.
|
44
|
-
const(:FLUSH_SIZE,
|
44
|
+
const(:FLUSH_SIZE, 800)
|
45
45
|
|
46
46
|
# flush the queue roughly every FLUSH_TIMEOUT seconds.
|
47
47
|
# This value must be >= 2 and it must be a multiple of 2.
|
48
|
-
const(:FLUSH_TIMEOUT,
|
48
|
+
const(:FLUSH_TIMEOUT, 30)
|
49
49
|
if FLUSH_TIMEOUT < 2 or FLUSH_TIMEOUT % 2 != 0
|
50
50
|
STDERR.write("Invalid FLUSH_TIMEOUT=#{FLUSH_TIMEOUT}\n")
|
51
51
|
exit 1
|
@@ -75,16 +75,16 @@ HTTP_LOG.level = Logger::WARN
|
|
75
75
|
def flush_queue
|
76
76
|
if not @queue.empty?
|
77
77
|
req = Net::HTTP::Post.new('/_bulk')
|
78
|
-
req.body = @queue.join("\n")
|
78
|
+
req.body = @queue.join("\n")
|
79
79
|
response = nil
|
80
80
|
begin
|
81
81
|
response = @@http.request(req)
|
82
82
|
ensure
|
83
83
|
if response.nil? or response.code != '200'
|
84
84
|
LOG.error(response.body) if not response.nil?
|
85
|
-
#FIXME: might be a good idea to push the undelivered log records to another queue in redis.
|
86
85
|
LOG.warn("Failed sending bulk request(#{@queue.size} records) to ES! Logging the request body instead.")
|
87
86
|
LOG.info("Failed request body:\n"+req.body)
|
87
|
+
#FIXME: write the queue to another file. This would allow us to resend these records on startup.
|
88
88
|
end
|
89
89
|
end
|
90
90
|
@queue.clear()
|
@@ -107,8 +107,12 @@ def enqueue(logstr)
|
|
107
107
|
log = JSON.load(logstr.force_encoding(LOG_ENCODING))
|
108
108
|
|
109
109
|
# add header for each entry according to http://www.elasticsearch.org/guide/reference/api/bulk/
|
110
|
-
@queue <<
|
111
|
-
|
110
|
+
@queue << (
|
111
|
+
{"index" => {"_index" => es_index(log["@timestamp"]), "_type" => log["@type"]}}.to_json <<
|
112
|
+
"\n" <<
|
113
|
+
log.to_json <<
|
114
|
+
"\n"
|
115
|
+
)
|
112
116
|
end
|
113
117
|
|
114
118
|
def load_redis_script
|
data/lib/log2json/railslogger.rb
CHANGED
@@ -1,95 +1,59 @@
|
|
1
1
|
# Use it like this in your config/environments/{staging,production}.rb:
|
2
2
|
#
|
3
|
-
#
|
3
|
+
# require 'log2json/railslogger'
|
4
|
+
# config.logger = ::Log2Json::create_custom_ralis_logger(config)
|
4
5
|
#
|
5
6
|
# Also, in unicorn.rb, add it like this:
|
6
7
|
#
|
7
|
-
#
|
8
|
+
# if %w(staging production).include?(ENV['RAILS_ENV'])
|
9
|
+
# require 'log2json/railslogger'
|
10
|
+
# logger ::Log2Json::create_custom_unicorn_logger(self)
|
11
|
+
# end
|
8
12
|
#
|
9
13
|
|
10
14
|
require 'logger'
|
11
|
-
|
15
|
+
|
12
16
|
module Log2Json
|
13
17
|
|
14
|
-
def self.
|
15
|
-
|
16
|
-
logger.formatter = proc do |severity, datetime, progname, msg|
|
18
|
+
def self.log_formatter
|
19
|
+
proc do |severity, datetime, progname, msg|
|
17
20
|
"#{datetime.strftime('%Y-%m-%dT%H:%M:%S%z')}: [#{severity}] #{$$} #{msg.gsub(/\n/, '#012')}\n"
|
21
|
+
# Note: Following rsyslog's convention, all newlines are converted to '#012'.
|
18
22
|
end
|
19
|
-
logger
|
20
23
|
end
|
21
24
|
|
22
|
-
# Create a custom logger that
|
23
|
-
#
|
24
|
-
# parsing by a log2json log monitoring script.
|
25
|
+
# Create a custom logger that uses its own formatting for easier parsing
|
26
|
+
# by a log2json log monitoring script.
|
25
27
|
#
|
26
|
-
def self.create_custom_rails_logger(
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
28
|
+
def self.create_custom_rails_logger(config)
|
29
|
+
if config.respond_to?(:paths) # assume rails 3
|
30
|
+
path = config.paths["log"].first
|
31
|
+
unless File.exist? File.dirname path
|
32
|
+
FileUtils.mkdir_p File.dirname path
|
33
|
+
end
|
34
|
+
config.colorize_logging = false
|
35
|
+
else # assume rails 2
|
36
|
+
path = config.log_path
|
37
|
+
config.active_record.colorize_logging = false
|
38
|
+
end
|
39
|
+
logger = ::Logger.new(path)
|
40
|
+
logger.formatter = ::Log2Json::log_formatter
|
41
|
+
if defined?(ActiveSupport::TaggedLogging)
|
42
|
+
ActiveSupport::TaggedLogging.new(logger)
|
43
|
+
else
|
44
|
+
logger
|
31
45
|
end
|
32
|
-
f = File.open path, 'a'
|
33
|
-
f.binmode
|
34
|
-
f.sync = true # make sure every write flushes
|
35
|
-
|
36
|
-
logger = ActiveSupport::TaggedLogging.new(
|
37
|
-
ActiveSupport::BufferedLogger.new(f)
|
38
|
-
)
|
39
|
-
logger.level = ActiveSupport::BufferedLogger.const_get(config.log_level.to_s.upcase)
|
40
|
-
|
41
|
-
logger.extend(::Log2Json::Logger.broadcast(
|
42
|
-
::Log2Json::create_custom_logger(to_path)))
|
43
46
|
end
|
44
47
|
|
45
48
|
# Simiar to the custom rails logger, but for unicorn.
|
46
49
|
#
|
47
|
-
def self.create_custom_unicorn_logger(
|
50
|
+
def self.create_custom_unicorn_logger(config)
|
48
51
|
logger = ::Logger.new(config.set[:stderr_path])
|
49
|
-
logger.
|
50
|
-
|
52
|
+
logger.formatter = ::Log2Json::log_formatter
|
53
|
+
logger
|
51
54
|
end
|
52
55
|
|
53
56
|
|
54
|
-
# Code stolen from activesupport-4.0.0
|
55
|
-
class Logger < ::Logger
|
56
|
-
|
57
|
-
# Broadcasts logs to multiple loggers.
|
58
|
-
def self.broadcast(logger) # :nodoc:
|
59
|
-
Module.new do
|
60
|
-
define_method(:add) do |*args, &block|
|
61
|
-
logger.add(*args, &block)
|
62
|
-
super(*args, &block)
|
63
|
-
end
|
64
|
-
|
65
|
-
define_method(:<<) do |x|
|
66
|
-
logger << x
|
67
|
-
super(x)
|
68
|
-
end
|
69
|
-
|
70
|
-
define_method(:close) do
|
71
|
-
logger.close
|
72
|
-
super()
|
73
|
-
end
|
74
|
-
|
75
|
-
define_method(:progname=) do |name|
|
76
|
-
logger.progname = name
|
77
|
-
super(name)
|
78
|
-
end
|
79
|
-
|
80
|
-
define_method(:formatter=) do |formatter|
|
81
|
-
logger.formatter = formatter
|
82
|
-
super(formatter)
|
83
|
-
end
|
84
|
-
|
85
|
-
define_method(:level=) do |level|
|
86
|
-
logger.level = level
|
87
|
-
super(level)
|
88
|
-
end
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
57
|
end
|
94
58
|
|
95
59
|
|
data/log2json-loggers.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'log2json-loggers'
|
3
|
-
s.version = '0.1.
|
4
|
-
s.summary = "Custom loggers for Rails and Unicorn that
|
3
|
+
s.version = '0.1.9'
|
4
|
+
s.summary = "Custom loggers for Rails and Unicorn that use log2json's single-line log format."
|
5
5
|
s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
|
6
6
|
s.authors = ['Jack Kuan']
|
7
7
|
s.email = 'jack.kuan@thescore.com'
|
data/log2json.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'log2json'
|
3
|
-
s.version = '0.1.
|
3
|
+
s.version = '0.1.10'
|
4
4
|
s.summary = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
|
5
5
|
s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
|
6
6
|
s.authors = ['Jack Kuan']
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: log2json
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.10
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-10-
|
12
|
+
date: 2013-10-11 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: jls-grok
|