log2json 0.1.14 → 0.1.15
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/redis2es +6 -1
- data/log2json.gemspec +1 -1
- metadata +2 -2
data/bin/redis2es
CHANGED
|
@@ -64,6 +64,9 @@ HTTP_LOG.level = Logger::WARN
|
|
|
64
64
|
# Note: if the ES server can handle the load, we might be able to run multiple instances
|
|
65
65
|
# of this script to process the queue and send logs ES with multiple connections.
|
|
66
66
|
|
|
67
|
+
# only renew a connection that's been idle for 5 mintues.
|
|
68
|
+
:idle_timeout => 60 * 5,
|
|
69
|
+
|
|
67
70
|
# we'll retry posting to ES since having duplicate data in ES is better than not having them.
|
|
68
71
|
:force_retry => true,
|
|
69
72
|
:url => "http://#{ES_HOST}:#{ES_PORT}"
|
|
@@ -105,6 +108,8 @@ end
|
|
|
105
108
|
def enqueue(logstr)
|
|
106
109
|
# add header for each entry according to http://www.elasticsearch.org/guide/reference/api/bulk/
|
|
107
110
|
log = JSON.load(logstr.force_encoding(LOG_ENCODING))
|
|
111
|
+
#FIXME: we can parse it ourselves. we only need to extract the @timestamp value.
|
|
112
|
+
|
|
108
113
|
@queue << (
|
|
109
114
|
{"index" => {"_index" => es_index(log["@timestamp"]), "_type" => log["@type"]}}.to_json <<
|
|
110
115
|
"\n" <<
|
|
@@ -150,7 +155,7 @@ def main
|
|
|
150
155
|
rescue Redis::CommandError => e
|
|
151
156
|
#Note: redis script reload logic and code are copied from logstash.
|
|
152
157
|
if e.to_s =~ /NOSCRIPT/ then
|
|
153
|
-
|
|
158
|
+
LOG.warn("Redis may have been restarted, reloading redis batch EVAL script", :exception => e);
|
|
154
159
|
load_redis_script()
|
|
155
160
|
retry
|
|
156
161
|
else
|
data/log2json.gemspec
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'log2json'
|
|
3
|
-
s.version = '0.1.
|
|
3
|
+
s.version = '0.1.15'
|
|
4
4
|
s.summary = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
|
|
5
5
|
s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
|
|
6
6
|
s.authors = ['Jack Kuan']
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: log2json
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.15
|
|
5
5
|
prerelease:
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2013-10-
|
|
12
|
+
date: 2013-10-28 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: jls-grok
|