log2json 0.1.14 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/redis2es +6 -1
- data/log2json.gemspec +1 -1
- metadata +2 -2
data/bin/redis2es
CHANGED
|
@@ -64,6 +64,9 @@ HTTP_LOG.level = Logger::WARN
|
|
|
64
64
|
# Note: if the ES server can handle the load, we might be able to run multiple instances
|
|
65
65
|
# of this script to process the queue and send logs ES with multiple connections.
|
|
66
66
|
|
|
67
|
+
# only renew a connection that's been idle for 5 mintues.
|
|
68
|
+
:idle_timeout => 60 * 5,
|
|
69
|
+
|
|
67
70
|
# we'll retry posting to ES since having duplicate data in ES is better than not having them.
|
|
68
71
|
:force_retry => true,
|
|
69
72
|
:url => "http://#{ES_HOST}:#{ES_PORT}"
|
|
@@ -105,6 +108,8 @@ end
|
|
|
105
108
|
def enqueue(logstr)
|
|
106
109
|
# add header for each entry according to http://www.elasticsearch.org/guide/reference/api/bulk/
|
|
107
110
|
log = JSON.load(logstr.force_encoding(LOG_ENCODING))
|
|
111
|
+
#FIXME: we can parse it ourselves. we only need to extract the @timestamp value.
|
|
112
|
+
|
|
108
113
|
@queue << (
|
|
109
114
|
{"index" => {"_index" => es_index(log["@timestamp"]), "_type" => log["@type"]}}.to_json <<
|
|
110
115
|
"\n" <<
|
|
@@ -150,7 +155,7 @@ def main
|
|
|
150
155
|
rescue Redis::CommandError => e
|
|
151
156
|
#Note: redis script reload logic and code are copied from logstash.
|
|
152
157
|
if e.to_s =~ /NOSCRIPT/ then
|
|
153
|
-
|
|
158
|
+
LOG.warn("Redis may have been restarted, reloading redis batch EVAL script", :exception => e);
|
|
154
159
|
load_redis_script()
|
|
155
160
|
retry
|
|
156
161
|
else
|
data/log2json.gemspec
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'log2json'
|
|
3
|
-
s.version = '0.1.
|
|
3
|
+
s.version = '0.1.15'
|
|
4
4
|
s.summary = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
|
|
5
5
|
s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
|
|
6
6
|
s.authors = ['Jack Kuan']
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: log2json
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.15
|
|
5
5
|
prerelease:
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2013-10-
|
|
12
|
+
date: 2013-10-28 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: jls-grok
|