anschel 0.6.0 → 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Readme.md +19 -14
- data/VERSION +1 -1
- data/lib/anschel/filter/index.rb +22 -5
- data/lib/anschel/filter/parse.rb +9 -0
- data/lib/anschel/filter/stamp.rb +12 -3
- data/lib/anschel/output/device.rb +1 -1
- data/lib/anschel/output/elasticsearch.rb +10 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8f81ae5fdcf71f580680520067255911285629aa
|
4
|
+
data.tar.gz: 68ab5e077fa13d9d4e142fe798f222912dce5f6a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0179da5dc7e53fc18e3c490f4e49f521e210184a0df80a5d834052237b2f476f4b99fbcad40bd390f1f318a9af9559f9609117098d8e7cf02d35829a560d2aab
|
7
|
+
data.tar.gz: 95309336805570a5e7cd50dfa1d47e43677caa3a72f78ea1a8e8b7fadc68146c26e00feb73706b0a2620868f73777af78b94c4b6368eeb8f4bc1c8e30c74d88f
|
data/Readme.md
CHANGED
@@ -54,10 +54,13 @@ It's kinda like a JSON version of the Logstash config language:
|
|
54
54
|
"pattern": "[%d] %p %m (%c)%n"
|
55
55
|
},
|
56
56
|
|
57
|
-
//
|
58
|
-
|
59
|
-
|
60
|
-
|
57
|
+
// Specify any number of inputs
|
58
|
+
"input": [
|
59
|
+
|
60
|
+
// Kafka is the primary input; see the `jruby-kafka` homepage for
|
61
|
+
// more details: https://github.com/joekiller/jruby-kafka
|
62
|
+
{
|
63
|
+
"kind": "kafka",
|
61
64
|
"queue_size": 2000,
|
62
65
|
"zk_connect": "localhost:2181",
|
63
66
|
"zk_connect_timeout": 6000,
|
@@ -81,12 +84,15 @@ It's kinda like a JSON version of the Logstash config language:
|
|
81
84
|
"consumer_timeout_ms": -1,
|
82
85
|
"consumer_restart_sleep_ms": 0
|
83
86
|
}
|
84
|
-
|
87
|
+
],
|
88
|
+
|
89
|
+
// Specify any number of outputs
|
90
|
+
"output": [
|
85
91
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
92
|
+
// Elasticsearch is the primary output; see the `elasticsearch-ruby`
|
93
|
+
// homepage for more: https://github.com/elastic/elasticsearch-ruby
|
94
|
+
{
|
95
|
+
"kind": "elasticsearch",
|
90
96
|
"queue_size": 2000,
|
91
97
|
"bulk_size": 200,
|
92
98
|
"hosts": [ "localhost:9200" ],
|
@@ -95,7 +101,7 @@ It's kinda like a JSON version of the Logstash config language:
|
|
95
101
|
"reload_on_failure": true,
|
96
102
|
"sniffer_timeout": 5
|
97
103
|
}
|
98
|
-
|
104
|
+
],
|
99
105
|
|
100
106
|
// Just like Logstash, Anschel has a notion of filters
|
101
107
|
"filter": {
|
@@ -148,7 +154,6 @@ You might deploy Anschel with Upstart. Here's a minimal config:
|
|
148
154
|
|
149
155
|
start on startup
|
150
156
|
stop on shutdown
|
151
|
-
respawn
|
152
157
|
|
153
158
|
exec java -jar anschel-x.y.z.jar \
|
154
159
|
--config /etc/anschel.json --log /var/log/anschel.log
|
@@ -161,7 +166,7 @@ You might deploy Anschel with Upstart. Here's a minimal config:
|
|
161
166
|
|
162
167
|
_In development_
|
163
168
|
|
164
|
-
-
|
165
|
-
- Support for file (device) output (v0.4)
|
169
|
+
- Allow multiple input and output configurations (v0.6)
|
166
170
|
- Support for RabbitMQ input (v0.5)
|
167
|
-
-
|
171
|
+
- Support for file (device) output (v0.4)
|
172
|
+
- Intial implementation of the Kafka-to-Elasticsearch pipeline
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.6.
|
1
|
+
0.6.1
|
data/lib/anschel/filter/index.rb
CHANGED
@@ -12,7 +12,9 @@ module Anschel
|
|
12
12
|
stamp = conf.delete(:stamp) || '@timestamp'
|
13
13
|
prefix = conf.delete(:prefix) || 'logs-%{type}-'
|
14
14
|
suffix = conf.delete(:suffix) || '%Y.%m.%d'
|
15
|
-
format = conf.delete(:format) || "yyyy-MM-dd'T'HH:mm:ss.
|
15
|
+
format = conf.delete(:format) || "yyyy-MM-dd'T'HH:mm:ss.SSSZZ" # ISO8601
|
16
|
+
|
17
|
+
error_tag = conf.has_key?(:error_tag) ? conf[:error_tag] : 'index-error'
|
16
18
|
|
17
19
|
stamp = stamp.to_sym
|
18
20
|
|
@@ -26,11 +28,26 @@ module Anschel
|
|
26
28
|
|
27
29
|
lambda do |event|
|
28
30
|
return event unless event.has_key? stamp
|
29
|
-
millis = joda.parseMillis event[stamp]
|
30
31
|
idx_prefix = prefix % event
|
31
|
-
|
32
|
-
|
33
|
-
|
32
|
+
begin
|
33
|
+
millis = joda.parseMillis event[stamp]
|
34
|
+
idx_suffix = Time.at(0.001 * millis).strftime(suffix)
|
35
|
+
event[:_index] = idx_prefix + idx_suffix
|
36
|
+
filtered event, conf
|
37
|
+
rescue java.lang.IllegalArgumentException => e
|
38
|
+
event[:_index] = idx_prefix + Time.now.strftime(suffix)
|
39
|
+
log.warn \
|
40
|
+
event: 'filter-index-warning',
|
41
|
+
reason: 'could not parse event',
|
42
|
+
remediation: 'added bogus index',
|
43
|
+
remediation: "sending to best-guess index '#{event[:_index]}'",
|
44
|
+
raw_event: event
|
45
|
+
if error_tag
|
46
|
+
event[:tags] ||= []
|
47
|
+
event[:tags] << error_tag
|
48
|
+
end
|
49
|
+
filtered event, conf
|
50
|
+
end
|
34
51
|
end
|
35
52
|
end
|
36
53
|
end
|
data/lib/anschel/filter/parse.rb
CHANGED
@@ -22,6 +22,15 @@ module Anschel
|
|
22
22
|
lambda do |event|
|
23
23
|
return event unless event.has_key? field
|
24
24
|
mdata = pattern.match event[field]
|
25
|
+
if mdata.nil?
|
26
|
+
log.error \
|
27
|
+
event: 'parse-filter-error',
|
28
|
+
reason: 'regexp did not match',
|
29
|
+
field: field,
|
30
|
+
pattern: pattern,
|
31
|
+
raw_event: event
|
32
|
+
return filtered(event, conf)
|
33
|
+
end
|
25
34
|
mdata.names.each do |group|
|
26
35
|
event[group.to_sym] = mdata[group]
|
27
36
|
end
|
data/lib/anschel/filter/stamp.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# {
|
2
2
|
# "stamp": {
|
3
|
+
# "utc?": false,
|
3
4
|
# "field": "timestamp",
|
4
5
|
# "pattern": [ "YYYY-MM-dd HH:mm:ss.SSS" ]
|
5
6
|
# }
|
@@ -7,6 +8,7 @@
|
|
7
8
|
module Anschel
|
8
9
|
class Filter
|
9
10
|
def stamp conf, log
|
11
|
+
utc = conf.delete :utc?
|
10
12
|
field = conf.delete :field
|
11
13
|
pattern = conf.delete :pattern
|
12
14
|
target = conf.delete :target
|
@@ -27,6 +29,7 @@ module Anschel
|
|
27
29
|
joda = joda.withOffsetParsed
|
28
30
|
end
|
29
31
|
|
32
|
+
offset_s = utc ? Time.zone_offset(Time.now.zone).to_f : 0.0
|
30
33
|
|
31
34
|
log.debug event: 'compiled-filter', filter: 'stamp', \
|
32
35
|
field: field, pattern: pattern, target: target, error_tag: error_tag
|
@@ -34,19 +37,25 @@ module Anschel
|
|
34
37
|
lambda do |event|
|
35
38
|
return event unless event.has_key? field
|
36
39
|
parsers.each do |joda|
|
37
|
-
millis = joda.parseMillis event[field]
|
38
40
|
begin
|
39
|
-
|
41
|
+
millis = joda.parseMillis event[field]
|
42
|
+
event[target] = Time.at(0.001 * millis + offset_s).iso8601(3)
|
40
43
|
return filtered(event, conf)
|
41
44
|
rescue
|
42
45
|
end
|
43
46
|
end
|
44
47
|
|
48
|
+
log.warn \
|
49
|
+
event: 'stamp-filter-warning',
|
50
|
+
reason: 'could not parse event',
|
51
|
+
remediation: 'using current time for stamp',
|
52
|
+
raw_event: event
|
45
53
|
if error_tag
|
46
54
|
event[:tags] ||= []
|
47
55
|
event[:tags] << error_tag
|
48
56
|
end
|
49
|
-
event
|
57
|
+
event[target] = Time.now.utc.iso8601(3)
|
58
|
+
filtered event, conf
|
50
59
|
end
|
51
60
|
end
|
52
61
|
end
|
@@ -9,7 +9,7 @@ module Anschel
|
|
9
9
|
def initialize config, stats, log
|
10
10
|
log.trace event: 'output', kind: 'device', config: config
|
11
11
|
|
12
|
-
path = config.delete(:path)
|
12
|
+
path = config.delete(:path) || '/dev/stdout'
|
13
13
|
qsize = config.delete(:queue_size) || 2000
|
14
14
|
@queue = SizedQueue.new qsize
|
15
15
|
|
@@ -12,6 +12,7 @@ module Anschel
|
|
12
12
|
class Elasticsearch < Base
|
13
13
|
def initialize config, stats, log
|
14
14
|
log.trace event: 'output', kind: 'elasticsearch', config: config
|
15
|
+
default_index = config.delete(:default_index) || 'logs-anschel'
|
15
16
|
qsize = config.delete(:queue_size) || 2000
|
16
17
|
bsize = config.delete(:bulk_size) || 500
|
17
18
|
timeout = config.delete(:bulk_timeout) || 0.5
|
@@ -37,7 +38,15 @@ module Anschel
|
|
37
38
|
next if events.empty?
|
38
39
|
|
39
40
|
body = events.map do |e|
|
40
|
-
index = e.delete(:_index)
|
41
|
+
index = e.delete(:_index)
|
42
|
+
if index.nil?
|
43
|
+
log.error \
|
44
|
+
event: 'elasticsearch-output-error',
|
45
|
+
reason: 'event was not indexed',
|
46
|
+
remediation: "sending to default index '#{default_index}'",
|
47
|
+
raw_event: e
|
48
|
+
index = default_index
|
49
|
+
end
|
41
50
|
{ index: { _index: index, _type: e[:type], data: e } }
|
42
51
|
end
|
43
52
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: anschel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sean Clemmer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-09-
|
11
|
+
date: 2015-09-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|