anschel 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Readme.md +19 -14
- data/VERSION +1 -1
- data/lib/anschel/filter/index.rb +22 -5
- data/lib/anschel/filter/parse.rb +9 -0
- data/lib/anschel/filter/stamp.rb +12 -3
- data/lib/anschel/output/device.rb +1 -1
- data/lib/anschel/output/elasticsearch.rb +10 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8f81ae5fdcf71f580680520067255911285629aa
|
4
|
+
data.tar.gz: 68ab5e077fa13d9d4e142fe798f222912dce5f6a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0179da5dc7e53fc18e3c490f4e49f521e210184a0df80a5d834052237b2f476f4b99fbcad40bd390f1f318a9af9559f9609117098d8e7cf02d35829a560d2aab
|
7
|
+
data.tar.gz: 95309336805570a5e7cd50dfa1d47e43677caa3a72f78ea1a8e8b7fadc68146c26e00feb73706b0a2620868f73777af78b94c4b6368eeb8f4bc1c8e30c74d88f
|
data/Readme.md
CHANGED
@@ -54,10 +54,13 @@ It's kinda like a JSON version of the Logstash config language:
|
|
54
54
|
"pattern": "[%d] %p %m (%c)%n"
|
55
55
|
},
|
56
56
|
|
57
|
-
//
|
58
|
-
|
59
|
-
|
60
|
-
|
57
|
+
// Specify any number of inputs
|
58
|
+
"input": [
|
59
|
+
|
60
|
+
// Kafka is the primary input; see the `jruby-kafka` homepage for
|
61
|
+
// more details: https://github.com/joekiller/jruby-kafka
|
62
|
+
{
|
63
|
+
"kind": "kafka",
|
61
64
|
"queue_size": 2000,
|
62
65
|
"zk_connect": "localhost:2181",
|
63
66
|
"zk_connect_timeout": 6000,
|
@@ -81,12 +84,15 @@ It's kinda like a JSON version of the Logstash config language:
|
|
81
84
|
"consumer_timeout_ms": -1,
|
82
85
|
"consumer_restart_sleep_ms": 0
|
83
86
|
}
|
84
|
-
|
87
|
+
],
|
88
|
+
|
89
|
+
// Specify any number of outputs
|
90
|
+
"output": [
|
85
91
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
92
|
+
// Elasticsearch is the primary output; see the `elasticsearch-ruby`
|
93
|
+
// homepage for more: https://github.com/elastic/elasticsearch-ruby
|
94
|
+
{
|
95
|
+
"kind": "elasticsearch",
|
90
96
|
"queue_size": 2000,
|
91
97
|
"bulk_size": 200,
|
92
98
|
"hosts": [ "localhost:9200" ],
|
@@ -95,7 +101,7 @@ It's kinda like a JSON version of the Logstash config language:
|
|
95
101
|
"reload_on_failure": true,
|
96
102
|
"sniffer_timeout": 5
|
97
103
|
}
|
98
|
-
|
104
|
+
],
|
99
105
|
|
100
106
|
// Just like Logstash, Anschel has a notion of filters
|
101
107
|
"filter": {
|
@@ -148,7 +154,6 @@ You might deploy Anschel with Upstart. Here's a minimal config:
|
|
148
154
|
|
149
155
|
start on startup
|
150
156
|
stop on shutdown
|
151
|
-
respawn
|
152
157
|
|
153
158
|
exec java -jar anschel-x.y.z.jar \
|
154
159
|
--config /etc/anschel.json --log /var/log/anschel.log
|
@@ -161,7 +166,7 @@ You might deploy Anschel with Upstart. Here's a minimal config:
|
|
161
166
|
|
162
167
|
_In development_
|
163
168
|
|
164
|
-
-
|
165
|
-
- Support for file (device) output (v0.4)
|
169
|
+
- Allow multiple input and output configurations (v0.6)
|
166
170
|
- Support for RabbitMQ input (v0.5)
|
167
|
-
-
|
171
|
+
- Support for file (device) output (v0.4)
|
172
|
+
- Intial implementation of the Kafka-to-Elasticsearch pipeline
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.6.
|
1
|
+
0.6.1
|
data/lib/anschel/filter/index.rb
CHANGED
@@ -12,7 +12,9 @@ module Anschel
|
|
12
12
|
stamp = conf.delete(:stamp) || '@timestamp'
|
13
13
|
prefix = conf.delete(:prefix) || 'logs-%{type}-'
|
14
14
|
suffix = conf.delete(:suffix) || '%Y.%m.%d'
|
15
|
-
format = conf.delete(:format) || "yyyy-MM-dd'T'HH:mm:ss.
|
15
|
+
format = conf.delete(:format) || "yyyy-MM-dd'T'HH:mm:ss.SSSZZ" # ISO8601
|
16
|
+
|
17
|
+
error_tag = conf.has_key?(:error_tag) ? conf[:error_tag] : 'index-error'
|
16
18
|
|
17
19
|
stamp = stamp.to_sym
|
18
20
|
|
@@ -26,11 +28,26 @@ module Anschel
|
|
26
28
|
|
27
29
|
lambda do |event|
|
28
30
|
return event unless event.has_key? stamp
|
29
|
-
millis = joda.parseMillis event[stamp]
|
30
31
|
idx_prefix = prefix % event
|
31
|
-
|
32
|
-
|
33
|
-
|
32
|
+
begin
|
33
|
+
millis = joda.parseMillis event[stamp]
|
34
|
+
idx_suffix = Time.at(0.001 * millis).strftime(suffix)
|
35
|
+
event[:_index] = idx_prefix + idx_suffix
|
36
|
+
filtered event, conf
|
37
|
+
rescue java.lang.IllegalArgumentException => e
|
38
|
+
event[:_index] = idx_prefix + Time.now.strftime(suffix)
|
39
|
+
log.warn \
|
40
|
+
event: 'filter-index-warning',
|
41
|
+
reason: 'could not parse event',
|
42
|
+
remediation: 'added bogus index',
|
43
|
+
remediation: "sending to best-guess index '#{event[:_index]}'",
|
44
|
+
raw_event: event
|
45
|
+
if error_tag
|
46
|
+
event[:tags] ||= []
|
47
|
+
event[:tags] << error_tag
|
48
|
+
end
|
49
|
+
filtered event, conf
|
50
|
+
end
|
34
51
|
end
|
35
52
|
end
|
36
53
|
end
|
data/lib/anschel/filter/parse.rb
CHANGED
@@ -22,6 +22,15 @@ module Anschel
|
|
22
22
|
lambda do |event|
|
23
23
|
return event unless event.has_key? field
|
24
24
|
mdata = pattern.match event[field]
|
25
|
+
if mdata.nil?
|
26
|
+
log.error \
|
27
|
+
event: 'parse-filter-error',
|
28
|
+
reason: 'regexp did not match',
|
29
|
+
field: field,
|
30
|
+
pattern: pattern,
|
31
|
+
raw_event: event
|
32
|
+
return filtered(event, conf)
|
33
|
+
end
|
25
34
|
mdata.names.each do |group|
|
26
35
|
event[group.to_sym] = mdata[group]
|
27
36
|
end
|
data/lib/anschel/filter/stamp.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# {
|
2
2
|
# "stamp": {
|
3
|
+
# "utc?": false,
|
3
4
|
# "field": "timestamp",
|
4
5
|
# "pattern": [ "YYYY-MM-dd HH:mm:ss.SSS" ]
|
5
6
|
# }
|
@@ -7,6 +8,7 @@
|
|
7
8
|
module Anschel
|
8
9
|
class Filter
|
9
10
|
def stamp conf, log
|
11
|
+
utc = conf.delete :utc?
|
10
12
|
field = conf.delete :field
|
11
13
|
pattern = conf.delete :pattern
|
12
14
|
target = conf.delete :target
|
@@ -27,6 +29,7 @@ module Anschel
|
|
27
29
|
joda = joda.withOffsetParsed
|
28
30
|
end
|
29
31
|
|
32
|
+
offset_s = utc ? Time.zone_offset(Time.now.zone).to_f : 0.0
|
30
33
|
|
31
34
|
log.debug event: 'compiled-filter', filter: 'stamp', \
|
32
35
|
field: field, pattern: pattern, target: target, error_tag: error_tag
|
@@ -34,19 +37,25 @@ module Anschel
|
|
34
37
|
lambda do |event|
|
35
38
|
return event unless event.has_key? field
|
36
39
|
parsers.each do |joda|
|
37
|
-
millis = joda.parseMillis event[field]
|
38
40
|
begin
|
39
|
-
|
41
|
+
millis = joda.parseMillis event[field]
|
42
|
+
event[target] = Time.at(0.001 * millis + offset_s).iso8601(3)
|
40
43
|
return filtered(event, conf)
|
41
44
|
rescue
|
42
45
|
end
|
43
46
|
end
|
44
47
|
|
48
|
+
log.warn \
|
49
|
+
event: 'stamp-filter-warning',
|
50
|
+
reason: 'could not parse event',
|
51
|
+
remediation: 'using current time for stamp',
|
52
|
+
raw_event: event
|
45
53
|
if error_tag
|
46
54
|
event[:tags] ||= []
|
47
55
|
event[:tags] << error_tag
|
48
56
|
end
|
49
|
-
event
|
57
|
+
event[target] = Time.now.utc.iso8601(3)
|
58
|
+
filtered event, conf
|
50
59
|
end
|
51
60
|
end
|
52
61
|
end
|
@@ -9,7 +9,7 @@ module Anschel
|
|
9
9
|
def initialize config, stats, log
|
10
10
|
log.trace event: 'output', kind: 'device', config: config
|
11
11
|
|
12
|
-
path = config.delete(:path)
|
12
|
+
path = config.delete(:path) || '/dev/stdout'
|
13
13
|
qsize = config.delete(:queue_size) || 2000
|
14
14
|
@queue = SizedQueue.new qsize
|
15
15
|
|
@@ -12,6 +12,7 @@ module Anschel
|
|
12
12
|
class Elasticsearch < Base
|
13
13
|
def initialize config, stats, log
|
14
14
|
log.trace event: 'output', kind: 'elasticsearch', config: config
|
15
|
+
default_index = config.delete(:default_index) || 'logs-anschel'
|
15
16
|
qsize = config.delete(:queue_size) || 2000
|
16
17
|
bsize = config.delete(:bulk_size) || 500
|
17
18
|
timeout = config.delete(:bulk_timeout) || 0.5
|
@@ -37,7 +38,15 @@ module Anschel
|
|
37
38
|
next if events.empty?
|
38
39
|
|
39
40
|
body = events.map do |e|
|
40
|
-
index = e.delete(:_index)
|
41
|
+
index = e.delete(:_index)
|
42
|
+
if index.nil?
|
43
|
+
log.error \
|
44
|
+
event: 'elasticsearch-output-error',
|
45
|
+
reason: 'event was not indexed',
|
46
|
+
remediation: "sending to default index '#{default_index}'",
|
47
|
+
raw_event: e
|
48
|
+
index = default_index
|
49
|
+
end
|
41
50
|
{ index: { _index: index, _type: e[:type], data: e } }
|
42
51
|
end
|
43
52
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: anschel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sean Clemmer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-09-
|
11
|
+
date: 2015-09-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|