anschel 0.6.1 → 0.6.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/anschel/filter/convert.rb +1 -2
- data/lib/anschel/filter/gsub.rb +1 -1
- data/lib/anschel/filter/index.rb +1 -1
- data/lib/anschel/filter/parse.rb +1 -1
- data/lib/anschel/filter/scan.rb +1 -1
- data/lib/anschel/filter/stamp.rb +5 -3
- data/lib/anschel/filter.rb +3 -2
- data/lib/anschel/input/kafka.rb +0 -1
- data/lib/anschel/input/rabbitmq.rb +0 -2
- data/lib/anschel/input.rb +20 -7
- data/lib/anschel/main.rb +51 -24
- data/lib/anschel/output/device.rb +0 -2
- data/lib/anschel/output/elasticsearch.rb +1 -2
- data/lib/anschel/output.rb +6 -4
- data/lib/anschel/stats.rb +1 -0
- data/lib/anschel/store.rb +27 -0
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 70986c1daf98761ecae7d688a9cac9dca77d5245
|
4
|
+
data.tar.gz: 423be3728f6d0b12816965df20cb48c3ccc77198
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 82ff4b34d365f33a3a39cc3bac477cac4c8c449ebfadb8317a15a92372bc6af858ee2715afd6329d976fa70e2f07e7c4cff14f0ccf6a14a61cefb30d1a748052
|
7
|
+
data.tar.gz: db692abab264dc1ad1ccee814560071e398d81430717e2b61d76469e83754128bf0277820eadc55231e131719271d79851aa4bff51902bf7372ca40f08c1c2a2
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.6.
|
1
|
+
0.6.2
|
@@ -7,7 +7,6 @@
|
|
7
7
|
module Anschel
|
8
8
|
class Filter
|
9
9
|
def convert conf, log
|
10
|
-
log.debug event: 'compile-filter', filter: 'convert', conf: conf
|
11
10
|
field = conf.delete :field
|
12
11
|
type = conf.delete :type
|
13
12
|
|
@@ -22,7 +21,7 @@ module Anschel
|
|
22
21
|
'string' => :to_s
|
23
22
|
}
|
24
23
|
|
25
|
-
log.
|
24
|
+
log.trace event: 'filter-compiled', kind: 'convert', \
|
26
25
|
field: field, type: type
|
27
26
|
|
28
27
|
lambda do |event|
|
data/lib/anschel/filter/gsub.rb
CHANGED
data/lib/anschel/filter/index.rb
CHANGED
data/lib/anschel/filter/parse.rb
CHANGED
data/lib/anschel/filter/scan.rb
CHANGED
data/lib/anschel/filter/stamp.rb
CHANGED
@@ -2,7 +2,8 @@
|
|
2
2
|
# "stamp": {
|
3
3
|
# "utc?": false,
|
4
4
|
# "field": "timestamp",
|
5
|
-
# "pattern": [ "YYYY-MM-dd HH:mm:ss.SSS" ]
|
5
|
+
# "pattern": [ "YYYY-MM-dd HH:mm:ss.SSS" ],
|
6
|
+
# "target": "@timestamp"
|
6
7
|
# }
|
7
8
|
# }
|
8
9
|
module Anschel
|
@@ -31,8 +32,9 @@ module Anschel
|
|
31
32
|
|
32
33
|
offset_s = utc ? Time.zone_offset(Time.now.zone).to_f : 0.0
|
33
34
|
|
34
|
-
|
35
|
-
|
35
|
+
|
36
|
+
log.trace event: 'filter-compiled', kind: 'stamp', \
|
37
|
+
utc?: utc, field: field, pattern: pattern, target: target
|
36
38
|
|
37
39
|
lambda do |event|
|
38
40
|
return event unless event.has_key? field
|
data/lib/anschel/filter.rb
CHANGED
@@ -12,7 +12,8 @@ module Anschel
|
|
12
12
|
attr_reader :filters
|
13
13
|
|
14
14
|
def initialize config, stats, log
|
15
|
-
log.
|
15
|
+
log.info event: 'filter-loading'
|
16
|
+
log.debug event: 'filter-config', config: config
|
16
17
|
config ||= {} # Allow for nil config
|
17
18
|
@filters = Hash.new { |h,k| h[k] = [] }
|
18
19
|
config.each do |event_type, filter_defns|
|
@@ -22,7 +23,7 @@ module Anschel
|
|
22
23
|
@filters[event_type] << self.send(filter_type, filter_conf, log)
|
23
24
|
end
|
24
25
|
end
|
25
|
-
log.info event: 'filter-loaded'
|
26
|
+
log.info event: 'filter-fully-loaded'
|
26
27
|
end
|
27
28
|
|
28
29
|
|
data/lib/anschel/input/kafka.rb
CHANGED
data/lib/anschel/input.rb
CHANGED
@@ -4,34 +4,47 @@ require_relative 'input/rabbitmq'
|
|
4
4
|
|
5
5
|
module Anschel
|
6
6
|
class Input
|
7
|
-
def initialize config, qsize, stats, log
|
8
|
-
|
9
|
-
|
10
|
-
end
|
7
|
+
def initialize config, qsize, stats, log, leftovers=[]
|
8
|
+
log.info event: 'output-loading'
|
9
|
+
log.debug event: 'output-config', config: config, qsize: qsize
|
11
10
|
|
12
11
|
@queue = SizedQueue.new qsize || 2000
|
13
12
|
|
13
|
+
Thread.new do
|
14
|
+
leftovers ||= []
|
15
|
+
log.trace event: 'input-leftovers', leftovers_size: leftovers.size
|
16
|
+
leftovers.each { |l| @queue << l }
|
17
|
+
end
|
18
|
+
|
14
19
|
@inputs = []
|
15
20
|
|
16
21
|
config.each do |input|
|
17
22
|
case input.delete(:kind)
|
18
23
|
when 'kafka'
|
19
24
|
@inputs << Input::Kafka.new(@queue, input, stats, log)
|
25
|
+
log.trace event: 'input-loaded', kind: 'kafka'
|
20
26
|
when 'rabbitmq'
|
21
27
|
@inputs << Input::RabbitMQ.new(@queue, input, stats, log)
|
28
|
+
log.trace event: 'input-loaded', kind: 'rabbitmq'
|
22
29
|
else
|
23
30
|
raise 'Uknown input type'
|
24
31
|
end
|
25
32
|
end
|
26
33
|
|
27
|
-
log.info event: 'input-loaded'
|
34
|
+
log.info event: 'input-fully-loaded'
|
28
35
|
end
|
29
36
|
|
30
37
|
|
31
38
|
def stop
|
32
|
-
return if @
|
39
|
+
return @leftovers if defined? @leftovers
|
33
40
|
@inputs.map &:stop
|
34
|
-
@
|
41
|
+
@leftovers = []
|
42
|
+
@leftovers << shift until @queue.empty?
|
43
|
+
end
|
44
|
+
|
45
|
+
|
46
|
+
def leftovers
|
47
|
+
return @leftovers if defined? @leftovers
|
35
48
|
end
|
36
49
|
|
37
50
|
|
data/lib/anschel/main.rb
CHANGED
@@ -5,6 +5,7 @@ require 'jrjackson'
|
|
5
5
|
|
6
6
|
require_relative 'mjolnir'
|
7
7
|
require_relative 'metadata'
|
8
|
+
require_relative 'store'
|
8
9
|
require_relative 'stats'
|
9
10
|
require_relative 'input'
|
10
11
|
require_relative 'filter'
|
@@ -46,44 +47,70 @@ module Anschel
|
|
46
47
|
options: options.to_hash,
|
47
48
|
num_cpus: num_cpus
|
48
49
|
|
49
|
-
|
50
|
-
|
50
|
+
input, filter, output, store, stats, ts = \
|
51
|
+
nil, nil, nil, nil, nil, nil, nil
|
51
52
|
|
52
|
-
|
53
|
+
begin
|
54
|
+
config = JrJackson::Json.load \
|
55
|
+
File.read(options.config), symbolize_keys: true
|
53
56
|
|
54
|
-
|
57
|
+
setup_log4j config[:log4j]
|
55
58
|
|
56
|
-
|
59
|
+
store = Store.new config[:store], log
|
57
60
|
|
58
|
-
|
61
|
+
stats = Stats.new log, options.stats_interval
|
59
62
|
|
60
|
-
|
63
|
+
filter = Filter.new config[:filter], stats, log
|
61
64
|
|
62
|
-
|
63
|
-
input.stop
|
64
|
-
output.stop
|
65
|
-
log.info event: 'goodbye', version: VERSION
|
66
|
-
exit
|
67
|
-
end
|
65
|
+
output = Output.new config[:output], stats, log
|
68
66
|
|
67
|
+
input = Input.new \
|
68
|
+
config[:input], config[:queue_size], stats, log, store[:input]
|
69
69
|
|
70
|
-
|
71
|
-
|
70
|
+
stats.create 'event'
|
71
|
+
stats.get 'event'
|
72
72
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
73
|
+
ts = num_cpus.times.map do
|
74
|
+
Thread.new do
|
75
|
+
loop do
|
76
|
+
event = JrJackson::Json.load \
|
77
|
+
input.shift, symbolize_keys: true
|
78
|
+
output.push filter.apply(event)
|
79
|
+
stats.inc 'event'
|
80
|
+
end
|
80
81
|
end
|
81
82
|
end
|
83
|
+
|
84
|
+
rescue => e
|
85
|
+
log.fatal \
|
86
|
+
event: 'exception',
|
87
|
+
exception: e.inspect,
|
88
|
+
class: e.class,
|
89
|
+
message: e.message,
|
90
|
+
backtrace: e.backtrace.join("\n")
|
91
|
+
bye output, input, store, log, :error
|
92
|
+
exit 2
|
93
|
+
end
|
94
|
+
|
95
|
+
log.info event: 'all-systems-clear'
|
96
|
+
|
97
|
+
trap('SIGINT') do
|
98
|
+
bye output, input, store, log
|
99
|
+
exit
|
82
100
|
end
|
83
101
|
|
84
|
-
log.info event: 'fully-loaded'
|
85
102
|
ts.map &:join
|
86
|
-
|
103
|
+
end
|
104
|
+
|
105
|
+
|
106
|
+
private
|
107
|
+
def bye output, input, store, log, level=:info
|
108
|
+
if input && output
|
109
|
+
output.stop ; input.stop
|
110
|
+
store[:input] = input.leftovers
|
111
|
+
log.debug event: 'flush'
|
112
|
+
end
|
113
|
+
log.send level, event: 'goodbye', version: VERSION
|
87
114
|
end
|
88
115
|
|
89
116
|
end
|
@@ -7,8 +7,6 @@ module Anschel
|
|
7
7
|
class Output
|
8
8
|
class Device < Base
|
9
9
|
def initialize config, stats, log
|
10
|
-
log.trace event: 'output', kind: 'device', config: config
|
11
|
-
|
12
10
|
path = config.delete(:path) || '/dev/stdout'
|
13
11
|
qsize = config.delete(:queue_size) || 2000
|
14
12
|
@queue = SizedQueue.new qsize
|
@@ -11,8 +11,7 @@ module Anschel
|
|
11
11
|
class Output
|
12
12
|
class Elasticsearch < Base
|
13
13
|
def initialize config, stats, log
|
14
|
-
|
15
|
-
default_index = config.delete(:default_index) || 'logs-anschel'
|
14
|
+
default_index = config.delete(:default_index) || '.anschel'
|
16
15
|
qsize = config.delete(:queue_size) || 2000
|
17
16
|
bsize = config.delete(:bulk_size) || 500
|
18
17
|
timeout = config.delete(:bulk_timeout) || 0.5
|
data/lib/anschel/output.rb
CHANGED
@@ -5,9 +5,8 @@ require_relative 'output/device'
|
|
5
5
|
module Anschel
|
6
6
|
class Output
|
7
7
|
def initialize config, stats, log
|
8
|
-
|
9
|
-
|
10
|
-
end
|
8
|
+
log.info event: 'output-loading'
|
9
|
+
log.debug event: 'output-config', config: config
|
11
10
|
|
12
11
|
@outputs = []
|
13
12
|
|
@@ -15,14 +14,16 @@ module Anschel
|
|
15
14
|
case output.delete(:kind)
|
16
15
|
when 'device'
|
17
16
|
@outputs << Output::Device.new(output, stats, log)
|
17
|
+
log.trace event: 'output-loaded', kind: 'device'
|
18
18
|
when 'elasticsearch'
|
19
19
|
@outputs << Output::Elasticsearch.new(output, stats, log)
|
20
|
+
log.trace event: 'output-loaded', kind: 'elasticsearch'
|
20
21
|
else
|
21
22
|
raise 'Unkown output type'
|
22
23
|
end
|
23
24
|
end
|
24
25
|
|
25
|
-
log.info event: 'output-loaded'
|
26
|
+
log.info event: 'output-fully-loaded'
|
26
27
|
end
|
27
28
|
|
28
29
|
|
@@ -32,6 +33,7 @@ module Anschel
|
|
32
33
|
@stopped = true
|
33
34
|
end
|
34
35
|
|
36
|
+
|
35
37
|
def push event
|
36
38
|
@outputs.each do |output|
|
37
39
|
output.push event
|
data/lib/anschel/stats.rb
CHANGED
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'jrjackson'
|
2
|
+
|
3
|
+
|
4
|
+
module Anschel
|
5
|
+
class Store
|
6
|
+
def initialize path, log
|
7
|
+
@path = path || '/tmp/anschel.db'
|
8
|
+
@hash = nil
|
9
|
+
if File.exist? @path
|
10
|
+
@hash = JrJackson::Json.load \
|
11
|
+
File.read(@path), symbolize_keys: true
|
12
|
+
end
|
13
|
+
@hash ||= {}
|
14
|
+
log.info event: 'stats-loaded'
|
15
|
+
end
|
16
|
+
|
17
|
+
def [] k ; @hash[k] end
|
18
|
+
|
19
|
+
def []= k,v ; @hash[k] = v ; save end
|
20
|
+
|
21
|
+
def save
|
22
|
+
File.open(@path, 'w') do |f|
|
23
|
+
f.puts JrJackson::Json.dump(@hash)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: anschel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sean Clemmer
|
@@ -139,6 +139,7 @@ files:
|
|
139
139
|
- lib/anschel/output/device.rb
|
140
140
|
- lib/anschel/output/elasticsearch.rb
|
141
141
|
- lib/anschel/stats.rb
|
142
|
+
- lib/anschel/store.rb
|
142
143
|
homepage: https://github.com/sczizzo/anschel
|
143
144
|
licenses:
|
144
145
|
- ISC
|