metrics-capacitor-engine 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/metrics-capacitor.rb +2 -0
- data/lib/metrics-capacitor/engine.rb +138 -0
- data/lib/metrics-capacitor/processor/aggregator.rb +19 -0
- data/lib/metrics-capacitor/processor/core.rb +63 -0
- data/lib/metrics-capacitor/processor/listener.rb +19 -0
- data/lib/metrics-capacitor/processor/scrubber.rb +30 -0
- data/lib/metrics-capacitor/processor/writer.rb +67 -0
- data/lib/metrics-capacitor/sidekiq.rb +49 -0
- metadata +111 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2f466bef259df899e75151d5904a203924869aec
|
4
|
+
data.tar.gz: 986d0e7f768fa95088884dda0da44942eb2febbe
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: addf8bbb991ba5dcad978876d70f9ae69a5caa2f50a6d257e7cd5d7ba0b5e966a56a6f4522183bcb2bf80a6d8e98af1e51113255a0db4dadc108b46039b3b16a
|
7
|
+
data.tar.gz: 789b12ca634e3db7f65a34412f517d2b6b283c4c963ff7a5d9e9fd3aad41c054c63a04fc567472e6c9a2c63769341e2a093f50d30b47da77ab2c2ab7296bac1e
|
@@ -0,0 +1,138 @@
|
|
1
|
+
require 'redis'
|
2
|
+
require 'syslog'
|
3
|
+
require 'syslog/logger'
|
4
|
+
require 'metrics-capacitor/config'
|
5
|
+
|
6
|
+
require_relative 'processor/core'
|
7
|
+
require_relative 'processor/writer'
|
8
|
+
require_relative 'processor/aggregator'
|
9
|
+
require_relative 'processor/listener'
|
10
|
+
|
11
|
+
module MetricsCapacitor
|
12
|
+
class Engine
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
$0 = 'metrics-capacitor (engine)'
|
16
|
+
Config.load!
|
17
|
+
@exit_flag = false
|
18
|
+
@pids = []
|
19
|
+
@pids_kiq = []
|
20
|
+
%w(TERM INT).each do |sig|
|
21
|
+
Signal.trap(sig) do
|
22
|
+
@pids_kiq.each { |pid| Process.kill('INT', pid) rescue true }
|
23
|
+
@pids.each { |pid| Process.kill(sig, pid) rescue true }
|
24
|
+
Process.waitall
|
25
|
+
terminate_loggers
|
26
|
+
end
|
27
|
+
end
|
28
|
+
init_logger
|
29
|
+
log :info, "Engine warmed-up :-)"
|
30
|
+
end
|
31
|
+
|
32
|
+
def fork_processor(args = {})
|
33
|
+
log :debug, "Spawning #{args[:name]}"
|
34
|
+
args[:proc_num] ||= 1
|
35
|
+
args[:exit_on] ||= %w{INT TERM}
|
36
|
+
args[:proc_num].times do |num|
|
37
|
+
@logpipe["#{args[:name]}_#{num}".to_sym], logpipe = IO.pipe
|
38
|
+
@pids << Process.fork do
|
39
|
+
$0 = "metrics-capacitor (#{args[:name]})" if args[:name]
|
40
|
+
@logpipe["#{args[:name]}_#{num}".to_sym].close
|
41
|
+
remove_instance_variable(:@logpipe)
|
42
|
+
p = Kernel.const_get("MetricsCapacitor::Processor::#{args[:name].capitalize}").new(logpipe)
|
43
|
+
args[:exit_on].each { |sig| Signal.trap(sig) { p.shutdown! } }
|
44
|
+
p.start!
|
45
|
+
end
|
46
|
+
log :debug, "#{args[:name].capitalize} spawned as PID #{@pids.last.to_s}"
|
47
|
+
logpipe.close
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def fork_scrubber
|
52
|
+
log :debug, "Spawning scrubbers"
|
53
|
+
Config.scrubber[:processes].times do |num|
|
54
|
+
@logpipe["scrubber_#{num}".to_sym], logpipe = IO.pipe
|
55
|
+
@pids_kiq << Process.fork do
|
56
|
+
@logpipe["scrubber_#{num}".to_sym].close
|
57
|
+
remove_instance_variable(:@logpipe)
|
58
|
+
require_relative 'sidekiq'
|
59
|
+
Sidekiq.configure_server do |config|
|
60
|
+
Sidekiq::Logging.logger = ::Logger.new(logpipe)
|
61
|
+
Sidekiq::Logging.logger.level = log_level
|
62
|
+
Sidekiq::Logging.logger.progname = "scrubber"
|
63
|
+
Sidekiq::Logging.logger.formatter = proc { |severity, _, progname, msg| "#{progname}##{Process.pid}|||#{severity}|||#{msg}\n" }
|
64
|
+
config.redis = { url: Config.redis[:url] }
|
65
|
+
end
|
66
|
+
Sidekiq.configure_client do |config|
|
67
|
+
config.redis = { url: Config.redis[:url] }
|
68
|
+
end
|
69
|
+
$TESTING = 0
|
70
|
+
kiq = Sidekiq::CLI.instance
|
71
|
+
kiq.parse(['-c', Config.scrubber[:threads].to_s, '-r', File.expand_path('..', __FILE__)+'/processor/scrubber.rb', '-q', 'scrubber'])
|
72
|
+
kiq.run
|
73
|
+
end
|
74
|
+
logpipe.close
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def log(severity = :info, msg)
|
79
|
+
s = Kernel.const_get("Logger::#{severity.to_s.upcase}")
|
80
|
+
@logger_semaphore.synchronize do
|
81
|
+
@logger.log s, msg, 'engine#' + Process.pid.to_s
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def log_level
|
86
|
+
Config.debug ? ::Logger::DEBUG : ::Logger::INFO
|
87
|
+
end
|
88
|
+
|
89
|
+
def init_logger
|
90
|
+
@logpipe = {}
|
91
|
+
@logger = ::Logger.new(STDOUT)
|
92
|
+
@logger.level = log_level
|
93
|
+
@logger.formatter = proc { |severity, datetime, progname, msg| [datetime.to_s, progname, severity, "#{msg}\n"].join(" ") }
|
94
|
+
@logger_threads = []
|
95
|
+
@logger_semaphore = Mutex.new
|
96
|
+
end
|
97
|
+
|
98
|
+
def spawn_logger_threads
|
99
|
+
@logpipe.each do |name, pipe|
|
100
|
+
@logger_threads << Thread.new do
|
101
|
+
Thread.current[:name] = "logger-#{name}"
|
102
|
+
while ! pipe.eof?
|
103
|
+
msg = pipe.gets
|
104
|
+
(progname,severity,message) = msg.split('|||')
|
105
|
+
@logger_semaphore.synchronize do
|
106
|
+
@logger.log Kernel.const_get("Logger::#{severity}"), message.chomp, progname
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def terminate_loggers
|
114
|
+
@logger_threads.each(&:join)
|
115
|
+
end
|
116
|
+
|
117
|
+
def run!
|
118
|
+
log :info, 'Engine is starting up'
|
119
|
+
fork_scrubber
|
120
|
+
fork_processor name: 'writer', proc_num: Config.writer[:processes]
|
121
|
+
fork_processor name: 'aggregator'
|
122
|
+
fork_processor name: 'listener'
|
123
|
+
spawn_logger_threads
|
124
|
+
log :info, 'Engine has started :-)'
|
125
|
+
# TODO: unix socket for control and status reporting ;)
|
126
|
+
begin
|
127
|
+
::Process.waitall
|
128
|
+
log :warn, 'Terminating!'
|
129
|
+
rescue Interrupt
|
130
|
+
retry
|
131
|
+
end
|
132
|
+
log :info, 'Terminating loggers'
|
133
|
+
terminate_loggers
|
134
|
+
log :warn, 'Engine is shutting down!'
|
135
|
+
end
|
136
|
+
|
137
|
+
end
|
138
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'metrics-capacitor/model'
|
2
|
+
|
3
|
+
module MetricsCapacitor
|
4
|
+
module Processor
|
5
|
+
class Core
|
6
|
+
include MetricsCapacitor::Model
|
7
|
+
|
8
|
+
def initialize(logpipe)
|
9
|
+
Config.load!
|
10
|
+
@_logger = ::Logger.new(logpipe)
|
11
|
+
@_name = self.class.to_s.split('::').last.downcase
|
12
|
+
@_logger.progname = @_name
|
13
|
+
@_logger.level = log_level
|
14
|
+
@_logger.formatter = proc { |severity, datetime, progname, msg| "#{progname}##{Process.pid}|||#{severity}|||#{msg}\n" }
|
15
|
+
logger.info "Initializing processor"
|
16
|
+
post_init
|
17
|
+
end
|
18
|
+
|
19
|
+
# implement in the processor Class
|
20
|
+
def post_init
|
21
|
+
end
|
22
|
+
|
23
|
+
# implement in the processor Class
|
24
|
+
def process
|
25
|
+
loop { logger.debug "alive"; sleep 10 }
|
26
|
+
end
|
27
|
+
|
28
|
+
# implement in the processor Class
|
29
|
+
def shutdown
|
30
|
+
exit 1
|
31
|
+
end
|
32
|
+
|
33
|
+
def logger
|
34
|
+
@_logger
|
35
|
+
end
|
36
|
+
|
37
|
+
def start!
|
38
|
+
logger.info "Starting processor"
|
39
|
+
begin
|
40
|
+
process
|
41
|
+
rescue StandardError => e
|
42
|
+
logger.fatal 'SHUTTING DOWN DUE TO AN EXCEPTION!'
|
43
|
+
logger.fatal [e.class, e.message].join(' -> ')
|
44
|
+
logger.fatal e.backtrace
|
45
|
+
logger.fatal 'Sending SIGINT to the engine'
|
46
|
+
Process.kill('INT', Process.ppid)
|
47
|
+
shutdown!
|
48
|
+
end
|
49
|
+
logger.info "Processor finished"
|
50
|
+
end
|
51
|
+
|
52
|
+
def shutdown!
|
53
|
+
shutdown
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
def log_level
|
58
|
+
Config.debug ? ::Logger::DEBUG : ::Logger::INFO
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'metrics-capacitor/config'
|
2
|
+
require 'metrics-capacitor/model'
|
3
|
+
|
4
|
+
module MetricsCapacitor
|
5
|
+
module Processor
|
6
|
+
class Scrubber
|
7
|
+
include Sidekiq::Worker
|
8
|
+
include MetricsCapacitor::Model
|
9
|
+
|
10
|
+
sidekiq_options retry: true
|
11
|
+
sidekiq_options queue: :scrubber
|
12
|
+
sidekiq_options backtrace: true
|
13
|
+
|
14
|
+
REDIS = ConnectionPool.new(size: 2) { Redis.new(url: Config.redis[:url]) }
|
15
|
+
|
16
|
+
def perform *args
|
17
|
+
logger.debug 'Picking redis client from connection-pool'
|
18
|
+
REDIS.with do |r|
|
19
|
+
logger.debug 'Parsing metrics data'
|
20
|
+
metrics = Metrics.new args
|
21
|
+
logger.debug metrics.to_redis
|
22
|
+
logger.debug 'Sending data to writer'
|
23
|
+
r.rpush 'writer', metrics.to_redis
|
24
|
+
logger.debug 'Data sent'
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'elasticsearch'
|
2
|
+
|
3
|
+
module MetricsCapacitor
|
4
|
+
module Processor
|
5
|
+
class Writer < Core
|
6
|
+
|
7
|
+
def post_init
|
8
|
+
@elastic = Elasticsearch::Client.new(
|
9
|
+
url: Config.elasticsearch[:urls],
|
10
|
+
reload_connections: 100,
|
11
|
+
retry_on_failure: Config.elasticsearch[:retry],
|
12
|
+
sniffer_timeout: 5,
|
13
|
+
)
|
14
|
+
logger.debug 'Elastic connection set up'
|
15
|
+
|
16
|
+
@redis = Redis.new(url: Config.redis[:url])
|
17
|
+
logger.debug 'Redis connection set up'
|
18
|
+
|
19
|
+
logger.debug 'Setting ES index templates'
|
20
|
+
@elastic.indices.put_template name: 'metrics', body: INDEX_TEMPLATE
|
21
|
+
logger.debug 'ES templates set up'
|
22
|
+
|
23
|
+
@exit = false
|
24
|
+
end
|
25
|
+
|
26
|
+
def process
|
27
|
+
logger.debug 'Randomizing startup time'
|
28
|
+
sleep rand(Config.writer[:bulk_wait])
|
29
|
+
until @exit
|
30
|
+
logger.debug 'Gathering mertics bulk'
|
31
|
+
metric = nil
|
32
|
+
metrics = Metrics.new
|
33
|
+
indexing_result = nil
|
34
|
+
begin
|
35
|
+
while !@exit && metrics.length < Config.writer[:bulk_max] && ( metric = @redis.blpop('writer', timeout: Config.writer[:bulk_wait]) )
|
36
|
+
metrics << Metric.new(metric[1])
|
37
|
+
metric = nil
|
38
|
+
end
|
39
|
+
rescue Redis::CannotConnectError, Redis::TimeoutError => e
|
40
|
+
logger.error "Can't connect to redis: #{e.message}"
|
41
|
+
sleep 1
|
42
|
+
retry
|
43
|
+
end
|
44
|
+
unless metrics.empty?
|
45
|
+
logger.debug "Writing #{metrics.length} metrics"
|
46
|
+
logger.debug metrics.to_elastic.to_json
|
47
|
+
indexing_result = @elastic.bulk(index: Time.now.strftime(Config.elasticsearch[:index]), type: Config.writer[:doc_type], body: metrics.to_elastic)
|
48
|
+
if indexing_result['errors']
|
49
|
+
logger.error 'Failed to write metrics!'
|
50
|
+
logger.error indexing_result['items'].to_json
|
51
|
+
else
|
52
|
+
logger.info "Written #{metrics.length} metrics, took #{indexing_result['took']}ms"
|
53
|
+
end
|
54
|
+
else
|
55
|
+
logger.warn 'No metrics to write :-('
|
56
|
+
end
|
57
|
+
metrics = nil
|
58
|
+
indexing_result = nil
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def shutdown
|
63
|
+
@exit = true
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'sidekiq'
|
2
|
+
require 'sidekiq/cli'
|
3
|
+
require 'sidekiq/logging'
|
4
|
+
|
5
|
+
module Sidekiq
|
6
|
+
class CLI
|
7
|
+
|
8
|
+
PROCTITLES[0] = proc { 'metrics-capacitor'.freeze }
|
9
|
+
PROCTITLES[1] = proc { '(scrubber)'.freeze }
|
10
|
+
|
11
|
+
def run
|
12
|
+
@code = nil
|
13
|
+
|
14
|
+
boot_system
|
15
|
+
|
16
|
+
self_read, self_write = IO.pipe
|
17
|
+
|
18
|
+
%w(INT USR1 USR2 TTIN).each do |sig|
|
19
|
+
begin
|
20
|
+
trap sig do
|
21
|
+
self_write.puts(sig)
|
22
|
+
end
|
23
|
+
rescue ArgumentError
|
24
|
+
puts "Signal #{sig} not supported"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
ver = Sidekiq.redis_info['redis_version']
|
29
|
+
raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
|
30
|
+
fire_event(:startup)
|
31
|
+
logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
|
32
|
+
logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
|
33
|
+
require 'sidekiq/launcher'
|
34
|
+
@launcher = Sidekiq::Launcher.new(options)
|
35
|
+
begin
|
36
|
+
launcher.run
|
37
|
+
while readable_io = IO.select([self_read])
|
38
|
+
signal = readable_io.first[0].gets.strip
|
39
|
+
handle_signal(signal)
|
40
|
+
end
|
41
|
+
rescue Interrupt
|
42
|
+
logger.info 'Shutting down'
|
43
|
+
launcher.stop
|
44
|
+
logger.info "Bye!"
|
45
|
+
exit(0)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
metadata
ADDED
@@ -0,0 +1,111 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: metrics-capacitor-engine
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Radek 'blufor' Slavicinsky
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-05-27 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: metrics-capacitor
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
- - '>='
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 0.0.1
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- - '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 0.0.1
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: sidekiq
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ~>
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '4.1'
|
40
|
+
- - '>='
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: 4.1.2
|
43
|
+
type: :runtime
|
44
|
+
prerelease: false
|
45
|
+
version_requirements: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - ~>
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: '4.1'
|
50
|
+
- - '>='
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 4.1.2
|
53
|
+
- !ruby/object:Gem::Dependency
|
54
|
+
name: elasticsearch
|
55
|
+
requirement: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - ~>
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: '1.0'
|
60
|
+
- - '>='
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: 1.0.17
|
63
|
+
type: :runtime
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ~>
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '1.0'
|
70
|
+
- - '>='
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: 1.0.17
|
73
|
+
description: Engine (workers) for Metrics Capacitor
|
74
|
+
email: radek@blufor.cz
|
75
|
+
executables: []
|
76
|
+
extensions: []
|
77
|
+
extra_rdoc_files: []
|
78
|
+
files:
|
79
|
+
- lib/metrics-capacitor.rb
|
80
|
+
- lib/metrics-capacitor/engine.rb
|
81
|
+
- lib/metrics-capacitor/processor/aggregator.rb
|
82
|
+
- lib/metrics-capacitor/processor/core.rb
|
83
|
+
- lib/metrics-capacitor/processor/listener.rb
|
84
|
+
- lib/metrics-capacitor/processor/scrubber.rb
|
85
|
+
- lib/metrics-capacitor/processor/writer.rb
|
86
|
+
- lib/metrics-capacitor/sidekiq.rb
|
87
|
+
homepage: https://github.com/metrics-capacitor/metrics-capacitor-engine
|
88
|
+
licenses:
|
89
|
+
- GPLv3
|
90
|
+
metadata: {}
|
91
|
+
post_install_message:
|
92
|
+
rdoc_options: []
|
93
|
+
require_paths:
|
94
|
+
- lib
|
95
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
96
|
+
requirements:
|
97
|
+
- - '>='
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: 2.0.0
|
100
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - '>='
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
requirements: []
|
106
|
+
rubyforge_project:
|
107
|
+
rubygems_version: 2.4.8
|
108
|
+
signing_key:
|
109
|
+
specification_version: 4
|
110
|
+
summary: Metrics Capacitor (engine)
|
111
|
+
test_files: []
|