metrics-capacitor-engine 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 2f466bef259df899e75151d5904a203924869aec
4
+ data.tar.gz: 986d0e7f768fa95088884dda0da44942eb2febbe
5
+ SHA512:
6
+ metadata.gz: addf8bbb991ba5dcad978876d70f9ae69a5caa2f50a6d257e7cd5d7ba0b5e966a56a6f4522183bcb2bf80a6d8e98af1e51113255a0db4dadc108b46039b3b16a
7
+ data.tar.gz: 789b12ca634e3db7f65a34412f517d2b6b283c4c963ff7a5d9e9fd3aad41c054c63a04fc567472e6c9a2c63769341e2a093f50d30b47da77ab2c2ab7296bac1e
@@ -0,0 +1,2 @@
1
+ module MetricsCapacitor
2
+ end
@@ -0,0 +1,138 @@
1
+ require 'redis'
2
+ require 'syslog'
3
+ require 'syslog/logger'
4
+ require 'metrics-capacitor/config'
5
+
6
+ require_relative 'processor/core'
7
+ require_relative 'processor/writer'
8
+ require_relative 'processor/aggregator'
9
+ require_relative 'processor/listener'
10
+
11
+ module MetricsCapacitor
12
+ class Engine
13
+
14
+ def initialize
15
+ $0 = 'metrics-capacitor (engine)'
16
+ Config.load!
17
+ @exit_flag = false
18
+ @pids = []
19
+ @pids_kiq = []
20
+ %w(TERM INT).each do |sig|
21
+ Signal.trap(sig) do
22
+ @pids_kiq.each { |pid| Process.kill('INT', pid) rescue true }
23
+ @pids.each { |pid| Process.kill(sig, pid) rescue true }
24
+ Process.waitall
25
+ terminate_loggers
26
+ end
27
+ end
28
+ init_logger
29
+ log :info, "Engine warmed-up :-)"
30
+ end
31
+
32
+ def fork_processor(args = {})
33
+ log :debug, "Spawning #{args[:name]}"
34
+ args[:proc_num] ||= 1
35
+ args[:exit_on] ||= %w{INT TERM}
36
+ args[:proc_num].times do |num|
37
+ @logpipe["#{args[:name]}_#{num}".to_sym], logpipe = IO.pipe
38
+ @pids << Process.fork do
39
+ $0 = "metrics-capacitor (#{args[:name]})" if args[:name]
40
+ @logpipe["#{args[:name]}_#{num}".to_sym].close
41
+ remove_instance_variable(:@logpipe)
42
+ p = Kernel.const_get("MetricsCapacitor::Processor::#{args[:name].capitalize}").new(logpipe)
43
+ args[:exit_on].each { |sig| Signal.trap(sig) { p.shutdown! } }
44
+ p.start!
45
+ end
46
+ log :debug, "#{args[:name].capitalize} spawned as PID #{@pids.last.to_s}"
47
+ logpipe.close
48
+ end
49
+ end
50
+
51
+ def fork_scrubber
52
+ log :debug, "Spawning scrubbers"
53
+ Config.scrubber[:processes].times do |num|
54
+ @logpipe["scrubber_#{num}".to_sym], logpipe = IO.pipe
55
+ @pids_kiq << Process.fork do
56
+ @logpipe["scrubber_#{num}".to_sym].close
57
+ remove_instance_variable(:@logpipe)
58
+ require_relative 'sidekiq'
59
+ Sidekiq.configure_server do |config|
60
+ Sidekiq::Logging.logger = ::Logger.new(logpipe)
61
+ Sidekiq::Logging.logger.level = log_level
62
+ Sidekiq::Logging.logger.progname = "scrubber"
63
+ Sidekiq::Logging.logger.formatter = proc { |severity, _, progname, msg| "#{progname}##{Process.pid}|||#{severity}|||#{msg}\n" }
64
+ config.redis = { url: Config.redis[:url] }
65
+ end
66
+ Sidekiq.configure_client do |config|
67
+ config.redis = { url: Config.redis[:url] }
68
+ end
69
+ $TESTING = 0
70
+ kiq = Sidekiq::CLI.instance
71
+ kiq.parse(['-c', Config.scrubber[:threads].to_s, '-r', File.expand_path('..', __FILE__)+'/processor/scrubber.rb', '-q', 'scrubber'])
72
+ kiq.run
73
+ end
74
+ logpipe.close
75
+ end
76
+ end
77
+
78
+ def log(severity = :info, msg)
79
+ s = Kernel.const_get("Logger::#{severity.to_s.upcase}")
80
+ @logger_semaphore.synchronize do
81
+ @logger.log s, msg, 'engine#' + Process.pid.to_s
82
+ end
83
+ end
84
+
85
+ def log_level
86
+ Config.debug ? ::Logger::DEBUG : ::Logger::INFO
87
+ end
88
+
89
+ def init_logger
90
+ @logpipe = {}
91
+ @logger = ::Logger.new(STDOUT)
92
+ @logger.level = log_level
93
+ @logger.formatter = proc { |severity, datetime, progname, msg| [datetime.to_s, progname, severity, "#{msg}\n"].join(" ") }
94
+ @logger_threads = []
95
+ @logger_semaphore = Mutex.new
96
+ end
97
+
98
+ def spawn_logger_threads
99
+ @logpipe.each do |name, pipe|
100
+ @logger_threads << Thread.new do
101
+ Thread.current[:name] = "logger-#{name}"
102
+ while ! pipe.eof?
103
+ msg = pipe.gets
104
+ (progname,severity,message) = msg.split('|||')
105
+ @logger_semaphore.synchronize do
106
+ @logger.log Kernel.const_get("Logger::#{severity}"), message.chomp, progname
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
112
+
113
+ def terminate_loggers
114
+ @logger_threads.each(&:join)
115
+ end
116
+
117
+ def run!
118
+ log :info, 'Engine is starting up'
119
+ fork_scrubber
120
+ fork_processor name: 'writer', proc_num: Config.writer[:processes]
121
+ fork_processor name: 'aggregator'
122
+ fork_processor name: 'listener'
123
+ spawn_logger_threads
124
+ log :info, 'Engine has started :-)'
125
+ # TODO: unix socket for control and status reporting ;)
126
+ begin
127
+ ::Process.waitall
128
+ log :warn, 'Terminating!'
129
+ rescue Interrupt
130
+ retry
131
+ end
132
+ log :info, 'Terminating loggers'
133
+ terminate_loggers
134
+ log :warn, 'Engine is shutting down!'
135
+ end
136
+
137
+ end
138
+ end
@@ -0,0 +1,19 @@
1
+ module MetricsCapacitor
2
+ module Processor
3
+ class Aggregator < Core
4
+
5
+ # def post_init
6
+ # # TODO
7
+ # end
8
+ #
9
+ # def process
10
+ # # TODO
11
+ # end
12
+ #
13
+ # def shutdown
14
+ # # TODO
15
+ # end
16
+
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,63 @@
1
+ require 'metrics-capacitor/model'
2
+
3
+ module MetricsCapacitor
4
+ module Processor
5
+ class Core
6
+ include MetricsCapacitor::Model
7
+
8
+ def initialize(logpipe)
9
+ Config.load!
10
+ @_logger = ::Logger.new(logpipe)
11
+ @_name = self.class.to_s.split('::').last.downcase
12
+ @_logger.progname = @_name
13
+ @_logger.level = log_level
14
+ @_logger.formatter = proc { |severity, datetime, progname, msg| "#{progname}##{Process.pid}|||#{severity}|||#{msg}\n" }
15
+ logger.info "Initializing processor"
16
+ post_init
17
+ end
18
+
19
+ # implement in the processor Class
20
+ def post_init
21
+ end
22
+
23
+ # implement in the processor Class
24
+ def process
25
+ loop { logger.debug "alive"; sleep 10 }
26
+ end
27
+
28
+ # implement in the processor Class
29
+ def shutdown
30
+ exit 1
31
+ end
32
+
33
+ def logger
34
+ @_logger
35
+ end
36
+
37
+ def start!
38
+ logger.info "Starting processor"
39
+ begin
40
+ process
41
+ rescue StandardError => e
42
+ logger.fatal 'SHUTTING DOWN DUE TO AN EXCEPTION!'
43
+ logger.fatal [e.class, e.message].join(' -> ')
44
+ logger.fatal e.backtrace
45
+ logger.fatal 'Sending SIGINT to the engine'
46
+ Process.kill('INT', Process.ppid)
47
+ shutdown!
48
+ end
49
+ logger.info "Processor finished"
50
+ end
51
+
52
+ def shutdown!
53
+ shutdown
54
+ end
55
+
56
+ private
57
+ def log_level
58
+ Config.debug ? ::Logger::DEBUG : ::Logger::INFO
59
+ end
60
+
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,19 @@
1
+ module MetricsCapacitor
2
+ module Processor
3
+ class Listener < Core
4
+
5
+ # def post_init
6
+ # # TODO
7
+ # end
8
+ #
9
+ # def process
10
+ # # TODO
11
+ # end
12
+ #
13
+ # def shutdown
14
+ # # TODO
15
+ # end
16
+
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,30 @@
1
+ require 'metrics-capacitor/config'
2
+ require 'metrics-capacitor/model'
3
+
4
+ module MetricsCapacitor
5
+ module Processor
6
+ class Scrubber
7
+ include Sidekiq::Worker
8
+ include MetricsCapacitor::Model
9
+
10
+ sidekiq_options retry: true
11
+ sidekiq_options queue: :scrubber
12
+ sidekiq_options backtrace: true
13
+
14
+ REDIS = ConnectionPool.new(size: 2) { Redis.new(url: Config.redis[:url]) }
15
+
16
+ def perform *args
17
+ logger.debug 'Picking redis client from connection-pool'
18
+ REDIS.with do |r|
19
+ logger.debug 'Parsing metrics data'
20
+ metrics = Metrics.new args
21
+ logger.debug metrics.to_redis
22
+ logger.debug 'Sending data to writer'
23
+ r.rpush 'writer', metrics.to_redis
24
+ logger.debug 'Data sent'
25
+ end
26
+ end
27
+
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,67 @@
1
+ require 'elasticsearch'
2
+
3
+ module MetricsCapacitor
4
+ module Processor
5
+ class Writer < Core
6
+
7
+ def post_init
8
+ @elastic = Elasticsearch::Client.new(
9
+ url: Config.elasticsearch[:urls],
10
+ reload_connections: 100,
11
+ retry_on_failure: Config.elasticsearch[:retry],
12
+ sniffer_timeout: 5,
13
+ )
14
+ logger.debug 'Elastic connection set up'
15
+
16
+ @redis = Redis.new(url: Config.redis[:url])
17
+ logger.debug 'Redis connection set up'
18
+
19
+ logger.debug 'Setting ES index templates'
20
+ @elastic.indices.put_template name: 'metrics', body: INDEX_TEMPLATE
21
+ logger.debug 'ES templates set up'
22
+
23
+ @exit = false
24
+ end
25
+
26
+ def process
27
+ logger.debug 'Randomizing startup time'
28
+ sleep rand(Config.writer[:bulk_wait])
29
+ until @exit
30
+ logger.debug 'Gathering mertics bulk'
31
+ metric = nil
32
+ metrics = Metrics.new
33
+ indexing_result = nil
34
+ begin
35
+ while !@exit && metrics.length < Config.writer[:bulk_max] && ( metric = @redis.blpop('writer', timeout: Config.writer[:bulk_wait]) )
36
+ metrics << Metric.new(metric[1])
37
+ metric = nil
38
+ end
39
+ rescue Redis::CannotConnectError, Redis::TimeoutError => e
40
+ logger.error "Can't connect to redis: #{e.message}"
41
+ sleep 1
42
+ retry
43
+ end
44
+ unless metrics.empty?
45
+ logger.debug "Writing #{metrics.length} metrics"
46
+ logger.debug metrics.to_elastic.to_json
47
+ indexing_result = @elastic.bulk(index: Time.now.strftime(Config.elasticsearch[:index]), type: Config.writer[:doc_type], body: metrics.to_elastic)
48
+ if indexing_result['errors']
49
+ logger.error 'Failed to write metrics!'
50
+ logger.error indexing_result['items'].to_json
51
+ else
52
+ logger.info "Written #{metrics.length} metrics, took #{indexing_result['took']}ms"
53
+ end
54
+ else
55
+ logger.warn 'No metrics to write :-('
56
+ end
57
+ metrics = nil
58
+ indexing_result = nil
59
+ end
60
+ end
61
+
62
+ def shutdown
63
+ @exit = true
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,49 @@
1
+ require 'sidekiq'
2
+ require 'sidekiq/cli'
3
+ require 'sidekiq/logging'
4
+
5
+ module Sidekiq
6
+ class CLI
7
+
8
+ PROCTITLES[0] = proc { 'metrics-capacitor'.freeze }
9
+ PROCTITLES[1] = proc { '(scrubber)'.freeze }
10
+
11
+ def run
12
+ @code = nil
13
+
14
+ boot_system
15
+
16
+ self_read, self_write = IO.pipe
17
+
18
+ %w(INT USR1 USR2 TTIN).each do |sig|
19
+ begin
20
+ trap sig do
21
+ self_write.puts(sig)
22
+ end
23
+ rescue ArgumentError
24
+ puts "Signal #{sig} not supported"
25
+ end
26
+ end
27
+
28
+ ver = Sidekiq.redis_info['redis_version']
29
+ raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
30
+ fire_event(:startup)
31
+ logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
32
+ logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
33
+ require 'sidekiq/launcher'
34
+ @launcher = Sidekiq::Launcher.new(options)
35
+ begin
36
+ launcher.run
37
+ while readable_io = IO.select([self_read])
38
+ signal = readable_io.first[0].gets.strip
39
+ handle_signal(signal)
40
+ end
41
+ rescue Interrupt
42
+ logger.info 'Shutting down'
43
+ launcher.stop
44
+ logger.info "Bye!"
45
+ exit(0)
46
+ end
47
+ end
48
+ end
49
+ end
metadata ADDED
@@ -0,0 +1,111 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: metrics-capacitor-engine
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Radek 'blufor' Slavicinsky
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-05-27 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: metrics-capacitor
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ - - '>='
21
+ - !ruby/object:Gem::Version
22
+ version: 0.0.1
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ - - '>='
31
+ - !ruby/object:Gem::Version
32
+ version: 0.0.1
33
+ - !ruby/object:Gem::Dependency
34
+ name: sidekiq
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ~>
38
+ - !ruby/object:Gem::Version
39
+ version: '4.1'
40
+ - - '>='
41
+ - !ruby/object:Gem::Version
42
+ version: 4.1.2
43
+ type: :runtime
44
+ prerelease: false
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - ~>
48
+ - !ruby/object:Gem::Version
49
+ version: '4.1'
50
+ - - '>='
51
+ - !ruby/object:Gem::Version
52
+ version: 4.1.2
53
+ - !ruby/object:Gem::Dependency
54
+ name: elasticsearch
55
+ requirement: !ruby/object:Gem::Requirement
56
+ requirements:
57
+ - - ~>
58
+ - !ruby/object:Gem::Version
59
+ version: '1.0'
60
+ - - '>='
61
+ - !ruby/object:Gem::Version
62
+ version: 1.0.17
63
+ type: :runtime
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ~>
68
+ - !ruby/object:Gem::Version
69
+ version: '1.0'
70
+ - - '>='
71
+ - !ruby/object:Gem::Version
72
+ version: 1.0.17
73
+ description: Engine (workers) for Metrics Capacitor
74
+ email: radek@blufor.cz
75
+ executables: []
76
+ extensions: []
77
+ extra_rdoc_files: []
78
+ files:
79
+ - lib/metrics-capacitor.rb
80
+ - lib/metrics-capacitor/engine.rb
81
+ - lib/metrics-capacitor/processor/aggregator.rb
82
+ - lib/metrics-capacitor/processor/core.rb
83
+ - lib/metrics-capacitor/processor/listener.rb
84
+ - lib/metrics-capacitor/processor/scrubber.rb
85
+ - lib/metrics-capacitor/processor/writer.rb
86
+ - lib/metrics-capacitor/sidekiq.rb
87
+ homepage: https://github.com/metrics-capacitor/metrics-capacitor-engine
88
+ licenses:
89
+ - GPLv3
90
+ metadata: {}
91
+ post_install_message:
92
+ rdoc_options: []
93
+ require_paths:
94
+ - lib
95
+ required_ruby_version: !ruby/object:Gem::Requirement
96
+ requirements:
97
+ - - '>='
98
+ - !ruby/object:Gem::Version
99
+ version: 2.0.0
100
+ required_rubygems_version: !ruby/object:Gem::Requirement
101
+ requirements:
102
+ - - '>='
103
+ - !ruby/object:Gem::Version
104
+ version: '0'
105
+ requirements: []
106
+ rubyforge_project:
107
+ rubygems_version: 2.4.8
108
+ signing_key:
109
+ specification_version: 4
110
+ summary: Metrics Capacitor (engine)
111
+ test_files: []