metrics-capacitor 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/metrics-capacitor +3 -0
- data/lib/metrics-capacitor.rb +3 -0
- data/lib/metrics-capacitor/cli.rb +69 -0
- data/lib/metrics-capacitor/config.rb +61 -0
- data/lib/metrics-capacitor/model.rb +6 -0
- data/lib/metrics-capacitor/model/index_template.rb +59 -0
- data/lib/metrics-capacitor/model/metric.rb +67 -0
- data/lib/metrics-capacitor/model/metrics.rb +25 -0
- data/lib/metrics-capacitor/status.rb +3 -0
- metadata +93 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2ba1e0c3121bf7c479cd51c71a5928f5d1cceb52
|
4
|
+
data.tar.gz: 65a6d6e97f87fb96d52e5f1e061a82fb28f70f18
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 3c033eb3cd61172b26aebc36f54c6248621128e63864e10e6396caec1f52eb75b358694fc75bb0271eca5b78769ad725dc55cb97ace644b376caf7b850fb66c3
|
7
|
+
data.tar.gz: 376e4a97aa8714174c5a4f2ad80409c44253643f7aee37a5da04d9b1037c9167f8ed3ad950330d73ca850baf5734538735dfde6a464d8ace55825acf6d63c369
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'thor'
|
2
|
+
|
3
|
+
module MetricsCapacitor
|
4
|
+
class CLI < Thor
|
5
|
+
package_name 'Metrics Capacitor'
|
6
|
+
|
7
|
+
desc 'engine', 'Start the engine :-)'
|
8
|
+
long_desc <<-LONGDESC
|
9
|
+
Usage: metrics-capacitor engine
|
10
|
+
|
11
|
+
all options are defined in /etc/metrics-capacitor.yaml
|
12
|
+
LONGDESC
|
13
|
+
def engine
|
14
|
+
require 'metrics-capacitor/engine'
|
15
|
+
Engine.new.run!
|
16
|
+
end
|
17
|
+
|
18
|
+
desc 'graphite', 'Send Graphite data'
|
19
|
+
long_desc <<-LONGDESC
|
20
|
+
This triggers Graphite metrics transformer. The data is awaited on STDIN.
|
21
|
+
AVOID EMPTY LINES AT THE END OF OUTPUT AT ALL COSTS
|
22
|
+
|
23
|
+
TAG_MAP option is mandatory and controls the behavior of graphite path
|
24
|
+
transformation into Elasticsearch fields, we call @tags and @name. Tag map must
|
25
|
+
correspond to the graphite data path.
|
26
|
+
|
27
|
+
There are 3 acceptable values: Integer (any sane positive whole number),
|
28
|
+
String (anything that Ruby parser can't convert to Integer) and special string `_`,
|
29
|
+
Integers are used as an array field indexes that are joined by `:` into @name field.
|
30
|
+
Strings are used as as @tags subfield names. And finally `_` is used if you want to
|
31
|
+
ignore the value on the position.
|
32
|
+
|
33
|
+
EXAMPLE:
|
34
|
+
|
35
|
+
Graphite path `server.node10.redis.hz` and TAG_MAP (-m) `_.host.0.1`
|
36
|
+
result metric with @name: `redis:hz` and @tags: `{ host: node10 }`
|
37
|
+
|
38
|
+
LONGDESC
|
39
|
+
option :tag_map, type: :string, required: true, aliases: '-m'
|
40
|
+
option :add_tag, type: :hash, aliases: '-t'
|
41
|
+
option :debug, type: :boolean, aliases: '-d'
|
42
|
+
option :counter_match, type: :string
|
43
|
+
def graphite
|
44
|
+
require 'metrics-capacitor/utils/graphite'
|
45
|
+
Utils::Graphite.new(options).run!
|
46
|
+
end
|
47
|
+
|
48
|
+
desc 'status', 'Report the state (TODO)'
|
49
|
+
def status
|
50
|
+
# TODO ...
|
51
|
+
end
|
52
|
+
|
53
|
+
desc 'aggregate', 'Manually trigger aggregator run (TODO)'
|
54
|
+
def aggregate
|
55
|
+
# TODO ...
|
56
|
+
end
|
57
|
+
|
58
|
+
desc 'expunge', 'Expunge old data (TODO)'
|
59
|
+
def expunge
|
60
|
+
# TODO ...
|
61
|
+
end
|
62
|
+
|
63
|
+
desc 'optimize', 'Optimize ElasticSearch indices (TODO)'
|
64
|
+
def optimize
|
65
|
+
# TODO ...
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
class ::Hash
|
3
|
+
def deep_merge(second)
|
4
|
+
merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
|
5
|
+
self.merge(second, &merger)
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
module MetricsCapacitor
|
10
|
+
module Config
|
11
|
+
|
12
|
+
extend self
|
13
|
+
attr_reader :_cfg
|
14
|
+
|
15
|
+
def load!
|
16
|
+
@_cfg = {
|
17
|
+
syslog: false,
|
18
|
+
debug: false,
|
19
|
+
redis: {
|
20
|
+
url: 'redis://127.0.0.1:6379/0',
|
21
|
+
timeout: 5,
|
22
|
+
},
|
23
|
+
elasticsearch: {
|
24
|
+
urls: ['http://localhost:9200/'],
|
25
|
+
index: 'metrics',
|
26
|
+
timeout: 10,
|
27
|
+
connections: 2,
|
28
|
+
},
|
29
|
+
scrubber: {
|
30
|
+
threads: 4,
|
31
|
+
processes: 2,
|
32
|
+
retry: 3,
|
33
|
+
tags: {}
|
34
|
+
},
|
35
|
+
writer: {
|
36
|
+
processes: 2,
|
37
|
+
doc_type: 'actual',
|
38
|
+
bulk_max: 5000,
|
39
|
+
bulk_wait: 10,
|
40
|
+
ttl: '1w'
|
41
|
+
},
|
42
|
+
aggregator: {
|
43
|
+
doc_type: 'aggregated',
|
44
|
+
aggregate_by: 600, # seconds
|
45
|
+
optimize_indices: true,
|
46
|
+
}
|
47
|
+
}
|
48
|
+
@_cfg = @_cfg.deep_merge YAML.load_file('/etc/metrics-capacitor.yaml') if File.exists? '/etc/metrics-capacitor.yaml'
|
49
|
+
end
|
50
|
+
|
51
|
+
def method_missing (name, *args, &block)
|
52
|
+
return @_cfg[name.to_sym] if @_cfg[name.to_sym] != nil
|
53
|
+
fail(NoMethodError, "Unknown configuration section Config.#{name}", caller)
|
54
|
+
rescue NoMethodError => e
|
55
|
+
$stderr.puts "ERROR config: #{e.class}: #{e.message}"
|
56
|
+
$stderr.puts e.backtrace.map { |l| "ERROR config: #{l}\n" }.join
|
57
|
+
exit! 1
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module MetricsCapacitor
|
2
|
+
module Model
|
3
|
+
INDEX_TEMPLATE = {
|
4
|
+
template: "metrics*",
|
5
|
+
settings: {
|
6
|
+
number_of_shards: 2
|
7
|
+
},
|
8
|
+
mappings: {
|
9
|
+
'_default_' => {
|
10
|
+
'_source' => { 'enabled' => true },
|
11
|
+
'dynamic_templates' => [
|
12
|
+
{
|
13
|
+
'values' => {
|
14
|
+
'mapping' => {
|
15
|
+
'index' => 'not_analyzed',
|
16
|
+
'type' => 'float'
|
17
|
+
},
|
18
|
+
'path_match' => '@values.*'
|
19
|
+
}
|
20
|
+
},
|
21
|
+
{
|
22
|
+
'tags' => {
|
23
|
+
'mapping' => {
|
24
|
+
'index' => 'not_analyzed',
|
25
|
+
'type' => 'string',
|
26
|
+
'copy_to' => '@uniq'
|
27
|
+
},
|
28
|
+
'path_match' => '@tags.*',
|
29
|
+
'path_unmatch' => '@tags._counter'
|
30
|
+
}
|
31
|
+
}
|
32
|
+
],
|
33
|
+
'properties' => {
|
34
|
+
'@uniq' => {
|
35
|
+
'type' => 'string',
|
36
|
+
'index' => 'not_analyzed'
|
37
|
+
},
|
38
|
+
'@name' => {
|
39
|
+
'type' => 'string',
|
40
|
+
'index' => 'not_analyzed',
|
41
|
+
'copy_to' => '@uniq'
|
42
|
+
},
|
43
|
+
'@timestamp' => {
|
44
|
+
'type' => 'date'
|
45
|
+
},
|
46
|
+
'@tags' => {
|
47
|
+
'properties' => {
|
48
|
+
'_counter' => {
|
49
|
+
'type' => 'boolean',
|
50
|
+
'index' => 'not_analyzed'
|
51
|
+
}
|
52
|
+
}
|
53
|
+
}
|
54
|
+
}
|
55
|
+
}
|
56
|
+
}
|
57
|
+
}
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module MetricsCapacitor
|
2
|
+
module Model
|
3
|
+
class Metric
|
4
|
+
extend Forwardable
|
5
|
+
|
6
|
+
def_delegators :@metric, :[], :[]=, :merge, :map
|
7
|
+
|
8
|
+
def initialize(data = {})
|
9
|
+
@metric = data if data.class == Hash
|
10
|
+
@metric ||= JSON.parse(data, symbolize_names: true)
|
11
|
+
end
|
12
|
+
|
13
|
+
def to_elastic
|
14
|
+
{ index: {
|
15
|
+
data: {
|
16
|
+
:@name => name,
|
17
|
+
:@timestamp => timestamp(:ms),
|
18
|
+
:@tags => tags,
|
19
|
+
:@values => values
|
20
|
+
}
|
21
|
+
}
|
22
|
+
}
|
23
|
+
end
|
24
|
+
|
25
|
+
def to_redis
|
26
|
+
@metric.to_json
|
27
|
+
end
|
28
|
+
|
29
|
+
def name
|
30
|
+
@metric[:name].to_s
|
31
|
+
end
|
32
|
+
|
33
|
+
def tags
|
34
|
+
return @metric[:tags] if ( @metric[:tags] || @metric[:tags].empty? )
|
35
|
+
{ capacitor: 'untagged' }
|
36
|
+
end
|
37
|
+
|
38
|
+
def values
|
39
|
+
case @metric[:values]
|
40
|
+
when Hash
|
41
|
+
return @metric[:values]
|
42
|
+
when Integer
|
43
|
+
return { value: @metric[:values].to_f }
|
44
|
+
when Float
|
45
|
+
return { value: @metric[:values] }
|
46
|
+
else
|
47
|
+
return { value: 0.0 }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def timestamp(scale = :ms)
|
52
|
+
m = case scale
|
53
|
+
when :ms
|
54
|
+
1000.0
|
55
|
+
when :us
|
56
|
+
1_000_000.0
|
57
|
+
when :ns
|
58
|
+
1_000_000_000.0
|
59
|
+
else
|
60
|
+
1.0
|
61
|
+
end
|
62
|
+
return (Time.now.to_f * m).to_i.to_s unless @metric[:timestamp]
|
63
|
+
(Time.at(@metric[:timestamp]).to_f * m).to_i.to_s
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module MetricsCapacitor
|
2
|
+
module Model
|
3
|
+
class Metrics
|
4
|
+
extend Forwardable
|
5
|
+
|
6
|
+
def_delegators :@metrics, :slice, :slice!, :map, :each, :empty?, :length, :<<
|
7
|
+
|
8
|
+
def initialize(data = [])
|
9
|
+
@metrics = data.map { |m| Metric.new(m) }
|
10
|
+
end
|
11
|
+
|
12
|
+
def proc_by_slices!(n)
|
13
|
+
@metrics.each_slice(n) { |s| yield Metrics.new(s) }
|
14
|
+
end
|
15
|
+
|
16
|
+
def to_elastic
|
17
|
+
@metrics.map(&:to_elastic)
|
18
|
+
end
|
19
|
+
|
20
|
+
def to_redis
|
21
|
+
@metrics.map(&:to_redis)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: metrics-capacitor
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Radek 'blufor' Slavicinsky
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-05-27 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: thor
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0.19'
|
20
|
+
- - '>='
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 0.19.1
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0.19'
|
30
|
+
- - '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 0.19.1
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: msgpack
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ~>
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0.7'
|
40
|
+
- - '>='
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: 0.7.6
|
43
|
+
type: :runtime
|
44
|
+
prerelease: false
|
45
|
+
version_requirements: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - ~>
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: '0.7'
|
50
|
+
- - '>='
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 0.7.6
|
53
|
+
description: Metrics igenstion service with ElasticSearch as storage
|
54
|
+
email: radek@blufor.cz
|
55
|
+
executables:
|
56
|
+
- metrics-capacitor
|
57
|
+
extensions: []
|
58
|
+
extra_rdoc_files: []
|
59
|
+
files:
|
60
|
+
- bin/metrics-capacitor
|
61
|
+
- lib/metrics-capacitor.rb
|
62
|
+
- lib/metrics-capacitor/cli.rb
|
63
|
+
- lib/metrics-capacitor/config.rb
|
64
|
+
- lib/metrics-capacitor/model.rb
|
65
|
+
- lib/metrics-capacitor/model/index_template.rb
|
66
|
+
- lib/metrics-capacitor/model/metric.rb
|
67
|
+
- lib/metrics-capacitor/model/metrics.rb
|
68
|
+
- lib/metrics-capacitor/status.rb
|
69
|
+
homepage: https://github.com/metrics-capacitor/metrics-capacitor
|
70
|
+
licenses:
|
71
|
+
- GPLv3
|
72
|
+
metadata: {}
|
73
|
+
post_install_message:
|
74
|
+
rdoc_options: []
|
75
|
+
require_paths:
|
76
|
+
- lib
|
77
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - '>='
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: 2.0.0
|
82
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
83
|
+
requirements:
|
84
|
+
- - '>='
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: '0'
|
87
|
+
requirements: []
|
88
|
+
rubyforge_project:
|
89
|
+
rubygems_version: 2.4.8
|
90
|
+
signing_key:
|
91
|
+
specification_version: 4
|
92
|
+
summary: Metrics Capacitor
|
93
|
+
test_files: []
|