metrics-capacitor 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2ba1e0c3121bf7c479cd51c71a5928f5d1cceb52
+  data.tar.gz: 65a6d6e97f87fb96d52e5f1e061a82fb28f70f18
+SHA512:
+  metadata.gz: 3c033eb3cd61172b26aebc36f54c6248621128e63864e10e6396caec1f52eb75b358694fc75bb0271eca5b78769ad725dc55cb97ace644b376caf7b850fb66c3
+  data.tar.gz: 376e4a97aa8714174c5a4f2ad80409c44253643f7aee37a5da04d9b1037c9167f8ed3ad950330d73ca850baf5734538735dfde6a464d8ace55825acf6d63c369

data/bin/metrics-capacitor

@@ -0,0 +1,3 @@
+#!/bin/env/ruby
+require 'metrics-capacitor/cli'
+MetricsCapacitor::CLI.start ARGV

data/lib/metrics-capacitor.rb

@@ -0,0 +1,3 @@
+module MetricsCapacitor; end
+require_relative 'metrics-capacitor/model'
+require_relative 'metrics-capacitor/config'

data/lib/metrics-capacitor/cli.rb

@@ -0,0 +1,69 @@
+require 'thor'
+
+module MetricsCapacitor
+  class CLI < Thor
+    package_name 'Metrics Capacitor'
+
+    desc 'engine', 'Start the engine :-)'
+    long_desc <<-LONGDESC
+      Usage: metrics-capacitor engine
+
+      all options are defined in /etc/metrics-capacitor.yaml
+    LONGDESC
+    def engine
+      require 'metrics-capacitor/engine'
+      Engine.new.run!
+    end
+
+    desc 'graphite', 'Send Graphite data'
+    long_desc <<-LONGDESC
+    This triggers Graphite metrics transformer. The data is awaited on STDIN.
+    AVOID EMPTY LINES AT THE END OF OUTPUT AT ALL COSTS
+
+    TAG_MAP option is mandatory and controls the behavior of graphite path
+    transformation into Elasticsearch fields, we call @tags and @name. Tag map must
+    correspond to the graphite data path.
+
+    There are 3 acceptable values: Integer (any sane positive whole number),
+    String (anything that Ruby parser can't convert to Integer) and special string `_`,
+    Integers are used as an array field indexes that are joined by `:` into @name field.
+    Strings are used as as @tags subfield names. And finally `_` is used if you want to
+    ignore the value on the position.
+
+    EXAMPLE:
+
+    Graphite path `server.node10.redis.hz` and TAG_MAP (-m) `_.host.0.1`
+    result metric with @name: `redis:hz` and @tags: `{ host: node10 }`
+
+    LONGDESC
+    option :tag_map, type: :string, required: true, aliases: '-m'
+    option :add_tag, type: :hash, aliases: '-t'
+    option :debug, type: :boolean, aliases: '-d'
+    option :counter_match, type: :string
+    def graphite
+      require 'metrics-capacitor/utils/graphite'
+      Utils::Graphite.new(options).run!
+    end
+
+    desc 'status', 'Report the state (TODO)'
+    def status
+      # TODO ...
+    end
+
+    desc 'aggregate', 'Manually trigger aggregator run (TODO)'
+    def aggregate
+      # TODO ...
+    end
+
+    desc 'expunge', 'Expunge old data (TODO)'
+    def expunge
+      # TODO ...
+    end
+
+    desc 'optimize', 'Optimize ElasticSearch indices (TODO)'
+    def optimize
+      # TODO ...
+    end
+
+  end
+end

data/lib/metrics-capacitor/config.rb

@@ -0,0 +1,61 @@
+require 'yaml'
+class ::Hash
+  def deep_merge(second)
+    merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
+    self.merge(second, &merger)
+  end
+end
+
+module MetricsCapacitor
+  module Config
+
+    extend self
+    attr_reader :_cfg
+
+    def load!
+      @_cfg = {
+        syslog: false,
+        debug: false,
+        redis: {
+          url: 'redis://127.0.0.1:6379/0',
+          timeout: 5,
+        },
+        elasticsearch: {
+          urls: ['http://localhost:9200/'],
+          index: 'metrics',
+          timeout: 10,
+          connections: 2,
+        },
+        scrubber: {
+          threads: 4,
+          processes: 2,
+          retry: 3,
+          tags: {}
+        },
+        writer: {
+          processes: 2,
+          doc_type: 'actual',
+          bulk_max: 5000,
+          bulk_wait: 10,
+          ttl: '1w'
+        },
+        aggregator: {
+          doc_type: 'aggregated',
+          aggregate_by: 600, # seconds
+          optimize_indices: true,
+        }
+      }
+      @_cfg = @_cfg.deep_merge YAML.load_file('/etc/metrics-capacitor.yaml') if File.exists? '/etc/metrics-capacitor.yaml'
+    end
+
+    def method_missing (name, *args, &block)
+      return @_cfg[name.to_sym] if @_cfg[name.to_sym] != nil
+      fail(NoMethodError, "Unknown configuration section Config.#{name}", caller)
+    rescue NoMethodError => e
+      $stderr.puts "ERROR config: #{e.class}: #{e.message}"
+      $stderr.puts e.backtrace.map { |l| "ERROR config: #{l}\n" }.join
+      exit! 1
+    end
+
+  end
+end

data/lib/metrics-capacitor/model.rb

@@ -0,0 +1,6 @@
+require 'msgpack'
+require 'json'
+require 'forwardable'
+require_relative 'model/metric'
+require_relative 'model/metrics'
+require_relative 'model/index_template'

data/lib/metrics-capacitor/model/index_template.rb

@@ -0,0 +1,59 @@
+module MetricsCapacitor
+  module Model
+    INDEX_TEMPLATE = {
+      template: "metrics*",
+      settings: {
+        number_of_shards: 2
+      },
+      mappings: {
+        '_default_' => {
+          '_source' => { 'enabled' => true },
+          'dynamic_templates' => [
+            {
+              'values' => {
+                'mapping' => {
+                  'index' => 'not_analyzed',
+                  'type' => 'float'
+                },
+                'path_match' => '@values.*'
+              }
+            },
+            {
+              'tags' => {
+                'mapping' => {
+                  'index' => 'not_analyzed',
+                  'type' => 'string',
+                  'copy_to' => '@uniq'
+                },
+                'path_match' => '@tags.*',
+                'path_unmatch' => '@tags._counter'
+              }
+            }
+          ],
+          'properties' => {
+            '@uniq' => {
+              'type' => 'string',
+              'index' => 'not_analyzed'
+            },
+            '@name' => {
+              'type' => 'string',
+              'index' => 'not_analyzed',
+              'copy_to' => '@uniq'
+            },
+            '@timestamp' => {
+              'type' => 'date'
+            },
+            '@tags' => {
+              'properties' => {
+                '_counter' => {
+                  'type' => 'boolean',
+                  'index' => 'not_analyzed'
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  end
+end

data/lib/metrics-capacitor/model/metric.rb

@@ -0,0 +1,67 @@
+module MetricsCapacitor
+  module Model
+    class Metric
+      extend Forwardable
+
+      def_delegators :@metric, :[], :[]=, :merge, :map
+
+      def initialize(data = {})
+        @metric = data if data.class == Hash
+        @metric ||= JSON.parse(data, symbolize_names: true)
+      end
+
+      def to_elastic
+        { index: {
+            data: {
+              :@name      => name,
+              :@timestamp => timestamp(:ms),
+              :@tags      => tags,
+              :@values    => values
+            }
+          }
+        }
+      end
+
+      def to_redis
+        @metric.to_json
+      end
+
+      def name
+        @metric[:name].to_s
+      end
+
+      def tags
+        return @metric[:tags] if ( @metric[:tags] || @metric[:tags].empty? )
+        { capacitor: 'untagged' }
+      end
+
+      def values
+        case @metric[:values]
+        when Hash
+          return @metric[:values]
+        when Integer
+          return { value: @metric[:values].to_f }
+        when Float
+          return { value: @metric[:values] }
+        else
+          return { value: 0.0 }
+        end
+      end
+
+      def timestamp(scale = :ms)
+        m = case scale
+            when :ms
+              1000.0
+            when :us
+              1_000_000.0
+            when :ns
+              1_000_000_000.0
+            else
+              1.0
+            end
+        return (Time.now.to_f * m).to_i.to_s unless @metric[:timestamp]
+        (Time.at(@metric[:timestamp]).to_f * m).to_i.to_s
+      end
+    end
+  end
+end

data/lib/metrics-capacitor/model/metrics.rb

@@ -0,0 +1,25 @@
+module MetricsCapacitor
+  module Model
+    class Metrics
+      extend Forwardable
+
+      def_delegators :@metrics, :slice, :slice!, :map, :each, :empty?, :length, :<<
+
+      def initialize(data = [])
+        @metrics = data.map { |m| Metric.new(m) }
+      end
+
+      def proc_by_slices!(n)
+        @metrics.each_slice(n) { |s| yield Metrics.new(s) }
+      end
+
+      def to_elastic
+        @metrics.map(&:to_elastic)
+      end
+
+      def to_redis
+        @metrics.map(&:to_redis)
+      end
+    end
+  end
+end

data/lib/metrics-capacitor/status.rb

@@ -0,0 +1,3 @@
+module MetricsCapacitor
+  class Status; end
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: metrics-capacitor
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Radek 'blufor' Slavicinsky
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.19'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.19'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.6
+description: Metrics igenstion service with ElasticSearch as storage
+email: radek@blufor.cz
+executables:
+- metrics-capacitor
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/metrics-capacitor
+- lib/metrics-capacitor.rb
+- lib/metrics-capacitor/cli.rb
+- lib/metrics-capacitor/config.rb
+- lib/metrics-capacitor/model.rb
+- lib/metrics-capacitor/model/index_template.rb
+- lib/metrics-capacitor/model/metric.rb
+- lib/metrics-capacitor/model/metrics.rb
+- lib/metrics-capacitor/status.rb
+homepage: https://github.com/metrics-capacitor/metrics-capacitor
+licenses:
+- GPLv3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Metrics Capacitor
+test_files: []