vmpooler 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6c0edeb60c2a23b1dc15da9f2535b9420b4de8c3
+  data.tar.gz: 667639acb53c6b83bb530a0134b506da57e51a32
+SHA512:
+  metadata.gz: 647a7292fb7b5c9bc991ce68e2d42998a6604bd77392221baca916c90ca696e58cd6879a4125ce863c97b5ff04dfca98ed6d5d7380c7d9251ff663c8eca3d61b
+  data.tar.gz: bf66abbb2ea1b1ba8e06e82ecaa293f2e160fab25959ce260f1b066b8ddeb5185c3c1fa40b14d3fc39f6f009bdf53b8dbaa8a64ff3380249ddea31af1c30a035

data/bin/vmpooler

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+require 'vmpooler'
+
+config = Vmpooler.config
+redis_host = config[:redis]['server']
+redis_port = config[:redis]['port']
+redis_password = config[:redis]['password']
+logger_file = config[:config]['logfile']
+
+metrics = Vmpooler.new_metrics(config)
+
+torun_threads = []
+if ARGV.count == 0
+  torun = ['api', 'manager']
+else
+  torun = []
+  torun << 'api' if ARGV.include? 'api'
+  torun << 'manager' if ARGV.include? 'manager'
+  exit(2) if torun.empty?
+end
+
+if torun.include? 'api'
+  api = Thread.new do
+    thr = Vmpooler::API.new
+    redis = Vmpooler.new_redis(redis_host, redis_port, redis_password)
+    thr.helpers.configure(config, redis, metrics)
+    thr.helpers.execute!
+  end
+  torun_threads << api
+end
+
+if torun.include? 'manager'
+  manager = Thread.new do
+    Vmpooler::PoolManager.new(
+      config,
+      Vmpooler.new_logger(logger_file),
+      Vmpooler.new_redis(redis_host, redis_port, redis_password),
+      metrics
+    ).execute!
+  end
+  torun_threads << manager
+end
+
+if ENV['VMPOOLER_DEBUG']
+  trap('INT') do
+    puts 'Shutting down.'
+    torun_threads.each(&:exit)
+  end
+end
+
+torun_threads.each do |th|
+  th.join
+end

data/lib/vmpooler.rb

@@ -0,0 +1,161 @@
+module Vmpooler
+  require 'date'
+  require 'json'
+  require 'open-uri'
+  require 'net/ldap'
+  require 'rbvmomi'
+  require 'redis'
+  require 'sinatra/base'
+  require 'time'
+  require 'timeout'
+  require 'yaml'
+  require 'set'
+
+  %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool providers].each do |lib|
+    require "vmpooler/#{lib}"
+  end
+
+  def self.config(filepath = 'vmpooler.yaml')
+    # Take the config either from an ENV config variable or from a config file
+    if ENV['VMPOOLER_CONFIG']
+      config_string = ENV['VMPOOLER_CONFIG']
+      # Parse the YAML config into a Hash
+      # Whitelist the Symbol class
+      parsed_config = YAML.safe_load(config_string, [Symbol])
+    else
+      # Take the name of the config file either from an ENV variable or from the filepath argument
+      config_file = ENV['VMPOOLER_CONFIG_FILE'] || filepath
+      parsed_config = YAML.load_file(config_file) if File.exist? config_file
+    end
+
+    parsed_config ||= { config: {} }
+
+    # Bail out if someone attempts to start vmpooler with dummy authentication
+    # without enbaling debug mode.
+    if parsed_config.has_key? :auth
+      if parsed_config[:auth]['provider'] == 'dummy'
+        unless ENV['VMPOOLER_DEBUG']
+          warning = [
+            'Dummy authentication should not be used outside of debug mode',
+            'please set environment variable VMPOOLER_DEBUG to \'true\' if you want to use dummy authentication'
+          ]
+
+          raise warning.join(";\s")
+        end
+      end
+    end
+
+    # Set some configuration defaults
+    parsed_config[:config]['task_limit'] = ENV['TASK_LIMIT'] || parsed_config[:config]['task_limit'] || 10
+    parsed_config[:config]['migration_limit'] = ENV['MIGRATION_LIMIT'] if ENV['MIGRATION_LIMIT']
+    parsed_config[:config]['vm_checktime'] = ENV['VM_CHECKTIME'] || parsed_config[:config]['vm_checktime'] || 15
+    parsed_config[:config]['vm_lifetime'] = ENV['VM_LIFETIME'] || parsed_config[:config]['vm_lifetime']  || 24
+    parsed_config[:config]['prefix'] = ENV['VM_PREFIX'] || parsed_config[:config]['prefix'] || ''
+
+    parsed_config[:config]['logfile'] = ENV['LOGFILE'] if ENV['LOGFILE']
+
+    parsed_config[:config]['site_name'] = ENV['SITE_NAME'] if ENV['SITE_NAME']
+    parsed_config[:config]['domain'] = ENV['DOMAIN_NAME'] if ENV['DOMAIN_NAME']
+    parsed_config[:config]['clone_target'] = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
+    parsed_config[:config]['timeout'] = ENV['TIMEOUT'] if ENV['TIMEOUT']
+    parsed_config[:config]['vm_lifetime_auth'] = ENV['VM_LIFETIME_AUTH'] if ENV['VM_LIFETIME_AUTH']
+    parsed_config[:config]['ssh_key'] = ENV['SSH_KEY'] if ENV['SSH_KEY']
+    parsed_config[:config]['max_tries'] = ENV['MAX_TRIES'] if ENV['MAX_TRIES']
+    parsed_config[:config]['retry_factor'] = ENV['RETRY_FACTOR'] if ENV['RETRY_FACTOR']
+    parsed_config[:config]['create_folders'] = ENV['CREATE_FOLDERS'] if ENV['CREATE_FOLDERS']
+    parsed_config[:config]['create_template_delta_disks'] = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
+    parsed_config[:config]['experimental_features'] = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
+
+    parsed_config[:redis] = parsed_config[:redis] || {}
+    parsed_config[:redis]['server'] = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost'
+    parsed_config[:redis]['port'] = ENV['REDIS_PORT'] if ENV['REDIS_PORT']
+    parsed_config[:redis]['password'] = ENV['REDIS_PASSWORD'] if ENV['REDIS_PASSWORD']
+    parsed_config[:redis]['data_ttl'] = ENV['REDIS_DATA_TTL'] || parsed_config[:redis]['data_ttl'] || 168
+
+    parsed_config[:statsd] = parsed_config[:statsd] || {} if ENV['STATSD_SERVER']
+    parsed_config[:statsd]['server'] = ENV['STATSD_SERVER'] if ENV['STATSD_SERVER']
+    parsed_config[:statsd]['prefix'] = ENV['STATSD_PREFIX'] if ENV['STATSD_PREFIX']
+    parsed_config[:statsd]['port'] = ENV['STATSD_PORT'] if ENV['STATSD_PORT']
+
+    parsed_config[:graphite] = parsed_config[:graphite] || {} if ENV['GRAPHITE_SERVER']
+    parsed_config[:graphite]['server'] = ENV['GRAPHITE_SERVER'] if ENV['GRAPHITE_SERVER']
+
+    parsed_config[:auth] = parsed_config[:auth] || {} if ENV['AUTH_PROVIDER']
+    if parsed_config.has_key? :auth
+      parsed_config[:auth]['provider'] = ENV['AUTH_PROVIDER'] if ENV['AUTH_PROVIDER']
+      parsed_config[:auth][:ldap] = parsed_config[:auth][:ldap] || {} if parsed_config[:auth]['provider'] == 'ldap'
+      parsed_config[:auth][:ldap]['server'] = ENV['LDAP_SERVER'] if ENV['LDAP_SERVER']
+      parsed_config[:auth][:ldap]['port'] = ENV['LDAP_PORT'] if ENV['LDAP_PORT']
+      parsed_config[:auth][:ldap]['base'] = ENV['LDAP_BASE'] if ENV['LDAP_BASE']
+      parsed_config[:auth][:ldap]['user_object'] = ENV['LDAP_USER_OBJECT'] if ENV['LDAP_USER_OBJECT']
+    end
+
+    # Create an index of pool aliases
+    parsed_config[:pool_names] = Set.new
+    unless parsed_config[:pools]
+      redis = new_redis(parsed_config[:redis]['server'], parsed_config[:redis]['port'], parsed_config[:redis]['password'])
+      parsed_config[:pools] = load_pools_from_redis(redis)
+    end
+
+    parsed_config[:pools].each do |pool|
+      parsed_config[:pool_names] << pool['name']
+      if pool['alias']
+        if pool['alias'].is_a?(Array)
+          pool['alias'].each do |a|
+            parsed_config[:alias] ||= {}
+            parsed_config[:alias][a] = pool['name']
+            parsed_config[:pool_names] << a
+          end
+        elsif pool['alias'].is_a?(String)
+          parsed_config[:alias][pool['alias']] = pool['name']
+          parsed_config[:pool_names] << pool['alias']
+        end
+      end
+    end
+
+    if parsed_config[:tagfilter]
+      parsed_config[:tagfilter].keys.each do |tag|
+        parsed_config[:tagfilter][tag] = Regexp.new(parsed_config[:tagfilter][tag])
+      end
+    end
+
+    parsed_config[:uptime] = Time.now
+
+    parsed_config
+  end
+
+  def self.load_pools_from_redis(redis)
+    pools = []
+    redis.smembers('vmpooler__pools').each do |pool|
+      pool_hash = {}
+      redis.hgetall("vmpooler__pool__#{pool}").each do |k, v|
+        pool_hash[k] = v
+      end
+      pool_hash['alias'] = pool_hash['alias'].split(',')
+      pools << pool_hash
+    end
+    pools
+  end
+
+  def self.new_redis(host = 'localhost', port = nil, password = nil)
+    Redis.new(host: host, port: port, password: password)
+  end
+
+  def self.new_logger(logfile)
+    Vmpooler::Logger.new logfile
+  end
+
+  def self.new_metrics(params)
+    if params[:statsd]
+      Vmpooler::Statsd.new(params[:statsd])
+    elsif params[:graphite]
+      Vmpooler::Graphite.new(params[:graphite])
+    else
+      Vmpooler::DummyStatsd.new
+    end
+  end
+
+  def self.pools(conf)
+    conf[:pools]
+  end
+end

data/lib/vmpooler/api.rb

@@ -0,0 +1,53 @@
+module Vmpooler
+  class API < Sinatra::Base
+    def initialize
+      super
+    end
+
+    not_found do
+      content_type :json
+
+      result = {
+        ok: false
+      }
+
+      JSON.pretty_generate(result)
+    end
+
+    get '/' do
+      redirect to('/dashboard/')
+    end
+
+    # Load dashboard components
+    begin
+      require 'dashboard'
+    rescue LoadError
+      require File.expand_path(File.join(File.dirname(__FILE__), 'dashboard'))
+    end
+
+    use Vmpooler::Dashboard
+
+    # Load API components
+    %w[helpers dashboard reroute v1].each do |lib|
+      begin
+        require "api/#{lib}"
+      rescue LoadError
+        require File.expand_path(File.join(File.dirname(__FILE__), 'api', lib))
+      end
+    end
+
+    use Vmpooler::API::Dashboard
+    use Vmpooler::API::Reroute
+    use Vmpooler::API::V1
+
+    def configure(config, redis, metrics)
+      self.settings.set :config, config
+      self.settings.set :redis, redis
+      self.settings.set :metrics, metrics
+    end
+
+    def execute!
+      self.settings.run!
+    end
+  end
+end

data/lib/vmpooler/api/dashboard.rb

@@ -0,0 +1,143 @@
+module Vmpooler
+  class API
+    class Dashboard < Sinatra::Base
+      # handle to the App's configuration information
+      def config
+        @config ||= Vmpooler::API.settings.config
+      end
+
+      # configuration setting for server hosting graph URLs to view
+      def graph_server
+        return @graph_server if @graph_server
+
+        if config[:graphs]
+          return false unless config[:graphs]['server']
+          @graph_server = config[:graphs]['server']
+        elsif config[:graphite]
+          return false unless config[:graphite]['server']
+          @graph_server = config[:graphite]['server']
+        else
+          false
+        end
+      end
+
+      # configuration setting for URL prefix for graphs to view
+      def graph_prefix
+        return @graph_prefix if @graph_prefix
+
+        if config[:graphs]
+          return 'vmpooler' unless config[:graphs]['prefix']
+          @graph_prefix = config[:graphs]['prefix']
+        elsif config[:graphite]
+          return false unless config[:graphite]['prefix']
+          @graph_prefix = config[:graphite]['prefix']
+        else
+          false
+        end
+      end
+
+      # what is the base URL for viewable graphs?
+      def graph_url
+        return false unless graph_server && graph_prefix
+        @graph_url ||= "http://#{graph_server}/render?target=#{graph_prefix}"
+      end
+
+      # return a full URL to a viewable graph for a given metrics target (graphite syntax)
+      def graph_link(target = '')
+        return '' unless graph_url
+        graph_url + target
+      end
+
+
+      get '/dashboard/stats/vmpooler/pool/?' do
+        content_type :json
+        result = {}
+
+        Vmpooler::API.settings.config[:pools].each do |pool|
+          result[pool['name']] ||= {}
+          result[pool['name']]['size'] = pool['size']
+          result[pool['name']]['ready'] = Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name'])
+        end
+
+        if params[:history]
+          if graph_url
+            history ||= {}
+
+            begin
+              buffer = open(graph_link('.ready.*&from=-1hour&format=json')).read
+              history = JSON.parse(buffer)
+
+              history.each do |pool|
+                if pool['target'] =~ /.*\.(.*)$/
+                  pool['name'] = Regexp.last_match[1]
+
+                  if result[pool['name']]
+                    pool['last'] = result[pool['name']]['size']
+                    result[pool['name']]['history'] ||= []
+
+                    pool['datapoints'].each do |metric|
+                      8.times do |_n|
+                        if metric[0]
+                          pool['last'] = metric[0].to_i
+                          result[pool['name']]['history'].push(metric[0].to_i)
+                        else
+                          result[pool['name']]['history'].push(pool['last'])
+                        end
+                      end
+                    end
+                  end
+                end
+              end
+            rescue
+            end
+          else
+            Vmpooler::API.settings.config[:pools].each do |pool|
+              result[pool['name']] ||= {}
+              result[pool['name']]['history'] = [Vmpooler::API.settings.redis.scard('vmpooler__ready__' + pool['name'])]
+            end
+          end
+        end
+        JSON.pretty_generate(result)
+      end
+
+      get '/dashboard/stats/vmpooler/running/?' do
+        content_type :json
+        result = {}
+
+        Vmpooler::API.settings.config[:pools].each do |pool|
+          running = Vmpooler::API.settings.redis.scard('vmpooler__running__' + pool['name'])
+          pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
+          result[pool['major']] ||= {}
+          result[pool['major']]['running'] = result[pool['major']]['running'].to_i + running.to_i
+        end
+
+        if params[:history]
+          if graph_url
+            begin
+              buffer = open(graph_link('.running.*&from=-1hour&format=json')).read
+              JSON.parse(buffer).each do |pool|
+                if pool['target'] =~ /.*\.(.*)$/
+                  pool['name'] = Regexp.last_match[1]
+                  pool['major'] = Regexp.last_match[1] if pool['name'] =~ /^(\w+)\-/
+                  result[pool['major']]['history'] ||= []
+
+                  for i in 0..pool['datapoints'].length
+                    if pool['datapoints'][i] && pool['datapoints'][i][0]
+                      pool['last'] = pool['datapoints'][i][0]
+                      result[pool['major']]['history'][i] ||= 0
+                      result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['datapoints'][i][0].to_i
+                    else
+                      result[pool['major']]['history'][i] = result[pool['major']]['history'][i].to_i + pool['last'].to_i
+                    end
+                  end
+                end
+              end
+            rescue
+            end
+          end
+        end
+        JSON.pretty_generate(result)
+      end
+    end
+  end
+end

data/lib/vmpooler/api/helpers.rb

@@ -0,0 +1,431 @@
+module Vmpooler
+
+  class API
+
+    module Helpers
+
+      def has_token?
+        request.env['HTTP_X_AUTH_TOKEN'].nil? ? false : true
+      end
+
+      def valid_token?(backend)
+        return false unless has_token?
+
+        backend.exists('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN']) ? true : false
+      end
+
+      def validate_token(backend)
+        if valid_token?(backend)
+          backend.hset('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'last', Time.now)
+
+          return true
+        end
+
+        content_type :json
+
+        result = { 'ok' => false }
+
+        headers['WWW-Authenticate'] = 'Basic realm="Authentication required"'
+        halt 401, JSON.pretty_generate(result)
+      end
+
+      def validate_auth(backend)
+        return if authorized?
+
+        content_type :json
+
+        result = { 'ok' => false }
+
+        headers['WWW-Authenticate'] = 'Basic realm="Authentication required"'
+        halt 401, JSON.pretty_generate(result)
+      end
+
+      def authorized?
+        @auth ||= Rack::Auth::Basic::Request.new(request.env)
+
+        if @auth.provided? and @auth.basic? and @auth.credentials
+          username, password = @auth.credentials
+
+          if authenticate(Vmpooler::API.settings.config[:auth], username, password)
+            return true
+          end
+        end
+
+        return false
+      end
+
+      def authenticate_ldap(port, host, user_object, base, username_str, password_str)
+        ldap = Net::LDAP.new(
+          :host => host,
+          :port => port,
+          :encryption => {
+            :method => :start_tls,
+            :tls_options => { :ssl_version => 'TLSv1' }
+          },
+          :base => base,
+          :auth => {
+            :method => :simple,
+            :username => "#{user_object}=#{username_str},#{base}",
+            :password => password_str
+          }
+        )
+
+        return true if ldap.bind
+        return false
+      end
+
+      def authenticate(auth, username_str, password_str)
+        case auth['provider']
+          when 'dummy'
+            return (username_str != password_str)
+          when 'ldap'
+            ldap_base = auth[:ldap]['base']
+            ldap_port = auth[:ldap]['port'] || 389
+
+            if ldap_base.is_a? Array
+              ldap_base.each do |search_base|
+                result = authenticate_ldap(
+                  ldap_port,
+                  auth[:ldap]['host'],
+                  auth[:ldap]['user_object'],
+                  search_base,
+                  username_str,
+                  password_str,
+                )
+                return true if result == true
+              end
+            else
+              result = authenticate_ldap(
+                ldap_port,
+                auth[:ldap]['host'],
+                auth[:ldap]['user_object'],
+                ldap_base,
+                username_str,
+                password_str,
+              )
+              return result
+            end
+
+            return false
+          end
+      end
+
+      def export_tags(backend, hostname, tags)
+        tags.each_pair do |tag, value|
+          next if value.nil? or value.empty?
+
+          backend.hset('vmpooler__vm__' + hostname, 'tag:' + tag, value)
+          backend.hset('vmpooler__tag__' + Date.today.to_s, hostname + ':' + tag, value)
+        end
+      end
+
+      def filter_tags(tags)
+        return unless Vmpooler::API.settings.config[:tagfilter]
+
+        tags.each_pair do |tag, value|
+          next unless filter = Vmpooler::API.settings.config[:tagfilter][tag]
+          tags[tag] = value.match(filter).captures.join if value.match(filter)
+        end
+
+        tags
+      end
+
+      def mean(list)
+        s = list.map(&:to_f).reduce(:+).to_f
+        (s > 0 && list.length > 0) ? s / list.length.to_f : 0
+      end
+
+      def validate_date_str(date_str)
+        /^\d{4}-\d{2}-\d{2}$/ === date_str
+      end
+
+      def hostname_shorten(hostname, domain=nil)
+        if domain && hostname =~ /^\w+\.#{domain}$/
+          hostname = hostname[/[^\.]+/]
+        end
+
+        hostname
+      end
+
+      def get_task_times(backend, task, date_str)
+        backend.hvals("vmpooler__#{task}__" + date_str).map(&:to_f)
+      end
+
+      def get_capacity_metrics(pools, backend)
+        capacity = {
+            current: 0,
+            total:   0,
+            percent: 0
+        }
+
+        pools.each do |pool|
+          pool['capacity'] = backend.scard('vmpooler__ready__' + pool['name']).to_i
+
+          capacity[:current] += pool['capacity']
+          capacity[:total]   += pool['size'].to_i
+        end
+
+        if capacity[:total] > 0
+          capacity[:percent] = ((capacity[:current].to_f / capacity[:total].to_f) * 100.0).round(1)
+        end
+
+        capacity
+      end
+
+      def get_queue_metrics(pools, backend)
+        queue = {
+            pending:   0,
+            cloning:   0,
+            booting:   0,
+            ready:     0,
+            running:   0,
+            completed: 0,
+            total:     0
+        }
+
+        pools.each do |pool|
+          queue[:pending]   += backend.scard('vmpooler__pending__' + pool['name']).to_i
+          queue[:ready]     += backend.scard('vmpooler__ready__' + pool['name']).to_i
+          queue[:running]   += backend.scard('vmpooler__running__' + pool['name']).to_i
+          queue[:completed] += backend.scard('vmpooler__completed__' + pool['name']).to_i
+        end
+
+        queue[:cloning] = backend.get('vmpooler__tasks__clone').to_i
+        queue[:booting] = queue[:pending].to_i - queue[:cloning].to_i
+        queue[:booting] = 0 if queue[:booting] < 0
+        queue[:total]   = queue[:pending].to_i + queue[:ready].to_i + queue[:running].to_i + queue[:completed].to_i
+
+        queue
+      end
+
+      def get_tag_metrics(backend, date_str, opts = {})
+        opts = {:only => false}.merge(opts)
+
+        tags = {}
+
+        backend.hgetall('vmpooler__tag__' + date_str).each do |key, value|
+          hostname = 'unknown'
+          tag = 'unknown'
+
+          if key =~ /\:/
+            hostname, tag = key.split(':', 2)
+          end
+
+          if opts[:only]
+            next unless tag == opts[:only]
+          end
+
+          tags[tag] ||= {}
+          tags[tag][value] ||= 0
+          tags[tag][value] += 1
+
+          tags[tag]['total'] ||= 0
+          tags[tag]['total'] += 1
+        end
+
+        tags
+      end
+
+      def get_tag_summary(backend, from_date, to_date, opts = {})
+        opts = {:only => false}.merge(opts)
+
+        result = {
+          tag: {},
+          daily: []
+        }
+
+        (from_date..to_date).each do |date|
+          daily = {
+            date: date.to_s,
+            tag: get_tag_metrics(backend, date.to_s, opts)
+          }
+          result[:daily].push(daily)
+        end
+
+        result[:daily].each do |daily|
+          daily[:tag].each_key do |tag|
+            result[:tag][tag] ||= {}
+
+            daily[:tag][tag].each do |key, value|
+              result[:tag][tag][key] ||= 0
+              result[:tag][tag][key] += value
+            end
+          end
+        end
+
+        result
+      end
+
+      def get_task_metrics(backend, task_str, date_str, opts = {})
+        opts = {:bypool => false, :only => false}.merge(opts)
+
+        task = {
+            duration: {
+                average: 0,
+                min:     0,
+                max:     0,
+                total:   0
+            },
+            count:    {
+                total: 0
+            }
+        }
+
+        task[:count][:total] = backend.hlen('vmpooler__' + task_str + '__' + date_str).to_i
+
+        if task[:count][:total] > 0
+          if opts[:bypool] == true
+            task_times_bypool = {}
+
+            task[:count][:pool]    = {}
+            task[:duration][:pool] = {}
+
+            backend.hgetall('vmpooler__' + task_str + '__' + date_str).each do |key, value|
+              pool     = 'unknown'
+              hostname = 'unknown'
+
+              if key =~ /\:/
+                pool, hostname = key.split(':')
+              else
+                hostname = key
+              end
+
+              task[:count][:pool][pool]    ||= {}
+              task[:duration][:pool][pool] ||= {}
+
+              task_times_bypool[pool] ||= []
+              task_times_bypool[pool].push(value.to_f)
+            end
+
+            task_times_bypool.each_key do |pool|
+              task[:count][:pool][pool][:total] = task_times_bypool[pool].length
+
+              task[:duration][:pool][pool][:total]                                   = task_times_bypool[pool].reduce(:+).to_f
+              task[:duration][:pool][pool][:average]                                 = (task[:duration][:pool][pool][:total] / task[:count][:pool][pool][:total]).round(1)
+              task[:duration][:pool][pool][:min], task[:duration][:pool][pool][:max] = task_times_bypool[pool].minmax
+            end
+          end
+
+          task_times = get_task_times(backend, task_str, date_str)
+
+          task[:duration][:total]                      = task_times.reduce(:+).to_f
+          task[:duration][:average]                    = (task[:duration][:total] / task[:count][:total]).round(1)
+          task[:duration][:min], task[:duration][:max] = task_times.minmax
+        end
+
+        if opts[:only]
+          task.each_key do |key|
+            task.delete(key) unless key.to_s == opts[:only]
+          end
+        end
+
+        task
+      end
+
+      def get_task_summary(backend, task_str, from_date, to_date, opts = {})
+        opts = {:bypool => false, :only => false}.merge(opts)
+
+        task_sym = task_str.to_sym
+
+        result = {
+          task_sym => {},
+          daily: []
+        }
+
+        (from_date..to_date).each do |date|
+          daily = {
+            date: date.to_s,
+            task_sym => get_task_metrics(backend, task_str, date.to_s, opts)
+          }
+          result[:daily].push(daily)
+        end
+
+        daily_task = {}
+        daily_task_bypool = {} if opts[:bypool] == true
+
+        result[:daily].each do |daily|
+          daily[task_sym].each_key do |type|
+            result[task_sym][type] ||= {}
+            daily_task[type] ||= {}
+
+            ['min', 'max'].each do |key|
+              if daily[task_sym][type][key]
+                daily_task[type][:data] ||= []
+                daily_task[type][:data].push(daily[task_sym][type][key])
+              end
+            end
+
+            result[task_sym][type][:total] ||= 0
+            result[task_sym][type][:total] += daily[task_sym][type][:total]
+
+            if opts[:bypool] == true
+              result[task_sym][type][:pool] ||= {}
+              daily_task_bypool[type] ||= {}
+
+              next unless daily[task_sym][type][:pool]
+
+              daily[task_sym][type][:pool].each_key do |pool|
+                result[task_sym][type][:pool][pool] ||= {}
+                daily_task_bypool[type][pool] ||= {}
+
+                ['min', 'max'].each do |key|
+                  if daily[task_sym][type][:pool][pool][key.to_sym]
+                    daily_task_bypool[type][pool][:data] ||= []
+                    daily_task_bypool[type][pool][:data].push(daily[task_sym][type][:pool][pool][key.to_sym])
+                  end
+                end
+
+                result[task_sym][type][:pool][pool][:total] ||= 0
+                result[task_sym][type][:pool][pool][:total] += daily[task_sym][type][:pool][pool][:total]
+              end
+            end
+          end
+        end
+
+        result[task_sym].each_key do |type|
+          if daily_task[type][:data]
+            result[task_sym][type][:min], result[task_sym][type][:max] = daily_task[type][:data].minmax
+            result[task_sym][type][:average] = mean(daily_task[type][:data])
+          end
+
+          if opts[:bypool] == true
+            result[task_sym].each_key do |type|
+              result[task_sym][type][:pool].each_key do |pool|
+                if daily_task_bypool[type][pool][:data]
+                  result[task_sym][type][:pool][pool][:min], result[task_sym][type][:pool][pool][:max] = daily_task_bypool[type][pool][:data].minmax
+                  result[task_sym][type][:pool][pool][:average] = mean(daily_task_bypool[type][pool][:data])
+                end
+              end
+            end
+          end
+        end
+
+        result
+      end
+
+      def pool_index(pools)
+        pools_hash = {}
+        index = 0
+        for pool in pools
+          pools_hash[pool['name']] = index
+          index += 1
+        end
+        pools_hash
+      end
+
+      def template_ready?(pool, backend)
+        prepared_template = backend.hget('vmpooler__template__prepared', pool['name'])
+        return false if prepared_template.nil?
+        return true if pool['template'] == prepared_template
+        return false
+      end
+
+      def is_integer?(x)
+        Integer(x)
+        true
+      rescue
+        false
+      end
+    end
+  end
+end