vpsb_client 0.0.2

data/lib/vpsb_client/builders/system_info_parser.rb

@@ -0,0 +1,127 @@
+module VpsbClient
+  module Builders
+    class SystemInfoParser
+      class NoMatchError < StandardError; end
+
+      def initialize(cmd)
+        @cmd = cmd
+      end
+
+      def lines
+        IO.popen(@cmd) { |f| f.readlines }
+      end
+
+      def find_matches(regex)
+        regex = Regexp.new(regex)
+        matches = nil
+        lines.each do |line|
+          matches = regex.match(line)
+          break if matches
+        end
+        matches
+      end
+
+      def find_matches!(regex)
+        matches = find_matches(regex)
+        raise NoMatchError, "Cannot find /#{regex}/" unless matches
+        matches
+      end
+    end
+
+    class MemoryParser < SystemInfoParser
+      attr_reader :used, :free, :total
+
+      REGEX_CACHE = 'cache:\s+(?<used>\d+)\s+(?<free>\d+)$'
+      REGEX_AVAIL = '^Mem:\s+\d+\s+(?<used>\d+)\s+\d+\s+\d+\s+\d+\s+(?<avail>\d+)'
+      REGEX_TOTAL = '^Mem:\s+(?<total>\d+)'
+
+      def initialize
+        super('free')
+      end
+
+      def parse
+        matches = find_matches(REGEX_CACHE)
+        if matches
+          @used = matches[:used].to_i
+          @free = matches[:free].to_i
+        elsif matches = find_matches(REGEX_AVAIL)
+          @used = matches[:used].to_i
+          @free = matches[:avail].to_i
+        else
+          raise NoMatchError, "No regex matches 'free' output"     
+        end
+        matches = find_matches!(REGEX_TOTAL)
+        @total = matches[:total].to_i
+      end
+    end
+
+    class UnameParser < SystemInfoParser
+      attr_reader :kernel, :os_type
+      # Linux lino 3.12.6-x86_64-linode36 #2 SMP Mon Jan 13 18:54:10 EST 2014 x86_64 x86_64 x86_64 GNU/Linux
+      REGEX = Regexp.new(/\w+ \S+ (?<kernel>\d+\.\d+\.\d+)-(?<type>[^\-\s]+)[\-\s]/)
+
+      def initialize
+        super('uname -a')
+      end
+
+      def parse
+        matches = find_matches!(REGEX)
+        @kernel  = matches[:kernel]
+        @os_type = matches[:type]
+      end
+    end
+
+    class CpuinfoParser < SystemInfoParser
+      attr_reader :model, :num, :mhz
+      # model name  : Intel(R) Xeon(R) CPU E5-2670 0 @ 2.60GHz
+      REGEX = Regexp.new('^model name\s*:\s*(?<model>.*$)')
+      REGEX_PROCESSOR = Regexp.new('^Processor\s*:\s*(?<model>.*$)')
+
+      def initialize
+        super('cat /proc/cpuinfo')
+      end
+
+      def parse
+        matches = find_matches(REGEX)
+        matches = find_matches!(REGEX_PROCESSOR) unless matches
+        @model = matches[:model]
+        parse_num_processors
+        parse_cpu_speed
+      end
+
+      private
+      def parse_num_processors
+        @num = 0
+        lines.each do |line|
+          if line =~ /^processor\s*:\s*(?<num>\d+)/
+            @num += 1
+          end
+        end
+      end
+
+      def parse_cpu_speed
+        matches = find_matches(/^cpu MHz\s*:\s*(?<mhz>\d+)/)
+        if matches
+          @mhz = matches[:mhz].to_i
+        else
+          @mhz = nil
+        end
+      end
+    end
+
+    class IssueParser < SystemInfoParser
+      attr_reader :os
+
+      REGEX = '^(?<os>[^\\\\]+)(?: \\\\n \\\\l)?$'
+
+      def initialize
+        super('cat /etc/issue')
+      end
+
+      def parse
+        matches = find_matches!(REGEX)
+        @os = matches[:os]
+      end
+    end
+  end
+end

data/lib/vpsb_client/builders/trial.rb

@@ -0,0 +1,54 @@
+module VpsbClient
+  module Builders
+    class Trial
+      def initialize(config, hoster_id=nil, application_id=nil, plan_id=nil)
+        @config = config
+        @hoster_id = hoster_id
+        @application_id = application_id
+        @plan_id = plan_id
+      end
+
+      def lookup_params
+        trial_params = {}
+        trial_params[:client_hostname] = @config['client_hostname']
+        trial_params
+      end
+
+      def create_params
+        trial_params = {}
+        trial_params[:started_at] = Time.now
+        trial_params[:hoster_id] = @hoster_id if @hoster_id
+        trial_params[:application_id] = @application_id if @application_id
+        trial_params[:plan_id] = @plan_id if @plan_id
+
+        trial_params[:comment] = @config['comment']
+
+        cpuinfo_parser = Builders::CpuinfoParser.new
+        cpuinfo_parser.parse
+        trial_params[:cpu_type] = cpuinfo_parser.model
+        trial_params[:num_cores] = cpuinfo_parser.num
+        trial_params[:cpu_mhz] = cpuinfo_parser.mhz
+
+        issue_parser = Builders::IssueParser.new
+        issue_parser.parse
+        trial_params[:os] = issue_parser.os
+
+        memory_parser = Builders::MemoryParser.new
+        memory_parser.parse
+        trial_params[:free_memory_mb] = memory_parser.free
+
+        uname_parser = Builders::UnameParser.new
+        uname_parser.parse
+        trial_params[:kernel] = uname_parser.kernel
+
+        trial_params[:client_hostname] = @config['client_hostname']
+        trial_params[:ruby_version] = RUBY_VERSION
+        trial_params[:rails_version] = defined?(Rails) ? Rails.version : nil
+
+        trial_params[:datacenter] = @config['datacenter']
+
+        trial_params
+      end
+    end
+  end
+end

data/lib/vpsb_client/client/upload_metrics.rb

@@ -0,0 +1,39 @@
+module VpsbClient
+  module Client
+    module UploadMetrics
+      def upload_metrics(trial)
+        unless enabled?
+          logger.debug "not running because vpsb_client is disabled"
+          return
+        end
+
+        prepare_logfiles
+
+        metric_ids = []
+        [ 10*60, 3600, 86400 ].each do |interval_length|
+          metric_ids += upload_for_interval_length(trial, interval_length)
+        end
+        metric_ids
+      end
+
+      private
+
+      def upload_for_interval_length(trial, interval_length)
+        last_started_at = trial_last_metric_started_at(trial['id'], interval_length)
+        if last_started_at
+          start_time = last_started_at + interval_length
+          force = true
+        else
+          trial_started_at = DateTime.parse(trial['started_at']).to_time
+          start_time = trial_started_at
+          force = false
+        end
+        logger.debug "[vpsb] upload_metrics: length=#{interval_length} start_time=#{start_time} force=#{force}"
+        interval_config = Metrics::IntervalConfig.new(start_time, interval_length, force: force)
+        metrics_manager = metrics_manager(trial['id'], interval_config)
+        metrics_manager.run
+        metrics_manager.created_metric_ids
+      end
+    end
+  end
+end

data/lib/vpsb_client/config.rb

@@ -0,0 +1,18 @@
+require 'yaml'
+
+module VpsbClient
+  class Config
+    def initialize(path)
+      raise ArgumentError, "Can't find #{path}" unless File.exist?(path)
+      @yml = YAML.load_file(path)
+    end
+
+    def fetch(name, default=nil)
+      default ? @yml.fetch(name.to_s, default) : @yml.fetch(name.to_s)
+    end
+
+    def [](name)
+      fetch(name.to_s)
+    end
+  end
+end

data/lib/vpsb_client/curl_wrapper.rb

@@ -0,0 +1,41 @@
+require 'curb'
+
+module VpsbClient
+  class CurlWrapper
+    def initialize(auth_token)
+      @auth_token = auth_token
+    end
+
+    def get(url, &block)
+      Curl.get(url) do |curl|
+        curl.ssl_verify_host = false
+        curl.ssl_verify_peer = false
+        curl.headers['Authorization'] = "Token #{@auth_token}"
+
+        yield curl if block_given?
+      end
+    end
+
+    def post(url, post_params, content_type, &block)
+      Curl.post(url, post_params) do |curl|
+        curl.ssl_verify_host = false
+        curl.ssl_verify_peer = false
+        curl.headers['content-type'] = content_type
+        curl.headers['Authorization'] = "Token #{@auth_token}"
+
+        yield curl if block_given?
+      end
+    end
+
+    def put(url, put_params, content_type, &block)
+      Curl.put(url, put_params) do |curl|
+        curl.ssl_verify_host = false
+        curl.ssl_verify_peer = false
+        curl.headers['content-type'] = content_type
+        curl.headers['Authorization'] = "Token #{@auth_token}"
+
+        yield curl if block_given?
+      end
+    end
+  end
+end

data/lib/vpsb_client/datafiles/formatted_sar_log_parser.rb

@@ -0,0 +1,38 @@
+require 'logfile_interval'
+require File.join(File.expand_path('..',  __FILE__), 'pxx_aggregator')
+
+module VpsbClient
+  module Datafiles
+    class FormattedSarLogParser < LogfileInterval::ParsedLine::Base
+
+      # # hostname;interval;timestamp;CPU;%user;%nice;%system;%iowait;%steal;%idle
+      # lino;300;1394780701;-1;4.19;0.00;0.40;0.87;5.49;89.05
+      set_regex /^\S+;\S+;(?<ts>\d+);\S+;(?<user>[\d\.]+);(?<nice>[\d\.]+);(?<system>[\d\.]+);(?<iowait>[\d\.]+);(?<steal>[\d\.]+);(?<idle>[\d\.]+)$/
+
+      PXX_RANGES = {
+        1  => 0.02,
+        10 => 0.1,
+        20 => 0.5,
+        100 => 1
+      }
+
+      add_column :name => 'timestamp', :pos => 1, :aggregator => :timestamp
+      add_column :name => 'user',      :pos => 2, :aggregator => :average, :conversion => :float
+      add_column :name => 'nice',      :pos => 3, :aggregator => :average, :conversion => :float
+      add_column :name => 'system',    :pos => 4, :aggregator => :average, :conversion => :float
+      add_column :name => 'iowait',    :pos => 5, :aggregator => :average, :conversion => :float
+      add_column :name => 'pxx_iowait',    :pos => 5, :aggregator => :pxx, :conversion => :float,
+                    :custom_options => { :ranges => PXX_RANGES, :pxx_keys => [ 75, 95, 99 ] }
+      add_column :name => 'cpu_steal',     :pos => 6, :aggregator => :average, :conversion => :float
+      add_column :name => 'pxx_cpusteal', :pos => 6, :aggregator => :pxx,     :conversion => :float,
+                    :custom_options => { :ranges => PXX_RANGES, :pxx_keys => [ 75, 95, 99 ] }
+      add_column :name => 'cpu_idle',      :pos => 7, :aggregator => :average, :conversion => :float
+      add_column :name => 'pxx_cpuidle',  :pos => 7, :aggregator => :one_hundred_minus_pxx, :conversion => :float,
+                    :custom_options => { :ranges => PXX_RANGES, :pxx_keys => [ 75, 95, 99 ] }
+
+      def time
+        Time.at(self.timestamp.to_i)
+      end
+    end
+  end
+end

data/lib/vpsb_client/datafiles/logfile_decompressor.rb

@@ -0,0 +1,48 @@
+module VpsbClient
+  module Datafiles
+    class LogfileDecompressor
+      class NotFoundError < StandardError; end
+
+      UNLIMITED_ROTATION_ID = 10000
+
+      def initialize(orig_path, target_path, options = {})
+        raise NotFoundError, "#{orig_path} is not a directory" unless File.directory?(orig_path)
+        @orig_path = orig_path
+        @target_path = target_path
+        @filename_prefix = options.fetch(:filename_prefix, '*')
+        @max_rotation_id = options.fetch(:max_rotation_id, UNLIMITED_ROTATION_ID)
+      end
+
+      def run
+        unzip
+      end
+
+      private
+
+      def unzip
+        timing_logs = Dir.glob("#{@orig_path}/#{@filename_prefix}.log.*.gz")
+        VpsbClient.logger.debug "Will gunzip #{timing_logs.inspect}"
+        timing_logs.each do |zipfile|
+          unzip_file(zipfile)
+        end
+      end
+
+      def unzip_file(zipfile)
+        if md = /[^\/]*\/(?<name>[^\/]+)\.(?<num>\d+)\.gz$/.match(zipfile)
+          return if md[:num].to_i > @max_rotation_id
+          unzipped_file = "#{@target_path}/#{md[:name]}.#{md[:num]}"
+        else
+          raise "Cannot convert to unzipped filename #{zipfile}"
+        end
+        VpsbClient.logger.debug "Will unzip #{zipfile} to #{unzipped_file}"
+        File.open(unzipped_file, 'w') do |fout|
+          File.open(zipfile) do |fin|
+            gz = Zlib::GzipReader.new(fin)
+            fout.write(gz.read)
+            gz.close
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vpsb_client/datafiles/pxx_aggregator.rb

@@ -0,0 +1,94 @@
+module LogfileInterval
+  module Aggregator
+    class Pxx < Base
+      def initialize(options)
+        super
+        @val = PxxBuckets.new(options)
+      end
+
+      def add(value, group_by = nil)
+        raise NotImplementedError if group_by
+        @val.increment(value)
+      end
+
+      def value(group)
+        @val.value(group.to_f / 100.0)
+      end
+
+      def single_value?
+        false
+      end
+    end
+
+    class OneHundredMinusPxx < Pxx
+      def add(value, group_by = nil)
+        raise NotImplementedError if group_by
+        @val.increment(100.0 - value)
+      end
+    end
+  end
+end
+
+class PxxBuckets
+  attr_reader :buckets, :ary, :total
+
+  def initialize(options)
+    i = 0
+    @ary = [ 0 ]
+    ranges = options.fetch(:ranges)
+    ranges.keys.sort.each do |ceiling|
+      while(i < ceiling)
+        i += ranges[ceiling]
+        i = i.round(2) if i.is_a?(Float) # float decimal errors
+        @ary << i
+      end
+    end
+
+    @buckets = {}
+    @ary.each do |min|
+      @buckets[min] = 0
+    end
+
+    @pxx_keys = options.fetch(:pxx_keys)
+    @total = 0
+  end
+
+  def keys
+    @pxx_keys
+  end
+
+  def each_key(&block)
+    keys.each(&block)
+  end
+
+  def [](key)
+    value(key.to_f/100.0)
+  end
+
+  def increment(n)
+    k = find_bucket_key(n)
+    @buckets[@ary[k]] += 1
+    @total += 1
+  end
+
+  def empty?
+    return total==0
+  end
+
+  def value(percent)
+    target = @total.to_f * (1.0 - percent)
+    sum = 0
+    @buckets.keys.sort.reverse.each do |floor|
+      sum += @buckets[floor]
+#       puts "target=#{target} floor=#{floor} sum=#{sum}"
+      return floor if sum >= target
+    end
+  end
+
+  def find_bucket_key(n)
+    idx = (0...@ary.size).bsearch do |i|
+      @ary[i] > n
+    end
+    idx ? idx - 1 : @ary.size - 1
+  end
+end

data/lib/vpsb_client/datafiles/sar_manager.rb

@@ -0,0 +1,71 @@
+module VpsbClient
+  module Datafiles
+    class SarManager
+      class NotFoundError < StandardError; end
+      class PermissionDeniedError < StandardError; end
+      class SadfError < StandardError; end
+
+      attr_reader :sadf_runner
+
+      def initialize(orig_path, target_path, sadf = Sadf)
+        raise NotFoundError, "#{orig_path} is not a directory" unless File.directory?(orig_path)
+        @orig_path = orig_path
+        @target_path = target_path
+        @sadf_runner = sadf
+      end
+
+      def run
+        create_target_path
+        create_daily_formatted
+        create_current_day_temp_formatted
+      end
+
+      private
+
+      def create_target_path
+        if File.directory?(@target_path)
+          raise PermissionDeniedError, "#{@target_path} is not writable" unless File.writable?(@target_path)
+          return
+        end
+        raise PermissionDeniedError unless File.writable?(File.dirname(@target_path))
+        Dir.mkdir(@target_path, 0755)
+      end
+
+      def create_daily_formatted
+        raw_sar_filenames = Dir.glob("#{@orig_path}/sa*")
+        raw_sar_filenames.each do |filename|
+          filename.match /sa(?<num>\d+)$/ do |matchdata|
+            fileday = matchdata[:num]
+            next if fileday.to_i == Time.now.day
+
+            formatted_filename = "#{@target_path}/formatted_sa#{fileday}"
+            next if File.exist?(formatted_filename)
+            sadf(filename, formatted_filename)
+          end
+        end
+      end
+
+      def create_current_day_temp_formatted
+        sa_filename = "#{@orig_path}/sa#{'%02d' % Time.now.day}"
+        formatted_filename = "#{@target_path}/formatted_sa#{'%02d' % Time.now.day}"
+        File.delete(formatted_filename) if File.exist?(formatted_filename)
+        sadf(sa_filename, formatted_filename)
+      end
+
+      def sadf(src, dest)
+        sadf_runner.run(src, dest)
+      end
+    end
+
+    class Sadf
+      SADF = '/usr/bin/sadf'
+
+      def self.run(src, dest)
+        raise NotFoundError unless File.executable?(SADF)
+        cmd = "#{SADF} -d #{src} -U > #{dest}"
+        ret = system cmd
+        raise VpsbClient::Datafiles::SarManager::SadfError, "\"#{cmd}\" failed" unless ret
+      end
+    end
+  end
+end

data/lib/vpsb_client/datafiles/timing_log_parser.rb

@@ -0,0 +1,32 @@
+require 'logfile_interval'
+require File.join(File.expand_path('..',  __FILE__), 'pxx_aggregator')
+
+module VpsbClient
+  module Datafiles
+    class TimingLogParser < LogfileInterval::ParsedLine::Base
+
+      # 1389477034 200 PackagesController#full_url              total=98 view=38.4 db=5.3 ip=66.249.76.30
+      set_regex /^(\d+)\s+(\d+)\s([\S]+)\s+total=([\d\.]+)\sview=([\d\.]+)\sdb=([\d\.]+)\sip=([\d\.]+)$/
+
+      PXX_RANGES = {
+        100 => 2,
+        200 => 10,
+        500 => 20,
+        1000 => 50,
+        3000 => 100
+      }
+
+      add_column :name => :timestamp,           :pos => 1, :aggregator => :timestamp
+      add_column :name => :resptime_total_ms,   :pos => 4, :aggregator => :average,   :conversion => :float
+      add_column :name => :resptime_view_ms,    :pos => 5, :aggregator => :average,   :conversion => :float
+      add_column :name => :resptime_db_ms,      :pos => 6, :aggregator => :average,   :conversion => :float
+      add_column :name => :num_requests,        :pos => 7, :aggregator => :num_lines
+      add_column :name => :pxx_total_ms,        :pos => 4, :aggregator => :pxx,       :conversion => :integer,
+                    :custom_options => { :ranges => PXX_RANGES, :pxx_keys => [ 50, 75, 95, 99 ] }
+
+      def time
+        Time.at(self.timestamp.to_i)
+      end
+    end
+  end
+end

data/lib/vpsb_client/http_client.rb

@@ -0,0 +1,60 @@
+module VpsbClient
+  class HttpClient
+    def initialize(curl_wrapper, protocol, hostname)
+      @protocol = protocol
+      @hostname = hostname
+      @curl_wrapper = curl_wrapper
+    end
+
+    def get(request)
+      @curl_wrapper.get(url(request))
+    end
+
+    def post(request)
+      post_params = post_params(request, request.content_type)
+      @curl_wrapper.post(url(request), post_params, request.content_type)
+    end
+
+    def put(request)
+      put_params = put_params(request, request.content_type)
+      @curl_wrapper.put(url(request), put_params, request.content_type)
+    end
+
+    private
+
+    def url(request)
+      query_string = url_encode(request.query_params)
+      "#{@protocol}://#{@hostname}#{request.url_path}#{suffix(request)}#{query_sep(query_string)}#{query_string}"
+    end
+
+    def query_sep(query_string)
+      query_string.empty? ? '' : '?'
+    end
+
+    def suffix(request)
+      '.json' if request.accept == 'application/json'
+    end
+
+    def post_params(request, content_type)
+      post_params = request.post_params
+      if request.content_type == 'application/json'
+        JSON.generate(post_params) # curl doesn't do the json encoding by itself
+      else
+        post_params # but curl does the www form encoding
+      end
+    end
+    alias_method :put_params, :post_params
+
+    def url_encode(params)
+      return '' if params.empty?
+      s = ''
+      i = 0
+      params.each do |k,v|
+        s << '&' unless i==0
+        s << "#{ERB::Util.url_encode(k)}=#{ERB::Util.url_encode(v)}"
+        i += 1
+      end
+      s
+    end
+  end
+end