vcap_common 1.0.10

data/lib/vcap/priority_queue.rb

@@ -0,0 +1,164 @@
+#a priority queue with the added twist of FIFO behavior for elements with equal priorities
+#implementation using binary max-heap on top of a ruby array.
+#the FIFO behavior is implemented by storing a FIFO bucket of same-priority values
+
+#The implementation is not meant to be high-performance, just decent, with two goals:
+#1. clean interface
+#2. proper time/space complexity of a binary heap
+#3. no silly memory leaks (Ah, three weapons of the Spanish Inquisition)
+
+#additionally, we implement a PrioritySet, that is a PriorityQueue
+#into which an element can only be inserted once. This PrioritySet
+#allows specifying identity for the object with a separate object.
+#The identity is for determining whether an object being inserted is a
+#duplicate, e.g.
+
+#     q.insert("boo", 1, "key")
+#     q.insert("zah", 1, "key")
+
+#will result in just one object in the queue, "boo"
+#
+#See spec/unit/priority_queue_set for
+#other examples
+
+require 'set'
+require 'pp'
+
+module VCAP
+  class PriorityQueueFIFO
+
+    attr_reader :size
+
+    def initialize
+      @heap_arr = []
+      @p2b = {} #hash mapping priorities to buckets
+      @size = 0
+    end
+
+    def empty?
+      size == 0
+    end
+
+    def insert(item, priority = 0)
+      raise ArgumentError, "priority can not be negative: #{priority}" if priority < 0
+
+      unless append_to_existing_priority_bucket(item, priority)
+        add_bucket_at_the_end_and_shift_up(item, priority)
+      end
+      @size += 1
+    end
+
+    def remove
+      return nil if empty?
+      bucket = top_bucket
+      priority = top_priority
+      elem = bucket.shift
+      @size -= 1
+      if empty?
+        @heap_arr.clear
+        @p2b.clear
+      elsif bucket.empty?
+        @heap_arr[0] = @heap_arr.pop
+        @p2b.delete(priority)
+        shift_down
+      else
+        #do nothing, we just shifted a value from a bucket and it still isn't empty, so no rearrangement is needed
+      end
+      elem
+    end
+
+    private
+
+    def add_bucket_at_the_end_and_shift_up(item, priority)
+      bucket = [item]
+      @p2b[priority] = bucket
+
+      #normal binary heap operation
+      @heap_arr.push priority
+      shift_up
+    end
+
+    def append_to_existing_priority_bucket(item, priority)
+      return false unless @p2b[priority]
+      @p2b[priority] << item
+      return true
+    end
+
+    def top_bucket
+      @p2b[top_priority]
+    end
+
+    def top_priority
+      priority_at(0)
+    end
+
+    def priority_at(index)
+      return -1 if index >= @heap_arr.size
+      @heap_arr[index]
+    end
+
+    def parent_index(index)
+      (index+1) / 2 - 1
+    end
+
+    def left_child_index(index)
+      (index+1) * 2 - 1
+    end
+
+    def right_child_index(index)
+      (index+1) * 2
+    end
+
+    def any_children_at?(index)
+      left_child_index(index) < @heap_arr.length
+    end
+
+    def shift_up
+      cur_index = @heap_arr.length - 1
+      while cur_index > 0 && priority_at(cur_index) > priority_at(parent_index(cur_index)) do
+        next_cur_index = parent_index cur_index
+        swap_at(cur_index, next_cur_index)
+        cur_index = next_cur_index
+      end
+    end
+
+    def index_of_max_priority_child_at(index)
+      #raise(ArgumentError, "no children at #{index}") unless any_children_at?(index)
+      l = left_child_index(index)
+      r = right_child_index(index)
+      return r if priority_at(r) > priority_at(l) #this is safe since priority will return -1 for non-existent right child
+      return l
+    end
+
+    def shift_down
+      cur_index = 0
+      while any_children_at?(cur_index) && priority_at(cur_index) < priority_at(index_of_max_priority_child_at(cur_index)) do
+        next_cur_index = index_of_max_priority_child_at cur_index
+        swap_at(cur_index, next_cur_index)
+        cur_index = next_cur_index
+      end
+    end
+
+    def swap_at(i,j)
+      @heap_arr[i], @heap_arr[j] = @heap_arr[j], @heap_arr[i]
+    end
+  end
+
+
+  class PrioritySet < PriorityQueueFIFO
+    def initialize
+      super
+      @set = Set.new #the set is used to check for duplicates
+    end
+
+    def insert(elem, priority = 0, key = nil)
+      super([elem,key], priority) if @set.add?(key || elem)
+    end
+
+    def remove
+      elem, key = super
+      @set.delete(key || elem)
+      elem
+    end
+  end
+end

data/lib/vcap/process_utils.rb

@@ -0,0 +1,43 @@
+require 'thread'
+
+require 'vcap/subprocess'
+
+module VCAP
+  module ProcessUtils
+    STAT_FIELDS = [
+      {:name => :rss,   :parse_method => :to_i},
+      {:name => :vsize, :parse_method => :to_i},
+      {:name => :pcpu,  :parse_method => :to_f},
+    ]
+
+    class << self
+
+      def get_stats(pid=nil)
+        pid ||= Process.pid
+
+        flags = STAT_FIELDS.map {|f| "-o #{f[:name]}=" }.join(' ')
+        begin
+          stdout, stderr, status = VCAP::Subprocess.run("ps #{flags} -p #{pid}")
+        rescue VCAP::SubprocessStatusError => se
+          # Process not running
+          if se.status.exitstatus == 1
+            return nil
+          else
+            raise se
+          end
+        end
+
+        ret = {}
+        stdout.split.each_with_index do |val, ii|
+          field = STAT_FIELDS[ii]
+          ret[field[:name]] = val.send(field[:parse_method])
+        end
+
+        ret
+      end
+
+    end # class << self
+  end   # ProcessUtils
+end     # VCAP
+
+

data/lib/vcap/quota.rb

@@ -0,0 +1,152 @@
+module VCAP
+  module Quota
+
+    class Command
+      class ValidationError < StandardError; end
+
+      def run
+        validate
+        command = build_command
+        result = execute(command)
+        parse_result(result)
+      end
+
+      def validate
+        nil
+      end
+
+      def build_command
+        raise NotImplementedError
+      end
+
+      def execute(command)
+        stdout = `#{command}`
+        [$?, stdout]
+      end
+
+      def parse_result(result)
+        result
+      end
+
+      private
+
+      def assert_at_least_one_of(*fields)
+        for field in fields
+          has_value = send(field.to_sym)
+          return if has_value
+        end
+        raise ValidationError, "At least one of {#{fields.join(', ')}} must be set"
+      end
+
+      def assert_at_most_one_of(*fields)
+        existing_fields = fields.inject([]) do |accum, field|
+          accum << field unless send(field.to_sym)
+          accum
+        end
+        unless existing_fields.length == 1
+          raise ValidationError, "At most one of #{fields.join(', ')} must be set"
+        end
+      end
+    end
+
+    class SetQuota < Command
+      attr_accessor :user
+      attr_accessor :group
+      attr_accessor :filesystem
+      attr_accessor :quotas
+
+      def initialize
+        @quotas = {
+          :block => {
+            :soft => 0,
+            :hard => 0,
+          },
+          :inode => {
+            :soft => 0,
+            :hard => 0,
+          },
+        }
+      end
+
+      def validate
+        assert_at_least_one_of(:user)
+        assert_at_least_one_of(:filesystem)
+        assert_at_least_one_of(:quotas)
+      end
+
+      private
+
+      def build_command
+        cmd = ['setquota']
+        cmd << ['-u', self.user] if self.user
+        cmd << ['-g', self.group] if self.group
+        cmd << [self.quotas[:block][:soft], self.quotas[:block][:hard],
+                self.quotas[:inode][:soft], self.quotas[:inode][:hard]]
+        cmd << self.filesystem
+        cmd.flatten.join(' ')
+      end
+    end
+
+    class RepQuota < Command
+      attr_accessor :report_groups
+      attr_accessor :report_users
+      attr_accessor :ids_only
+      attr_accessor :filesystem
+
+      def initialize
+        @report_users = true
+      end
+
+      def validate
+        assert_at_least_one_of(:report_groups, :report_users)
+        assert_at_least_one_of(:filesystem)
+      end
+
+      def build_command
+        cmd = ['repquota', '-p'] # -p reports grace as 0 when unset
+        cmd << '-u' if self.report_users
+        cmd << '-g' if self.report_groups
+        cmd << '-n' if self.ids_only
+        cmd << self.filesystem
+        cmd = cmd.flatten.join(' ')
+        cmd
+      end
+
+      def parse_result(result)
+        if result[0] == 0
+          quota_info = {}
+          result[1].lines.each do |line|
+            next unless line.match(/[^\s]+\s+[+-]+\s+\d+/)
+            fields = line.split(/\s+/)
+            if self.ids_only
+              match = fields[0].match(/^#(\d+)$/)
+              uid = match[1].to_i
+            else
+              uid = fields[0]
+            end
+            quota_info[uid] = {
+              :usage => {
+                :block => fields[2].to_i,
+                :inode => fields[6].to_i
+              },
+              :quotas => {
+                :block => {
+                  :soft => fields[3].to_i,
+                  :hard => fields[4].to_i,
+                },
+                :inode => {
+                  :soft => fields[7].to_i,
+                  :hard => fields[8].to_i,
+                },
+              }
+            }
+          end
+          [true, quota_info]
+        else
+          [false, result]
+        end
+      end
+    end
+
+  end # VCAP::Quota
+end

data/lib/vcap/rolling_metric.rb

@@ -0,0 +1,74 @@
+require "monitor"
+
+module VCAP
+
+  class RollingMetric
+
+    def initialize(duration, num_buckets = 60)
+      @duration = duration
+      num_buckets = [@duration, num_buckets].min
+      @bucket_duration = (@duration / num_buckets).to_i
+      @eviction_duration = @bucket_duration * 2
+      @buckets = []
+      num_buckets.times do
+        @buckets << {:timestamp => 0, :value => 0, :samples => 0}
+      end
+    end
+
+    def <<(value)
+      timestamp = Time.now.to_i
+      bucket = @buckets[(timestamp / @bucket_duration) % @buckets.length]
+      if timestamp - bucket[:timestamp] > @eviction_duration
+        bucket[:timestamp] = timestamp
+        bucket[:value] = value
+        bucket[:samples] = 1
+      else
+        bucket[:value] += value
+        bucket[:samples] += 1
+      end
+    end
+
+    def value
+      timestamp = Time.now.to_i
+      min_timestamp = timestamp - @duration
+
+      value = 0
+      samples = 0
+
+      @buckets.each do |bucket|
+        if bucket[:timestamp] > min_timestamp
+          value += bucket[:value]
+          samples += bucket[:samples]
+        end
+      end
+
+      {
+        :value => value,
+        :samples => samples
+      }
+    end
+
+    def to_json
+      Yajl::Encoder.encode(value)
+    end
+
+  end
+
+  class ThreadSafeRollingMetric < RollingMetric
+
+    def initialize(*args)
+      super(*args)
+      @lock = Monitor.new
+    end
+
+    def <<(*args)
+      @lock.synchronize { super(*args) }
+    end
+
+    def value(*args)
+      @lock.synchronize { super(*args) }
+    end
+
+  end
+
+end

data/lib/vcap/spec/em.rb

@@ -0,0 +1,32 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+module VCAP
+
+  module Spec
+
+    module EM
+
+      def em(options = {})
+        raise "no block given" unless block_given?
+        timeout = options[:timeout] ||= 1.0
+
+        ::EM.run {
+          quantum = 0.005
+          ::EM.set_quantum(quantum * 1000) # Lowest possible timer resolution
+          ::EM.set_heartbeat_interval(quantum) # Timeout connections asap
+          ::EM.add_timer(timeout) { raise "timeout" }
+          yield
+        }
+      end
+
+      def done
+        raise "reactor not running" if !::EM.reactor_running?
+
+        ::EM.next_tick {
+          # Assert something to show a spec-pass
+          :done.should == :done
+          ::EM.stop_event_loop
+        }
+      end
+    end
+  end
+end

data/lib/vcap/spec/forked_component/base.rb

@@ -0,0 +1,87 @@
+require 'vcap/common'
+
+
+module VCAP
+  module Spec
+    module ForkedComponent
+    end
+  end
+end
+
+class VCAP::Spec::ForkedComponent::Base
+  attr_reader :pid, :pid_filename, :output_basedir, :name, :cmd
+
+  attr_accessor :reopen_stdio
+
+  # @param  cmd             String  Command to run
+  # @param  name            String  Short name for this component (e.g. 'redis')
+  # @param  output_basedir  String  Stderr/stdout will be placed under this directory
+  # @param  pid_filename    String  If not nil, we ready the pid from this file instead
+  #                                 of using the pid returned from fork
+  def initialize(cmd, name, output_basedir='/tmp', pid_filename=nil)
+    @cmd  = cmd
+    @name = name
+    @output_basedir = output_basedir
+    @pid_filename   = pid_filename
+
+    @reopen_stdio = true
+
+  end
+
+  def start
+    pid = fork do
+
+      if @reopen_stdio
+        fn = File.join(@output_basedir, "#{@name}.#{Process.pid}.out")
+        outfile = File.new(fn, 'w+')
+        $stderr.reopen(outfile)
+        $stdout.reopen(outfile)
+      end
+
+      exec(@cmd)
+    end
+
+    if @pid_filename
+      wait_for(5) { File.exists?(@pid_filename) }
+      @pid = File.read(@pid_filename).chomp.to_i
+    else
+      @pid = pid
+    end
+
+    self
+  end
+
+  def stop
+    return unless @pid && VCAP.process_running?(@pid)
+    Process.kill('TERM', @pid)
+    Process.waitpid(@pid, 0)
+    FileUtils.rm_f(@pid_filename) if @pid_filename
+    @pid = nil
+
+    self
+  end
+
+  def running?
+    VCAP.process_running?(@pid)
+  end
+
+  def ready?
+    raise NotImplementedError
+  end
+
+  def wait_ready(timeout=1)
+    wait_for { ready? }
+  end
+
+  private
+
+  def wait_for(timeout=1, &predicate)
+    start = Time.now()
+    cond_met = predicate.call()
+    while !cond_met && ((Time.new() - start) < timeout)
+      cond_met = predicate.call()
+      sleep(0.2)
+    end
+    cond_met
+  end
+end

data/lib/vcap/spec/forked_component/nats_server.rb

@@ -0,0 +1,28 @@
+require 'nats/client'
+require 'uri'
+
+require 'vcap/spec/forked_component/base'
+
+module VCAP
+  module Spec
+    module ForkedComponent
+    end
+  end
+end
+
+class VCAP::Spec::ForkedComponent::NatsServer < VCAP::Spec::ForkedComponent::Base
+
+  attr_reader :uri, :port, :parsed_uri
+
+  def initialize(pid_filename, port, output_basedir='tmp')
+    cmd = "ruby -S bundle exec nats-server -p #{port} -P #{pid_filename} -V -D"
+    super(cmd, 'nats', output_basedir, pid_filename)
+    @port = port
+    @uri  = "nats://127.0.0.1:#{@port}"
+    @parsed_uri = URI.parse(@uri)
+  end
+
+  def ready?
+    running? && NATS.server_running?(@parsed_uri)
+  end
+end

data/lib/vcap/spec/forked_component.rb

@@ -0,0 +1,2 @@
+require 'vcap/spec/forked_component/base'
+require 'vcap/spec/forked_component/nats_server'