vcap_common 1.0.10

data/lib/vcap/common.rb

@@ -0,0 +1,236 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require 'fileutils'
+require 'socket'
+
+# VMware's Cloud Application Platform
+
+module VCAP
+
+  A_ROOT_SERVER = '198.41.0.4'
+
+  def self.local_ip(route = A_ROOT_SERVER)
+    route ||= A_ROOT_SERVER
+    orig, Socket.do_not_reverse_lookup = Socket.do_not_reverse_lookup, true
+    UDPSocket.open {|s| s.connect(route, 1); s.addr.last }
+  ensure
+    Socket.do_not_reverse_lookup = orig
+  end
+
+  def self.secure_uuid
+    result = File.open('/dev/urandom') { |x| x.read(16).unpack('H*')[0] }
+  end
+
+  def self.grab_ephemeral_port
+    socket = TCPServer.new('0.0.0.0', 0)
+    socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, true)
+    Socket.do_not_reverse_lookup = true
+    port = socket.addr[1]
+    socket.close
+    return port
+  end
+
+  def self.uptime_string(delta)
+    num_seconds = delta.to_i
+    days = num_seconds / (60 * 60 * 24);
+    num_seconds -= days * (60 * 60 * 24);
+    hours = num_seconds / (60 * 60);
+    num_seconds -= hours * (60 * 60);
+    minutes = num_seconds / 60;
+    num_seconds -= minutes * 60;
+    "#{days}d:#{hours}h:#{minutes}m:#{num_seconds}s"
+  end
+
+  def self.num_cores
+    if RUBY_PLATFORM =~ /linux/
+      return `cat /proc/cpuinfo | grep processor | wc -l`.to_i
+    elsif RUBY_PLATFORM =~ /darwin/
+      `hwprefs cpu_count`.strip.to_i
+    elsif RUBY_PLATFORM =~ /freebsd|netbsd/
+      `sysctl hw.ncpu`.strip.to_i
+    else
+      return 1 # unknown..
+    end
+  rescue
+    # hwprefs doesn't always exist, and so the block above can fail.
+    # In any case, let's always assume that there is 1 core
+    1
+  end
+
+  def self.defer(*args, &blk)
+    if args[0].kind_of?(Hash)
+      op = blk
+      opts = args[0]
+    else
+      op = args[0] || blk
+      opts = args[1] || {}
+    end
+
+    callback = opts[:callback]
+    logger = opts[:logger]
+    nobacktrace = opts[:nobacktrace]
+
+    wrapped_operation = exception_wrap_block(op, logger, nobacktrace)
+    wrapped_callback = callback ? exception_wrap_block(callback, logger, nobacktrace) : nil
+    EM.defer(wrapped_operation, wrapped_callback)
+  end
+
+  def self.exception_wrap_block(op, logger, nobacktrace=false)
+    Proc.new do |*args|
+      begin
+        op.call(*args)
+      rescue => e
+        err_str = "#{e} - #{e.backtrace.join("\n")}" unless nobacktrace
+        err_str = "#{e}" if nobacktrace
+        if logger
+          logger.fatal(err_str)
+        else
+          $stderr.puts(err_str)
+        end
+      end
+    end
+  end
+
+  def self.process_running?(pid)
+    return false unless pid && (pid > 0)
+    output = %x[ps -o rss= -p #{pid}]
+    return true if ($? == 0 && !output.empty?)
+    # fail otherwise..
+    return false
+  end
+
+  def self.pp_bytesize(bsize)
+    units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
+    base  = 1000
+    bsize = bsize.to_f()
+    quotient = unit = nil
+    units.each_with_index do |u, i|
+      unit = u
+      quotient = bsize / (base ** i)
+      break if quotient < base
+    end
+    "%0.2f%s" % [quotient, unit]
+  end
+
+  def self.symbolize_keys(hash)
+    if hash.is_a? Hash
+      new_hash = {}
+      hash.each {|k, v| new_hash[k.to_sym] = symbolize_keys(v) }
+      new_hash
+    else
+      hash
+    end
+  end
+
+  # Helper class to atomically create/update pidfiles and ensure that only one instance of a given
+  # process is running at all times.
+  #
+  # NB: Ruby doesn't have real destructors so if you want to be polite and clean up after yourself
+  # be sure to call unlink() before your process exits.
+  #
+  # usage:
+  #
+  # begin
+  #   pidfile = VCAP::PidFile.new('/tmp/foo')
+  # rescue => e
+  #   puts "Error creating pidfile: %s" % (e)
+  #   exit(1)
+  # end
+  class PidFile
+    class ProcessRunningError < StandardError
+    end
+
+    def initialize(pid_file, create_parents=true)
+      @pid_file = pid_file
+      @dirty = true
+      write(create_parents)
+    end
+
+    # Removes the created pidfile
+    def unlink()
+      return unless @dirty
+
+      # Swallowing exception here is fine. Removing the pid files is a courtesy.
+      begin
+        File.unlink(@pid_file)
+        @dirty = false
+      rescue
+      end
+      self
+    end
+
+    # Removes the created pidfile upon receipt of the supplied signals
+    def unlink_on_signals(*sigs)
+      return unless @dirty
+
+      sigs.each do |s|
+        Signal.trap(s) { unlink() }
+      end
+      self
+    end
+
+    def unlink_at_exit()
+      at_exit { unlink() }
+      self
+    end
+
+    def to_s()
+      @pid_file
+    end
+
+    protected
+
+    # Atomically writes the pidfile.
+    # NB: This throws exceptions if the pidfile contains the pid of another running process.
+    #
+    # +create_parents+  If true, all parts of the path up to the file's dirname will be created.
+    #
+    def write(create_parents=true)
+      FileUtils.mkdir_p(File.dirname(@pid_file)) if create_parents
+
+      # Protip from Wilson: binary mode keeps things sane under Windows
+      # Closing the fd releases our lock
+      File.open(@pid_file, 'a+b', 0644) do |f|
+        f.flock(File::LOCK_EX)
+
+        # Check if process is already running
+        pid = f.read().strip().to_i()
+        if pid == Process.pid()
+          return
+        elsif VCAP.process_running?(pid)
+          raise ProcessRunningError.new("Process already running (pid=%d)." % (pid))
+        end
+
+        # We're good to go, write our pid
+        f.truncate(0)
+        f.rewind()
+        f.write("%d\n" % (Process.pid()))
+        f.flush()
+      end
+    end
+  end # class PidFile
+
+end # module VCAP
+
+# Make the patch here for proper bytesize
+if RUBY_VERSION <= "1.8.6"
+  class String #:nodoc:
+    def bytesize; self.size; end
+  end
+end
+
+# FIXME, we should ditch ruby logger.
+# Monkey Patch to get rid of some deadlocks under load in CC, make it available for all.
+
+require 'logger'
+
+STDOUT.sync = true
+
+class Logger::LogDevice
+  def write(message)
+    @dev.syswrite(message)
+  end
+
+  def close
+    @dev.close
+  end
+end

data/lib/vcap/component.rb

@@ -0,0 +1,172 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require "eventmachine"
+require 'thin'
+require "yajl"
+require "nats/client"
+require "base64"
+require 'set'
+
+module VCAP
+
+  RACK_JSON_HDR = { 'Content-Type' => 'application/json' }
+  RACK_TEXT_HDR = { 'Content-Type' => 'text/plaintext' }
+
+  class Varz
+    def initialize(logger)
+      @logger = logger
+    end
+
+    def call(env)
+      @logger.debug "varz access"
+      varz = Yajl::Encoder.encode(Component.updated_varz, :pretty => true, :terminator => "\n")
+      [200, { 'Content-Type' => 'application/json', 'Content-Length' => varz.length.to_s }, varz]
+    rescue => e
+      @logger.error "varz error #{e.inspect} #{e.backtrace.join("\n")}"
+      raise e
+    end
+  end
+
+  class Healthz
+    def initialize(logger)
+      @logger = logger
+    end
+
+    def call(env)
+      @logger.debug "healthz access"
+      healthz = Component.updated_healthz
+      [200, { 'Content-Type' => 'application/json', 'Content-Length' => healthz.length.to_s }, healthz]
+    rescue => e
+      @logger.error "healthz error #{e.inspect} #{e.backtrace.join("\n")}"
+      raise e
+    end
+  end
+
+  # Common component setup for discovery and monitoring
+  class Component
+
+    # We will suppress these from normal varz reporting by default.
+    CONFIG_SUPPRESS = Set.new([:mbus, :service_mbus, :keys, :database_environment, :mysql, :password])
+
+    class << self
+
+      attr_reader   :varz
+      attr_accessor :healthz
+
+      def updated_varz
+        @last_varz_update ||= 0
+        if Time.now.to_f - @last_varz_update >= 1
+          # Snapshot uptime
+          @varz[:uptime] = VCAP.uptime_string(Time.now - @varz[:start])
+
+          # Grab current cpu and memory usage.
+          rss, pcpu = `ps -o rss=,pcpu= -p #{Process.pid}`.split
+          @varz[:mem] = rss.to_i
+          @varz[:cpu] = pcpu.to_f
+
+          @last_varz_update = Time.now.to_f
+        end
+        varz
+      end
+
+      def updated_healthz
+        @last_healthz_update ||= 0
+        if Time.now.to_f - @last_healthz_update >= 1
+          # ...
+          @last_healthz_update = Time.now.to_f
+        end
+
+        healthz
+      end
+
+      def start_http_server(host, port, auth, logger)
+        http_server = Thin::Server.new(host, port, :signals => false) do
+          Thin::Logging.silent = true
+          use Rack::Auth::Basic do |username, password|
+            [username, password] == auth
+          end
+          map '/healthz' do
+            run Healthz.new(logger)
+          end
+          map '/varz' do
+            run Varz.new(logger)
+          end
+        end
+        http_server.start!
+      end
+
+      def uuid
+        @discover[:uuid]
+      end
+
+      def register(opts)
+        uuid = VCAP.secure_uuid
+        type = opts[:type]
+        index = opts[:index]
+        uuid = "#{index}-#{uuid}" if index
+        host = opts[:host] || VCAP.local_ip
+        port = opts[:port] || VCAP.grab_ephemeral_port
+        nats = opts[:nats] || NATS
+        auth = [opts[:user] || VCAP.secure_uuid, opts[:password] || VCAP.secure_uuid]
+        logger = opts[:logger] || Logger.new(nil)
+
+        # Discover message limited
+        @discover = {
+          :type => type,
+          :index => index,
+          :uuid => uuid,
+          :host => "#{host}:#{port}",
+          :credentials => auth,
+          :start => Time.now
+        }
+
+        # Varz is customizable
+        @varz = @discover.dup
+        @varz[:num_cores] = VCAP.num_cores
+        @varz[:config] = sanitize_config(opts[:config]) if opts[:config]
+
+        @healthz = "ok\n".freeze
+
+        # Next steps require EM
+        raise "EventMachine reactor needs to be running" if !EventMachine.reactor_running?
+
+        # Startup the http endpoint for /varz and /healthz
+        start_http_server(host, port, auth, logger)
+
+        # Listen for discovery requests
+        nats.subscribe('vcap.component.discover') do |msg, reply|
+          update_discover_uptime
+          nats.publish(reply, @discover.to_json)
+        end
+
+        # Also announce ourselves on startup..
+        nats.publish('vcap.component.announce', @discover.to_json)
+      end
+
+      def update_discover_uptime
+        @discover[:uptime] = VCAP.uptime_string(Time.now - @discover[:start])
+      end
+
+      def clear_level(h)
+        h.each do |k, v|
+          if CONFIG_SUPPRESS.include?(k.to_sym)
+            h.delete(k)
+          else
+            clear_level(h[k]) if v.instance_of? Hash
+          end
+        end
+      end
+
+      def sanitize_config(config)
+        # Can't Marshal/Deep Copy logger instances that services use
+        if config[:logger]
+          config = config.dup
+          config.delete(:logger)
+        end
+        # Deep copy
+        config = Marshal.load(Marshal.dump(config))
+        clear_level(config)
+        config
+      end
+    end
+  end
+end

data/lib/vcap/config.rb

@@ -0,0 +1,32 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require 'yaml'
+
+require 'vcap/common'
+require 'vcap/json_schema'
+
+module VCAP
+  class Config
+    class << self
+      attr_reader :schema
+
+      def define_schema(&blk)
+        @schema = VCAP::JsonSchema.build(&blk)
+      end
+
+      def from_file(filename, symbolize_keys=true)
+        config = YAML.load_file(filename)
+        @schema.validate(config)
+        config = VCAP.symbolize_keys(config) if symbolize_keys
+        config
+      end
+
+      def to_file(config, out_filename)
+        @schema.validate(config)
+        File.open(out_filename, 'w+') do |f|
+          YAML.dump(config, f)
+        end
+      end
+    end
+
+  end
+end

data/lib/vcap/fiber_tracing.rb

@@ -0,0 +1,45 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+class Fiber
+  attr_accessor :trace_id
+  alias_method :orig_resume, :resume
+
+  class << self
+    @io=nil
+
+    alias_method :orig_yield, :yield
+
+    def enable_tracing(io)
+      raise ArgumentError, "You must pass in IO object, #{io.class} given" unless io.is_a? IO
+      @io = io
+    end
+
+    def yield(*args)
+      log_action('yield')
+      begin
+        orig_yield(*args)
+      rescue FiberError => fe
+        Fiber.log_action('yield_error', self)
+        raise fe
+      end
+    end
+
+    def log_action(action, f=nil)
+      return unless @io
+      f ||= Fiber.current
+      trace_id = f.trace_id || '-'
+      cname = Kernel.caller[1]
+      @io.puts("FT %-14s %-20s %-30s %s" % [action, trace_id, f.object_id, cname])
+      @io.flush
+    end
+  end
+
+  def resume(*args)
+    Fiber.log_action('resume', self)
+    begin
+      orig_resume(*args)
+    rescue FiberError => fe
+      Fiber.log_action('resume_error', self)
+      raise fe
+    end
+  end
+end

data/lib/vcap/json_schema.rb

@@ -0,0 +1,202 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+
+# This provides very basic support for creating declarative validators
+# for decoded json. Useful for validating things like configs or
+# messages that are published via NATS.
+#
+# Basic usage:
+#
+# schema = VCAP::JsonSchema.build do
+#   { :foo => [String],
+#     :bar => {:baz => Integer},
+#     optional(:jaz) => Float,
+#   }
+# end
+#
+# Fails:
+#
+# schema.validate({
+#   'foo' => ['bar', 5],
+#   'bar' => {'baz' => 7},
+# })
+#
+# Succeeds:
+#
+# schema.validate({
+#   'foo' => ['bar', 'baz'],
+#   'bar' => {'baz' => 7},
+# })
+#
+#
+module VCAP
+  module JsonSchema
+
+    class SyntaxError < StandardError; end
+    class ValidationError < StandardError
+      attr_accessor :message
+      def initialize(msg)
+        @message = msg
+      end
+
+      def to_s
+        @message
+      end
+    end
+
+    class ValueError      < ValidationError; end
+    class TypeError       < ValidationError; end
+    class MissingKeyError < ValidationError; end
+
+    # Defines an interface that all schemas must implement
+    class BaseSchema
+      # Verfies that val conforms to the schema being validated
+      # Throws exceptions derived from ValidationError upon schema violations
+      #
+      # @param  Object val  An object decoded from json
+      #
+      # @return nil
+      def validate(dec_json)
+        raise NotImplementedError.new("You must implement validate")
+      end
+   end
+
+    class BoolSchema < BaseSchema
+      def validate(dec_json)
+        unless dec_json.kind_of?(TrueClass) || dec_json.kind_of?(FalseClass)
+          raise TypeError, "Expected instance of TrueClass or FalseClass, got #{dec_json.class}"
+        end
+      end
+    end
+
+    # Checks that supplied value is an instance of a given class
+    class TypeSchema < BaseSchema
+      def initialize(klass)
+        raise ArgumentError, "You must supply a class #{klass} given" unless klass.kind_of?(Class)
+        @klass = klass
+      end
+
+      def validate(dec_json)
+        raise TypeError, "Expected instance of #{@klass}, got #{dec_json.class}" unless dec_json.kind_of?(@klass)
+      end
+    end
+
+    # Checks that supplied value is an array, and each value in the array validates
+    class ArraySchema < BaseSchema
+      attr_accessor :item_schema
+
+      def initialize(schema)
+        raise ArgumentError, "You must supply a schema, #{schema.class} given" unless schema.kind_of?(BaseSchema)
+        @item_schema = schema
+      end
+
+      def validate(dec_json)
+        raise TypeError, "Expected instance of Array, #{dec_json.class} given" unless dec_json.kind_of?(Array)
+        for v in dec_json
+          @item_schema.validate(v)
+        end
+      end
+    end
+
+    # Check that required keys are present, and that they validate
+    class HashSchema < BaseSchema
+      class Field
+        attr_reader :name, :schema
+        attr_accessor :optional
+
+        def initialize(name, schema, optional=false)
+          raise ArgumentError, "Schema must be an instance of a schema, #{schema.class} given #{schema.inspect}" unless schema.kind_of?(BaseSchema)
+          @name = name
+          @schema = schema
+          @optional = optional
+        end
+      end
+
+      def initialize(kvs={})
+        raise ArgumentError, "Expected Hash, #{kvs.class} given" unless kvs.is_a?(Hash)
+        # Convert symbols to strings. Validation will be performed against decoded json, which will have
+        # string keys.
+        @fields = {}
+        for k, v in kvs
+          raise ArgumentError, "Expected schema for key #{k}, got #{v.class}" unless v.kind_of?(BaseSchema)
+          k_s = k.to_s
+          @fields[k_s] = Field.new(k_s, v, false)
+        end
+      end
+
+      def required(name, schema)
+        name_s = name.to_s
+        @fields[name_s] = Field.new(name_s, schema, false)
+      end
+
+      def optional(name, schema)
+        name_s = name.to_s
+        @fields[name_s] = Field.new(name_s, schema, true)
+      end
+
+      def validate(dec_json)
+        raise TypeError, "Expected instance of Hash, got instance of #{dec_json.class}" unless dec_json.kind_of?(Hash)
+
+        missing_keys = []
+        for k in @fields.keys
+          missing_keys << k unless dec_json.has_key?(k) || @fields[k].optional
+        end
+        raise MissingKeyError, "Missing keys: #{missing_keys.join(', ')}" unless missing_keys.empty?
+
+        for k, f in @fields
+          next if f.optional && !dec_json.has_key?(k)
+          begin
+            f.schema.validate(dec_json[k])
+          rescue ValidationError => ve
+            ve.message = "'#{k}' => " + ve.message
+            raise ve
+          end
+        end
+      end
+    end
+
+    class << self
+      class OptionalKeyMarker
+        attr_reader :name
+        def initialize(name)
+          @name = name
+        end
+
+        def to_s
+          @name.to_s
+        end
+      end
+
+      def optional(key)
+        OptionalKeyMarker.new(key)
+      end
+
+      def build(&blk)
+        schema_def = instance_eval(&blk)
+        parse(schema_def)
+      end
+
+      def parse(schema_def)
+        case schema_def
+        when VCAP::JsonSchema::BaseSchema
+          schema_def
+        when Hash
+          schema = VCAP::JsonSchema::HashSchema.new
+          for k, v in schema_def
+            sym = k.kind_of?(OptionalKeyMarker) ? :optional : :required
+            schema.send(sym, k, parse(v))
+          end
+          schema
+        when Array
+          raise SyntaxError, "Schema definition for an array must have exactly 1 element" unless schema_def.size == 1
+          item_schema = parse(schema_def[0])
+          VCAP::JsonSchema::ArraySchema.new(item_schema)
+        when Class
+          VCAP::JsonSchema::TypeSchema.new(schema_def)
+        else
+          raise SyntaxError, "Don't know what to do with class #{schema_def.class}"
+        end
+      end
+    end
+
+  end # VCAP::JsonSchema
+end   # VCAP