vcap_services_base 0.2.10

data/lib/base/snapshot_v2/snapshot_client.rb

@@ -0,0 +1,158 @@
+require "redis"
+require "time"
+require_relative "../service_error"
+
+module VCAP::Services::Base::SnapshotV2
+  class SnapshotClient
+    include VCAP::Services::Base::Error
+
+    SNAPSHOT_KEY_PREFIX = "vcap:snapshotv2".freeze
+    SNAPSHOT_ID = "maxid".freeze
+    FILTER_KEYS = %w(snapshot_id date size name).freeze
+    MAX_NAME_LENGTH = 512
+
+    def initialize(redis_config)
+      @redis = ::Redis.new(redis_config)
+      # FIXME: use UUID?
+      redis_init
+    end
+
+    def create_empty_snapshot(service_id, name)
+      snapshot = {
+        'state' => 'empty',
+        'size'  => 0,
+        'name'  => name,
+        'snapshot_id' => new_snapshot_id,
+      }
+      msg = Yajl::Encoder.encode(snapshot)
+      client.hset(redis_key(service_id), snapshot['snapshot_id'], msg)
+      snapshot
+    end
+
+    # Get all snapshots related to a service instance
+    #
+    def service_snapshots(service_id)
+      return unless service_id
+      res = client.hgetall(redis_key(service_id))
+      res.values.map{|v| Yajl::Parser.parse(v)}
+    end
+
+    # Return total snapshots count
+    #
+    def service_snapshots_count(service_id)
+      return unless service_id
+      client.hlen(redis_key(service_id))
+    end
+
+    # Get detail information for a single snapshot
+    #
+    def snapshot_details(service_id, snapshot_id)
+      return unless service_id && snapshot_id
+      res = client.hget(redis_key(service_id), snapshot_id)
+      raise ServiceError.new(ServiceError::NOT_FOUND, "snapshot #{snapshot_id}") unless res
+      Yajl::Parser.parse(res)
+    end
+
+    # filter internal keys of a given snapshot object, return a new snapshot object in canonical format
+    def self.filter_keys(snapshot)
+      return unless snapshot.is_a? Hash
+      snapshot.select {|k,v| FILTER_KEYS.include? k.to_s}
+    end
+
+    # Generate a new unique id for a snapshot
+    def new_snapshot_id
+      client.incr(redis_key(SNAPSHOT_ID)).to_s
+    end
+
+    # Get the snapshot file path that service should save the dump file to.
+    # the snapshot path structure looks like <base_dir>\snapshots\<service-name>\<aa>\<bb>\<cc>\<aabbcc-rest-of-instance-guid>\snapshot_id\<service specific data>
+    def self.snapshot_filepath(base_dir, service_name, service_id, snapshot_id)
+      File.join(base_dir, "snapshots", service_name, service_id[0,2], service_id[2,2], service_id[4,2], service_id, snapshot_id.to_s)
+    end
+
+    # Update the name of given snapshot.
+    # This function is not protected by redis lock so a optimistic lock
+    # is applied to prevent concurrent update.
+    #
+    def update_name(service_id, snapshot_id, name)
+      return unless service_id && snapshot_id && name
+      verify_input_name(name)
+
+      key = self.class.redis_key(service_id)
+      # NOTE: idealy should watch on combination of (service_id, snapshot_id)
+      # but current design doesn't support such fine-grained watching.
+      client.watch(key)
+
+      snapshot = client.hget(redis_key(service_id), snapshot_id)
+      return nil unless snapshot
+      snapshot = Yajl::Parser.parse(snapshot)
+      snapshot["name"] = name
+
+      res = client.multi do
+        save_snapshot(service_id, snapshot)
+      end
+
+      unless res
+        raise ServiceError.new(ServiceError::REDIS_CONCURRENT_UPDATE)
+      end
+      true
+    end
+
+    def save_snapshot(service_id , snapshot)
+      return unless service_id && snapshot
+      # FIXME: srsly? where are we using symbols?
+      sid = snapshot[:snapshot_id] || snapshot["snapshot_id"]
+      return unless sid
+      msg = Yajl::Encoder.encode(snapshot)
+      client.hset(redis_key(service_id), sid, msg)
+    end
+
+    def delete_snapshot(service_id , snapshot_id)
+      return unless service_id && snapshot_id
+      client.hdel(redis_key(service_id), snapshot_id)
+    end
+
+
+    def self.fmt_time()
+      # UTC time in ISO 8601 format.
+      Time.now.utc.strftime("%FT%TZ")
+    end
+
+    def self.redis_key(key)
+      "#{SNAPSHOT_KEY_PREFIX}:#{key}"
+    end
+
+    private
+    def filter_keys(snapshot)
+      self.class.filter_keys(snapshot)
+    end
+
+    def snapshot_filepath(base_dir, service_name, service_id, snapshot_id)
+      self.class.snapshot_filepath(base_dir, service_name, service_id, snapshot_id)
+    end
+
+    def redis_key(key)
+      self.class.redis_key(key)
+    end
+
+    attr_reader :redis
+
+    # initialize necessary keys
+    def redis_init
+      @redis.setnx("#{SNAPSHOT_KEY_PREFIX}:#{SNAPSHOT_ID}", 1)
+    end
+
+    def client
+      redis
+    end
+
+    def verify_input_name(name)
+      return unless name
+
+      raise ServiceError.new(ServiceError::INVALID_SNAPSHOT_NAME,
+                             "Input name exceed the max allowed #{MAX_NAME_LENGTH} characters.") if name.size > MAX_NAME_LENGTH
+
+      #TODO: shall we sanitize the input?
+    end
+  end
+end

data/lib/base/snapshot_v2/snapshot_job.rb

@@ -0,0 +1,95 @@
+require "resque-status"
+require "fileutils"
+
+require_relative "../service_error"
+require_relative "snapshot_client"
+
+
+module VCAP::Services::Base::SnapshotV2
+  class MySqlSnapshotJob
+    def self.queue_lookup_key
+      'mysql'
+    end
+  end
+
+  # common utils for snapshot job
+  class SnapshotJob
+    include Resque::Plugins::Status
+
+    class QueueResolver
+      extend Forwardable
+      def_delegator :@job_class, :queue_lookup_key
+
+      def initialize(job_class)
+        @job_class = job_class
+      end
+
+      def resolve(*args)
+        result = nil
+        args.each do |arg|
+          result = arg[queue_lookup_key] if (arg.is_a? Hash) && (arg.has_key?(queue_lookup_key))
+        end
+        raise "no queue matched for look up key #{queue_lookup_key} and args #{args}" unless result
+        result
+      end
+    end
+
+    class << self
+      def queue_lookup_key
+        :node_id
+      end
+
+      def select_queue(*args)
+        QueueResolver.new(self).resolve(*args)
+      end
+    end
+
+    def initialize(*args)
+      super(*args)
+      parse_config
+      #client = SnapshotClient.new(Config.redis_config)
+      # @logger = Config.logger
+      # Snapshot.redis_connect
+    end
+
+    # def fmt_error(e)
+      # "#{e}: [#{e.backtrace.join(" | ")}]"
+    # end
+
+    # def required_options(*args)
+      # missing_opts = args.select{|arg| !options.has_key? arg.to_s}
+      # raise ArgumentError, "Missing #{missing_opts.join(', ')} in options: #{options.inspect}" unless missing_opts.empty?
+    # end
+
+    # def create_lock(lock_name)
+      # # lock_name = "lock:lifecycle:#{name}"
+      # ttl = @config['job_ttl'] || 600
+      # lock = Lock.new(lock_name, :logger => @logger, :ttl => ttl)
+      # lock
+    # end
+
+    # def get_dump_path(name, snapshot_id)
+      # snapshot_filepath(@config["snapshots_base_dir"], @config["service_name"], name, snapshot_id)
+    # end
+
+    # def cleanup(name, snapshot_id)
+      # return unless name && snapshot_id
+      # @logger.info("Clean up snapshot and files for #{name}, snapshot id: #{snapshot_id}")
+      # client.delete_snapshot(name, snapshot_id)
+      # FileUtils.rm_rf(get_dump_path(name, snapshot_id))
+    # end
+
+    # def handle_error(e)
+      # @logger.error("Error in #{self.class} uuid:#{@uuid}: #{fmt_error(e)}")
+      # err = (e.instance_of?(ServiceError)? e : ServiceError.new(ServiceError::INTERNAL_ERROR)).to_hash
+      # err_msg = Yajl::Encoder.encode(err["msg"])
+      # failed(err_msg)
+    # end
+
+    def parse_config
+      @config = Yajl::Parser.parse(ENV['WORKER_CONFIG'])
+      raise "Need environment variable: WORKER_CONFIG" unless @config
+      @config
+    end
+  end
+end

data/lib/base/utils.rb

@@ -0,0 +1,63 @@
+require "open3"
+
+module VCAP::Services::Base::Utils
+
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  module ClassMethods
+    def sh(*args)
+      options = args[-1].respond_to?(:to_hash) ? args.pop.to_hash: {}
+      options = { :timeout => 5.0, :max => 1024 * 1024, :sudo => true, :block => true }.merge(options)
+      arg = options[:sudo] == false ? args[0] : "sudo " << args[0]
+
+      begin
+        stdin, stdout, stderr, status = Open3.popen3(arg)
+        pid = status[:pid]
+        out_buf = ""
+        err_buf = ""
+        if options[:block]
+          start = Time.now
+          # Manually ping the process per second to check whether the process is alive or not
+          while (Time.now - start) < options[:timeout] && status.alive?
+            begin
+              out_buf << stdout.read_nonblock(4096)
+              err_buf << stderr.read_nonblock(4096)
+            rescue IO::WaitReadable, EOFError
+            end
+            sleep 0.2
+          end
+
+          if status.alive?
+            Process.kill("TERM", pid)
+            Process.detach(pid)
+            raise RuntimeError, "sh #{args} executed with failure and process with pid #{pid} timed out:\nstdout:\n#{out_buf}\nstderr:\n#{err_buf}"
+          end
+          exit_status = status.value.exitstatus
+          raise RuntimeError, "sh #{args} executed with failure and process with pid #{pid} exited with #{status.value.exitstatus}:\nstdout:\n#{out_buf}\nstderr:\n#{err_buf}" unless exit_status == 0
+          exit_status
+        else
+          # If the work is still not done after timeout, then kill the process and record an erorr log
+          Thread.new do
+            sleep options[:timeout]
+            if status.alive?
+              Process.kill("TERM", pid)
+              Process.detach(pid)
+              logger.error "sh #{args} executed with pid #{pid} timed out" if logger
+            else
+              logger.error "sh #{args} executed with failure, the exit status is #{status.value.exitstatus}" if status.value.exitstatus != 0 && logger
+            end
+          end
+          return 0
+        end
+      rescue Errno::EPERM
+        raise RuntimeError, "sh #{args} executed with failure and process with pid #{pid} cannot be killed (privilege issue?):\nstdout:\n#{out_buf}\nstderr:\n#{err_buf}"
+      rescue Errno::ESRCH
+        raise RuntimeError, "sh #{args} executed with failure and process with pid #{pid} does not exist:\nstdout:\n#{out_buf}\nstderr:\n#{err_buf}"
+      rescue => e
+        raise e
+      end
+    end
+  end
+end

data/lib/base/version.rb

@@ -0,0 +1,7 @@
+module VCAP
+  module Services
+    module Base
+      VERSION = "0.2.10"
+    end
+  end
+end

data/lib/base/warden/instance_utils.rb

@@ -0,0 +1,161 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require "warden/client"
+require "warden/protocol"
+
+$LOAD_PATH.unshift File.join("..", File.dirname(__FILE__))
+require "base/utils"
+require "base/abstract"
+require "base/service_error"
+
+module VCAP
+  module Services
+    module Base
+      module Warden
+      end
+    end
+  end
+end
+
+module VCAP::Services::Base::Warden::InstanceUtils
+
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  module ClassMethods
+    def warden_connect
+      warden_client = Warden::Client.new(warden_socket_path)
+      warden_client.connect
+      warden_client
+    end
+
+    def warden_socket_path
+      "/tmp/warden.sock"
+    end
+  end
+
+  # warden container operation helper
+  def container_start(bind_mounts=[])
+    warden = self.class.warden_connect
+    req = Warden::Protocol::CreateRequest.new
+    unless bind_mounts.empty?
+      req.bind_mounts = bind_mounts
+    end
+    rsp = warden.call(req)
+    handle = rsp.handle
+    handle
+  ensure
+    warden.disconnect if warden
+  end
+
+  def container_stop(handle, force=true)
+    warden = self.class.warden_connect
+    req = Warden::Protocol::StopRequest.new
+    req.handle = handle
+    req.background = !force
+    warden.call(req)
+    true
+  ensure
+    warden.disconnect if warden
+  end
+
+  def container_destroy(handle)
+    warden = self.class.warden_connect
+    req = Warden::Protocol::DestroyRequest.new
+    req.handle = handle
+    warden.call(req)
+    true
+  ensure
+    warden.disconnect if warden
+  end
+
+  def container_running?(handle)
+    handle != "" && container_info(handle) != nil
+  end
+
+  def container_run_command(handle, cmd, is_privileged=false)
+    warden = self.class.warden_connect
+    req = Warden::Protocol::RunRequest.new
+    req.handle = handle
+    req.script = cmd
+    req.privileged = is_privileged
+    res = warden.call(req)
+    if res.exit_status == 0
+      res
+    else
+      raise VCAP::Services::Base::Error::ServiceError::new(VCAP::Services::Base::Error::ServiceError::WARDEN_RUN_COMMAND_FAILURE, cmd, handle, res.exit_status, res.stdout, res.stderr)
+    end
+  ensure
+    warden.disconnect if warden
+  end
+
+  def container_spawn_command(handle, cmd, is_privileged=false)
+    warden = self.class.warden_connect
+    req = Warden::Protocol::SpawnRequest.new
+    req.handle = handle
+    req.script = cmd
+    req.privileged = is_privileged
+    res = warden.call(req)
+    res
+  ensure
+    warden.disconnect if warden
+  end
+
+  def container_info(handle)
+    warden = self.class.warden_connect
+    req = Warden::Protocol::InfoRequest.new
+    req.handle = handle
+    warden.call(req)
+  rescue => e
+    nil
+  ensure
+    warden.disconnect if warden
+  end
+
+  def limit_memory(handle, limit)
+    warden = self.class.warden_connect
+    req = Warden::Protocol::LimitMemoryRequest.new
+    req.handle = handle
+    req.limit_in_bytes = limit * 1024 * 1024
+    warden.call(req)
+    true
+  ensure
+    warden.disconnect if warden
+  end
+
+  def limit_bandwidth(handle, rate)
+    warden = self.class.warden_connect
+    req = Warden::Protocol::LimitBandwidthRequest.new
+    req.handle = handle
+    req.rate = (rate * 1024 * 1024).to_i
+    req.burst = (rate * 1 * 1024 * 1024).to_i # Set burst the same size as rate
+    warden.call(req)
+    true
+  ensure
+    warden.disconnect if warden
+  end
+
+  def map_port(handle, src_port, dest_port)
+    warden = self.class.warden_connect
+    req = Warden::Protocol::NetInRequest.new
+    req.handle = handle
+    req.host_port = src_port
+    req.container_port = dest_port
+    res = warden.call(req)
+    res
+  ensure
+    warden.disconnect if warden
+  end
+
+  def bind_mount_request(bind_dir)
+    bind = Warden::Protocol::CreateRequest::BindMount.new
+    bind.src_path = bind_dir[:src]
+    bind.dst_path = bind_dir[:dst] || bind_dir[:src]
+    if bind_dir[:read_only]
+      bind.mode = Warden::Protocol::CreateRequest::BindMount::Mode::RO
+    else
+      bind.mode = Warden::Protocol::CreateRequest::BindMount::Mode::RW
+    end
+    bind
+  end
+end