vcap_services_base 0.2.10

data/lib/base/backup.rb

@@ -0,0 +1,206 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'optparse'
+require 'timeout'
+require 'fileutils'
+require 'yaml'
+require 'pathname'
+require 'steno'
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', '..', '..')
+require 'vcap/common'
+
+$:.unshift File.dirname(__FILE__)
+require 'abstract'
+
+module VCAP
+  module Services
+    module Base
+    end
+  end
+end
+
+#@config_file Full path to config file
+#@config Config hash for config file
+#@logger
+#@nfs_base NFS base path
+class VCAP::Services::Base::Backup
+  abstract :default_config_file
+  abstract :backup_db
+
+  def initialize
+    @run_lock = Mutex.new
+    @shutdown = false
+    trap("TERM") { exit_fun }
+    trap("INT") { exit_fun }
+  end
+
+  def script_file
+    $0
+  end
+
+  def exit_fun
+    @shutdown = true
+    Thread.new do
+      @run_lock.synchronize { exit }
+    end
+  end
+
+  def single_app(&blk)
+    if File.open(script_file).flock(File::LOCK_EX|File::LOCK_NB)
+      blk.call
+    else
+      echo "Script #{ script_file } is already running",true
+    end
+  end
+
+  def start
+    single_app do
+      echo "#{File.basename(script_file)} starts"
+      @config_file = default_config_file
+      parse_options
+
+      echo "Load config file"
+      # load conf file
+      begin
+        @config = YAML.load(File.open(@config_file))
+      rescue => e
+        echo "Could not read configuration file: #{e}",true
+        exit
+      end
+
+      # Setup logger
+      echo @config["logging"]
+      logging_config = Steno::Config.from_hash(@config["logging"])
+      Steno.init(logging_config)
+      # Use running binary name for logger identity name.
+      @logger = Steno.logger(File.basename(script_file))
+
+      # Make pidfile
+      if @config["pid"]
+        pf = VCAP::PidFile.new(@config["pid"])
+        pf.unlink_at_exit
+      end
+
+      echo "Check mount points"
+      check_mount_points
+
+      # make sure backup dir on nfs storage exists
+      @nfs_base = @config["backup_base_dir"] + "/backups/" + @config["service_name"]
+      echo "Check NFS base"
+      if File.directory? @nfs_base
+        echo @nfs_base + " exists"
+      else
+        echo @nfs_base + " does not exist, create it"
+        begin
+          FileUtils.mkdir_p @nfs_base
+        rescue => e
+          echo "Could not create dir on nfs!",true
+          exit
+        end
+      end
+      echo "Run backup task"
+      @run_lock.synchronize { backup_db }
+      echo "#{File.basename(script_file)} task is completed"
+    end
+  rescue => e
+    echo "Error: #{e.message}\n #{e.backtrace}",true
+  rescue Interrupt => it
+    echo "Backup is interrupted!"
+  end
+
+  def get_dump_path(name, options={})
+    name = name.sub(/^(mongodb|redis)-/,'')
+    mode = options[:mode] || 0
+    time = options[:time] || Time.now
+    case mode
+    when 1
+      File.join(@config['backup_base_dir'], 'backups', @config['service_name'],name, time.to_i.to_s,@config['node_id'])
+    else
+      File.join(@config['backup_base_dir'], 'backups', @config['service_name'], name[0,2], name[2,2], name[4,2], name, time.to_i.to_s)
+    end
+  end
+
+  def check_mount_points
+    # make sure the backup base dir is mounted
+    pn = Pathname.new(@config["backup_base_dir"])
+    if !@tolerant && !pn.mountpoint?
+      echo @config["backup_base_dir"] + " is not mounted, exit",true
+      exit
+    end
+  end
+
+  def echo(output, err=false)
+    if err
+      $stderr.puts(output) unless @logger
+      @logger.error(output) if @logger
+    else
+      $stdout.puts(output) unless @logger
+      @logger.info(output) if @logger
+    end
+  end
+
+  def parse_options
+    OptionParser.new do |opts|
+      opts.banner = "Usage: #{File.basename(script_file)} [options]"
+      opts.on("-c", "--config [ARG]", "Node configuration File") do |opt|
+        @config_file = opt
+      end
+      opts.on("-h", "--help", "Help") do
+        puts opts
+        exit
+      end
+      opts.on("-t", "--tolerant",    "Tolerant mode") do
+        @tolerant = true
+      end
+      more_options(opts)
+    end.parse!
+  end
+
+  def more_options(opts)
+  end
+end
+
+class CMDHandle
+
+  def initialize(cmd, timeout=nil, &blk)
+    @cmd  = cmd
+    @timeout = timeout
+    @errback = blk
+  end
+
+  def run
+    pid = fork
+    if pid
+      # parent process
+      success = false
+      begin
+        success = Timeout::timeout(@timeout) do
+          Process.waitpid(pid)
+          value = $?.exitstatus
+          @errback.call(@cmd, value, "No message.") if value != 0 && @errback
+          return value == 0
+        end
+      rescue Timeout::Error
+        Process.detach(pid)
+        Process.kill("KILL", pid)
+        @errback.call(@cmd, -1, "Killed due to timeout.") if @errback
+        return false
+      end
+    else
+      begin
+        # child process
+        exec(@cmd)
+      rescue => e
+        exit!
+      end
+    end
+  end
+
+  def self.execute(cmd, timeout = nil, *args)
+    errb = args.pop if args.last.is_a? Proc
+    instance = self.new(cmd, timeout, &errb)
+    instance.run
+  end
+end
+

data/lib/base/barrier.rb

@@ -0,0 +1,54 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require "eventmachine"
+require "thread"
+
+module VCAP
+  module Services
+    module Base
+    end
+  end
+end
+
+class VCAP::Services::Base::Barrier
+
+  def initialize(options = {}, &callback)
+    raise ArgumentError unless options[:timeout] || options[:callbacks]
+    @lock = Mutex.new
+    @callback = callback
+    @expected_callbacks = options[:callbacks]
+    @timer = EM.add_timer(options[:timeout]) {on_timeout} if options[:timeout]
+    @callback_fired = false
+    @responses = []
+    @barrier_callback = Proc.new {|*args| call(*args)}
+  end
+
+  def on_timeout
+    @lock.synchronize do
+      unless @callback_fired
+        @callback_fired = true
+        @callback.call(@responses)
+      end
+    end
+  end
+
+  def call(*args)
+    @lock.synchronize do
+      unless @callback_fired
+        @responses << args
+        if @expected_callbacks
+          @expected_callbacks -= 1
+          if @expected_callbacks <= 0
+            EM.cancel_timer(@timer) if @timer
+            @callback_fired = true
+            @callback.call(@responses)
+          end
+        end
+      end
+    end
+  end
+
+  def callback
+    @barrier_callback
+  end
+
+end

data/lib/base/base.rb

@@ -0,0 +1,159 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require 'eventmachine'
+require 'vcap/common'
+require 'vcap/component'
+require 'nats/client'
+
+$LOAD_PATH.unshift File.dirname(__FILE__)
+require 'abstract'
+require 'service_error'
+
+module VCAP
+  module Services
+    module Base
+      class Base
+      end
+    end
+  end
+end
+
+class Object
+  def deep_dup
+    Marshal::load(Marshal.dump(self))
+  end
+end
+
+class VCAP::Services::Base::Base
+
+  include VCAP::Services::Base::Error
+
+  attr_reader :closing
+
+  def initialize(options)
+    @logger = options[:logger]
+    @options = options
+    @local_ip = VCAP.local_ip(options[:ip_route])
+    @logger.info("#{service_description}: Initializing")
+    @closing = false
+
+    @node_nats = nil
+    if options[:mbus]
+      NATS.on_error do |e|
+        # p $!
+        # puts $!.backtrace.join("\n")
+        @logger.error("Exiting due to NATS error: #{e}")
+        shutdown
+        exit
+      end
+      @logger.debug("Connecting with NATS")
+      @node_nats = NATS.connect(:uri => options[:mbus]) do
+        status_port = status_user = status_password = nil
+        if not options[:status].nil?
+          status_port = options[:status][:port]
+          status_user = options[:status][:user]
+          status_password = options[:status][:password]
+        end
+
+        @logger.debug("Registering with NATS")
+        VCAP::Component.register(
+          :nats => @node_nats,
+          :type => service_description,
+          :host => @local_ip,
+          :index => options[:index] || 0,
+          :config => options,
+          :port => status_port,
+          :user => status_user,
+          :password => status_password
+        )
+        on_connect_node
+      end
+    else
+      @logger.info("NATS is disabled")
+    end
+
+    @max_nats_payload = options[:max_nats_payload] || 1024 * 1024
+  end
+
+  def service_description()
+    return "#{service_name}-#{flavor}"
+  end
+
+  def publish(reply, msg)
+    # nats publish are only allowed to be called in reactor thread.
+    EM.schedule do
+      @node_nats.publish(reply, msg) if @node_nats
+    end
+  end
+
+  def update_varz()
+    vz = varz_details
+    vz.each { |k,v|
+      VCAP::Component.varz[k] = v
+    } if vz
+  end
+
+  def shutdown()
+    @closing = true
+    @logger.info("#{service_description}: Shutting down")
+    @node_nats.close if @node_nats
+  end
+
+  def group_handles_in_json(instances_list, bindings_list, size_limit)
+    while instances_list.count > 0 or bindings_list.count > 0
+      ins_list = []
+      bind_list = []
+      send_len = 0
+      idx_ins = 0
+      idx_bind = 0
+
+      instances_list.each do |ins|
+        len = ins.to_json.size + 1
+        if send_len + len < size_limit
+          send_len += len
+          idx_ins += 1
+          ins_list << ins
+        else
+          break
+        end
+      end
+      instances_list.slice!(0, idx_ins) if idx_ins > 0
+
+      bindings_list.each do |bind|
+        len = bind.to_json.size + 1
+        if send_len + len < size_limit
+          send_len += len
+          idx_bind += 1
+          bind_list << bind
+        else
+          break
+        end
+      end
+      bindings_list.slice!(0, idx_bind) if idx_bind > 0
+
+      # Generally, the size_limit is far more bigger than the length
+      # of any handles. If there's a huge handle or the size_limit is too
+      # small that the size_limit can't contain one handle the in one batch,
+      # we have to break the loop if no handle can be stuffed into batch.
+      if ins_list.count == 0 and bind_list.count == 0
+        @logger.warn("NATS message limit #{size_limit} is too small.")
+        break
+      else
+        yield ins_list, bind_list
+      end
+    end
+  end
+
+  # Subclasses VCAP::Services::Base::{Node,Provisioner} implement the
+  # following methods. (Note that actual service Provisioner or Node
+  # implementations should NOT need to touch these!)
+
+  # TODO on_connect_node should be on_connect_nats
+  abstract :on_connect_node
+  abstract :flavor # "Provisioner" or "Node"
+  abstract :varz_details
+
+  # Service Provisioner and Node classes must implement the following
+  # method
+  abstract :service_name
+
+end

data/lib/base/base_async_gateway.rb

@@ -0,0 +1,164 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+# XXX(mjp)
+require 'rubygems'
+
+require 'eventmachine'
+require 'em-http-request'
+require 'json'
+require 'sinatra/base'
+require 'uri'
+require 'thread'
+require 'json_message'
+require 'services/api'
+require 'services/api/const'
+
+$:.unshift(File.dirname(__FILE__))
+require 'service_error'
+
+module VCAP
+  module Services
+  end
+end
+
+# A simple service gateway that proxies requests onto an asynchronous service provisioners.
+# NB: Do not use this with synchronous provisioners, it will produce unexpected results.
+#
+# TODO(mjp): This needs to handle unknown routes
+class VCAP::Services::BaseAsynchronousServiceGateway < Sinatra::Base
+
+  include VCAP::Services::Base::Error
+
+  # Allow our exception handlers to take over
+  set :raise_errors, Proc.new {false}
+  set :show_exceptions, false
+
+  def initialize(opts)
+    super
+
+    @logger = opts[:logger]
+
+    setup(opts)
+  end
+
+  # Validate the incoming request
+  before do
+    Steno.config.context.data["request_guid"] = env['HTTP_X_VCAP_REQUEST_ID']
+
+    validate_incoming_request
+
+    content_type :json
+  end
+
+  after do
+    Steno.config.context.data.delete("request_guid")
+  end
+
+  # Handle errors that result from malformed requests
+  error [JsonMessage::ValidationError, JsonMessage::ParseError] do
+    error_msg = ServiceError.new(ServiceError::MALFORMATTED_REQ).to_hash
+    abort_request(error_msg)
+  end
+
+  # setup the environment
+  def setup(opts)
+  end
+
+  # Custom request validation
+  def validate_incoming_request
+  end
+
+  #################### Helpers ####################
+
+  helpers do
+
+    # Aborts the request with the supplied errs
+    #
+    # +errs+  Hash of section => err
+    def abort_request(error_msg)
+      err_body = error_msg['msg'].to_json()
+      halt(error_msg['status'], {'Content-Type' => Rack::Mime.mime_type('.json')}, err_body)
+    end
+
+    def auth_token
+      @auth_token ||= request_header(VCAP::Services::Api::GATEWAY_TOKEN_HEADER)
+      @auth_token
+    end
+
+    def request_body
+      request.body.rewind
+      request.body.read
+    end
+
+    def request_header(header)
+      # This is pretty ghetto but Rack munges headers, so we need to munge them as well
+      rack_hdr = "HTTP_" + header.upcase.gsub(/-/, '_')
+      env[rack_hdr]
+    end
+
+    def async_mode(timeout=@node_timeout)
+      request.env['__async_timer'] = EM.add_timer(timeout) do
+        @logger.warn("Request timeout in #{timeout} seconds.")
+        error_msg = ServiceError.new(ServiceError::SERVICE_UNAVAILABLE).to_hash
+        err_body = error_msg['msg'].to_json()
+        request.env['async.callback'].call(
+          [
+            error_msg['status'],
+            {'Content-Type' => Rack::Mime.mime_type('.json')},
+            err_body
+          ]
+        )
+      end unless request.env['done'] ||= false
+      throw :async
+    end
+
+    def async_reply(resp='{}')
+      async_reply_raw(200, {'Content-Type' => Rack::Mime.mime_type('.json')}, resp)
+    end
+
+    def async_reply_raw(status, headers, body)
+      @logger.debug("Reply status:#{status}, headers:#{headers}, body:#{body}")
+      @provisioner.update_responses_metrics(status) if @provisioner
+      request.env['done'] = true
+      EM.cancel_timer(request.env['__async_timer']) if request.env['__async_timer']
+      request.env['async.callback'].call([status, headers, body])
+    end
+
+    def async_reply_error(error_msg)
+      err_body = error_msg['msg'].to_json()
+      async_reply_raw(error_msg['status'], {'Content-Type' => Rack::Mime.mime_type('.json')}, err_body)
+    end
+  end
+
+  private
+
+  def add_proxy_opts(req)
+    req[:proxy] = @proxy_opts
+    # this is a workaround for em-http-requesr 0.3.0 so that headers are not lost
+    # more info: https://github.com/igrigorik/em-http-request/issues/130
+    req[:proxy][:head] = req[:head]
+  end
+
+  def create_http_request(args)
+    req = {
+      :head => args[:head],
+      :body => args[:body],
+    }
+
+    if (@proxy_opts)
+      add_proxy_opts(req)
+    end
+
+    req
+  end
+
+  def make_logger(level=Logger::INFO)
+    logger = Logger.new(STDOUT)
+    logger.level = level
+    logger
+  end
+
+  def http_uri(uri)
+    uri = "http://#{uri}" unless (uri.index('http://') == 0 || uri.index('https://') == 0)
+    uri
+  end
+end