vcap_services_base 0.2.10

data/lib/base/http_handler.rb

@@ -0,0 +1,101 @@
+require 'uaa'
+
+class HTTPHandler
+  HTTP_UNAUTHENTICATED_CODE = 401
+  attr_reader :logger
+
+  def initialize(options)
+    @logger = options[:logger]
+    @options = options
+    @cld_ctrl_uri = options[:cloud_controller_uri]
+  end
+
+  def cc_http_request(args, &block)
+    args[:uri] = "#{@cld_ctrl_uri}#{args[:uri]}"
+    args[:head] = cc_req_hdrs
+
+    max_attempts = args[:max_attempts] || 2
+    attempts = 0
+    while true
+      attempts += 1
+      logger.debug("#{args[:method].upcase} - #{args[:uri]}")
+      http = make_http_request(args)
+      if attempts < max_attempts && http.response_header.status == HTTP_UNAUTHENTICATED_CODE
+        args[:head] = refresh_client_auth_token
+      else
+        block.call(http)
+        return http
+      end
+    end
+  end
+
+  def refresh_client_auth_token
+    # Load the auth token to be sent out in Authorization header when making CCNG-v2 requests
+    credentials = @options.fetch(:uaa_client_auth_credentials)
+    client_id = @options.fetch(:uaa_client_id)
+
+    if ENV["AUTHORIZATION_TOKEN"]
+      uaa_client_auth_token = ENV["AUTHORIZATION_TOKEN"]
+    else
+      ti = CF::UAA::TokenIssuer.new(@options.fetch(:uaa_endpoint), client_id)
+      token = ti.implicit_grant_with_creds(credentials).info
+      uaa_client_auth_token = "#{token["token_type"]} #{token["access_token"]}"
+      expire_time = token["expires_in"].to_i
+      logger.info("Successfully refresh auth token for:\
+                  #{credentials[:username]}, token expires in \
+                  #{expire_time} seconds.")
+    end
+
+    @cc_req_hdrs = {
+      'Content-Type' => 'application/json',
+      'Authorization' => uaa_client_auth_token
+    }
+  end
+
+  def cc_req_hdrs
+    @cc_req_hdrs || refresh_client_auth_token
+  end
+
+  def generate_cc_advertise_offering_request(service, active = true)
+    svc = service.to_hash
+
+    # NOTE: In CCNG, multiple versions is expected to be supported via multiple plans
+    # The gateway will have to maintain a mapping of plan-name to version so that
+    # the correct version will be provisioned
+    plans = {}
+    svc.delete('plans').each do |p|
+      plans[p[:name]] = {
+        "unique_id" => p[:unique_id],
+        "name" => p[:name],
+        "description" => p[:description],
+        "free" => p[:free]
+      }
+    end
+
+    [svc, plans]
+  end
+
+  private
+  def make_http_request(args)
+    req = {
+      :head => args[:head],
+      :body => args[:body],
+    }
+
+    f = Fiber.current
+    http = EM::HttpRequest.new(args[:uri]).send(args[:method], req)
+    if http.error && http.error != ""
+      unless args[:need_raise]
+        @logger.error("CC Catalog Manager: Failed to connect to CC, the error is #{http.error}")
+        return
+      else
+        raise("CC Catalog Manager: Failed to connect to CC, the error is #{http.error}")
+      end
+    end
+    http.callback { f.resume(http, nil) }
+    http.errback { |e| f.resume(http, e) }
+    _, error = Fiber.yield
+    yield http, error if block_given?
+    http
+  end
+end

data/lib/base/job/async_job.rb

@@ -0,0 +1,71 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require "resque-status"
+
+$LOAD_PATH.unshift File.dirname(__FILE__)
+require "config"
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..')
+require "service_error"
+
+module Resque
+  extend self
+  # Patch Resque so we can determine queue by input args.
+  # Job class can define select_queue method and the result will be the queue name.
+  def enqueue(klass, *args)
+    queue = (klass.respond_to?(:select_queue) && klass.select_queue(*args)) || queue_from_class(klass)
+    enqueue_to(queue, klass, *args)
+  end
+
+end
+
+module Resque::Plugins::Status
+    class Hash
+      # new attributes
+      hash_accessor :complete_time
+    end
+end
+
+# A thin layer wraps resque-status
+module VCAP::Services::Base::AsyncJob
+  include VCAP::Services::Base::Error
+
+  def job_repo_setup
+    redis = Config.redis
+    @logger = Config.logger
+    raise "AsyncJob requires redis configuration." unless redis
+    @logger.debug("Initialize Resque using #{redis}") if @logger
+    ::Resque.redis = redis
+  end
+
+  def get_job(jobid)
+    res = Resque::Plugins::Status::Hash.get(jobid)
+    job_to_json(res)
+  end
+
+  def remove_job(jobid)
+    Resque::Plugins::Status::Hash.remove(jobid)
+  end
+
+  def get_all_jobs()
+    Resque::Plugins::Status::Hash.status_ids
+  end
+
+  def job_to_json(job)
+    return nil unless job
+    res = {
+      :job_id => job.uuid,
+      :status => job.status,
+      :start_time => job.time.to_s,
+      :description => job.options[:description] || "None"
+    }
+    res[:complete_time] = job.complete_time if job.complete_time
+    res[:result] = validate_message(job.message) if job.message
+    res
+  end
+
+  def validate_message(msg)
+    Yajl::Parser.parse(msg)
+  rescue => e
+    # generate internal error if we can't parse err msg
+    ServiceError.new(ServiceError::INTERNAL_ERROR).to_hash
+  end
+end

data/lib/base/job/config.rb

@@ -0,0 +1,27 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+require "redis"
+require "resque"
+
+module VCAP
+  module Services
+    module Base
+      module AsyncJob
+      end
+    end
+  end
+end
+
+class VCAP::Services::Base::AsyncJob::Config
+  class << self
+    attr_reader :redis_config, :redis, :logger
+    def redis_config=(config)
+      @redis_config = config
+      @redis = ::Redis.new config
+      Resque.redis = @redis
+    end
+
+    def logger=(logger)
+      @logger = logger
+    end
+  end
+end

data/lib/base/job/lock.rb

@@ -0,0 +1,153 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require "logger"
+require "redis"
+
+$LOAD_PATH.unshift File.dirname(__FILE__)
+require "config"
+
+# redis locking primitive using setnx.
+# http://redis.io/commands/setnx
+module VCAP::Services::Base::AsyncJob
+  class Lock
+    attr_reader :expiration, :timeout, :name
+    include VCAP::Services::Base::Error
+
+    # Options for lock
+    # name       - The uuid of the lock
+    # timeout    - The time that waits to acquire the lock, default 20 seconds
+    # expiration - Lock expires in given seconds if not refreshed, default 10 seconds
+    # logger     - The logger..
+    # ttl        - The max time that a thread can acquire the lock, default 600 seconds. Lock raise +JOB_TIMEOUT+ error once the ttl is exceeded.
+    def initialize(name, opts={})
+      @name = name
+      @timeout = opts[:timeout] || 20 #seconds
+      @expiration = opts[:expiration] || 10  # seconds
+      @ttl = opts[:ttl] || 600 # seconds
+      @logger = opts[:logger] || make_logger
+      config = Config.redis_config
+      raise "Can't find configuration of redis." unless config
+      @redis = ::Redis.new(config)
+      @released_thread = {}
+    end
+
+    def make_logger
+      logger = Logger.new(STDOUT)
+      logger.level = Logger::ERROR
+      logger
+    end
+
+    def lock
+      @logger.debug("Acquiring lock: #{@name}")
+      started = Time.now.to_f
+      expiration = started.to_f + @expiration + 1
+      until @redis.setnx(@name, expiration)
+        existing_lock = @redis.get(@name)
+        if existing_lock.to_f < Time.now.to_f
+          @logger.debug("Lock #{@name} is expired, trying to acquire it.")
+          break if watch_and_update(@redis, expiration)
+        end
+
+        raise ServiceError.new(ServiceError::JOB_QUEUE_TIMEOUT, @timeout)if Time.now.to_f - started > @timeout
+
+        sleep(1)
+
+        expiration = Time.now.to_f + @expiration + 1
+      end
+
+      @lock_expiration = expiration
+      refresh_thread = setup_refresh_thread
+      @logger.debug("Lock #{@name} is acquired, will expire at #{@lock_expiration}")
+
+      begin
+        Timeout::timeout(@ttl) do
+          yield if block_given?
+        end
+      rescue Timeout::Error =>e
+        raise ServiceError.new(ServiceError::JOB_TIMEOUT, @ttl)
+      ensure
+        release_thread(refresh_thread)
+        delete
+      end
+    end
+
+    def watch_and_update(redis, expiration)
+      redis.watch(@name)
+      res = redis.multi do
+        redis.set(@name, expiration)
+      end
+      if res
+        @logger.debug("Lock #{@name} is renewed and acquired.")
+      else
+        @logger.debug("Lock #{@name} was updated by others.")
+      end
+      res
+    end
+
+    def release_thread t
+      # gracefully terminate refresh thread.
+      @released_thread[t.object_id] = true
+      waited = 0
+      while (waited += 1) <= 5
+        # thread is terminated when t.status == nil or false
+        return unless t.status
+        sleep 1
+      end
+      # force terminate after wait 5 seconds.
+      t.exit
+    end
+
+    def released?
+      @released_thread[Thread.current.object_id]
+    end
+
+    def setup_refresh_thread
+      t = Thread.new do
+        redis = ::Redis.new(Config.redis_config)
+        sleep_interval = [1.0, @expiration/2].max.to_i
+        begin
+          while not released? do
+            @logger.debug("Renewing lock #{@name}")
+            redis.watch(@name)
+            existing_lock = redis.get(@name)
+
+            break if existing_lock.to_f > @lock_expiration # lock has been updated by others
+            expiration = Time.now.to_f + @expiration + 1
+            break unless watch_and_update(redis, expiration)
+            @lock_expiration = expiration
+            sleep_interval.times do
+              sleep 1
+              break if released?
+            end
+          end
+        rescue => e
+          @logger.error("Can't renew lock #{@name}, #{e}")
+        ensure
+          begin
+            @logger.debug("Lock renew thread for #{@name} exited.")
+            redis.quit
+          rescue => e
+            # just logging, ignore error
+            @logger.debug("Ignore error when quit: #{e}")
+          end
+        end
+      end
+      t
+    end
+
+    def delete
+      @logger.debug("Deleting lock: #{@name}")
+      existing_lock = @redis.get(@name)
+      @logger.debug("Lock #{@name} is acquired by others.")if existing_lock.to_f > @lock_expiration
+      @redis.watch(@name)
+      res = @redis.multi do
+        @redis.del(@name)
+      end
+      if res
+        @logger.debug("Lock #{@name} is deleted.")
+      else
+        @logger.debug("Lock #{@name} is acquired by others.")
+      end
+      true
+    end
+  end
+end

data/lib/base/job/package.rb

@@ -0,0 +1,112 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+require "zip/zip"
+require "yajl"
+require 'vcap/common'
+
+module VCAP::Services::Base::AsyncJob
+
+  class Package
+    include VCAP::Services::Base::Error
+
+    MANIFEST_FILE = "manifest".freeze
+    CONTENT_FOLDER = "content".freeze
+    attr_reader :manifest
+
+    class << self
+      def load path
+        raise "File #{path} not exists." unless File.exists? path
+        p = new(path)
+        p.load_manifest
+        p
+      end
+    end
+
+    def initialize(zipfile, opts={})
+      @zipfile = zipfile
+      @files = {}
+      @manifest = {}
+      @filemode = opts[:mode] || 0644
+    end
+
+    def add_files(files)
+      files =  Array(files)
+      files.each do |file|
+        raise "File #{file} not found." unless File.exists? file
+        raise "File #{file} is not readable." unless File.readable? file
+        basename = File.basename file
+        @files[basename] = file
+      end
+    end
+
+    # add +hash+ to manifest file.
+    def manifest=(hash)
+      return unless hash
+      raise "Input should be Hash" unless hash.is_a? Hash
+      @manifest.merge! VCAP.symbolize_keys(hash)
+    end
+
+    # package files and manifest in +zipfile+. If +force+ is true, we'll try to delete the target +zipfile+ if it already exists.
+    def pack(force=nil)
+      if File.exists? @zipfile
+        if force
+          File.delete @zipfile
+        else
+          raise "File #{@zipfile} already exists."
+        end
+      end
+
+      dirname = File.dirname(@zipfile)
+      raise "Dir #{dirname} is not exists." unless File.exists? dirname
+      raise "Dir #{dirname} is not writable." unless File.writable? dirname
+
+      Zip::ZipFile.open(@zipfile, Zip::ZipFile::CREATE) do |zf|
+        # manifest file
+        zf.get_output_stream(MANIFEST_FILE) {|o| o << Yajl::Encoder.encode(@manifest)}
+
+        @files.each do |f, path|
+          zf.add("#{CONTENT_FOLDER}/#{f}", path)
+        end
+      end
+
+      begin
+        File.chmod(@filemode, @zipfile)
+      rescue => e
+        raise "Fail to change the mode of #{@zipfile} to #{@filemode.to_s(8)}: #{e}"
+      end
+    end
+
+    # unpack the content to +path+ and return extraced file list.
+    def unpack path
+      raise "File #{@zipfile} not exists." unless File.exists? @zipfile
+      raise "unpack path: #{path} not found." unless Dir.exists? path
+      raise "unpack path: #{path} is not writable." unless File.writable? path
+
+      files = []
+      Zip::ZipFile.foreach(@zipfile) do |entry|
+        next if entry.to_s == MANIFEST_FILE
+        entry_name = File.basename entry.to_s
+        dst_path = File.join(path, entry_name)
+        dirname = File.dirname(dst_path)
+        FileUtils.mkdir_p(dirname) unless File.exists? dirname
+        files << dst_path
+        entry.extract(dst_path)
+      end
+      files.freeze
+      yield files if block_given?
+      files
+    rescue => e
+      # auto cleanup if error raised.
+      files.each{|f| File.delete f if File.exists? f} if files
+      raise ServiceError.new(ServiceError::FILE_CORRUPTED) if e.is_a? Zlib::DataError
+      raise e
+    end
+
+    # read manifest in a zip file
+    def load_manifest
+      zf = Zip::ZipFile.open(@zipfile)
+      @manifest = VCAP.symbolize_keys(Yajl::Parser.parse(zf.read(MANIFEST_FILE)))
+    rescue Errno::ENOENT => e
+      raise ServiceError.new(ServiceError::BAD_SERIALIZED_DATAFILE, "request. Missing manifest.")
+    end
+  end
+end