vcap_services_base 0.2.10

data/lib/base/service_error.rb

@@ -0,0 +1,122 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require "rubygems"
+require "yajl"
+
+module VCAP
+  module Services
+    module Base
+      module Error
+        class ServiceError < StandardError
+          attr_reader :http_status, :error_code, :error_msg
+
+          # HTTP status code
+          HTTP_BAD_REQUEST     = 400
+          HTTP_NOT_AUTHORIZED  = 401
+          HTTP_FORBIDDEN       = 403
+          HTTP_NOT_FOUND       = 404
+          HTTP_INTERNAL        = 500
+          HTTP_NOT_IMPLEMENTED = 501
+          HTTP_SERVICE_UNAVAIL = 503
+          HTTP_GATEWAY_TIMEOUT = 504
+
+          # Error Code is defined here
+          #
+          # e.g.
+          # ERR_NAME  = [err_code, http_status,     err_message_template]
+          # NOT_FOUND = [30300,    HTTP_NOT_FOUND,  '%s not found!'    ]
+
+          # 30000 - 30099  400 Bad Request
+          INVALID_CONTENT = [30000, HTTP_BAD_REQUEST, 'Invalid Content-Type']
+          MALFORMATTED_REQ = [30001, HTTP_BAD_REQUEST, 'Malformatted request']
+          UNKNOWN_PLAN_UNIQUE_ID = [30002, HTTP_BAD_REQUEST, 'Unknown unique_id']
+          UNKNOWN_PLAN = [30003, HTTP_BAD_REQUEST, 'Unknown plan %s']
+          UNSUPPORTED_VERSION = [30004, HTTP_BAD_REQUEST, 'Unsupported version %s']
+
+          # 30100 - 30199  401 Unauthorized
+          NOT_AUTHORIZED = [30100, HTTP_NOT_AUTHORIZED, 'Not authorized']
+
+          # 30200 - 30299  403 Forbidden
+
+          # 30300 - 30399  404 Not Found
+          NOT_FOUND = [30300, HTTP_NOT_FOUND, '%s not found']
+
+          # 30500 - 30599  500 Internal Error
+          INTERNAL_ERROR = [30500, HTTP_INTERNAL, 'Internal Error']
+          EXTENSION_NOT_IMPL = [30501, HTTP_NOT_IMPLEMENTED, "Service extension %s is not implemented."]
+          NODE_OPERATION_TIMEOUT = [30502, HTTP_INTERNAL, "Node operation timeout"]
+          SERVICE_START_TIMEOUT = [30503, HTTP_INTERNAL, "Service start timeout"]
+          WARDEN_RUN_COMMAND_FAILURE = [30504, HTTP_INTERNAL, "Failed to run command %s in warden container %s, exit status: %d, stdout: %s, stderr: %s"]
+
+          # 30600 - 30699  503 Service Unavailable
+          SERVICE_UNAVAILABLE = [30600, HTTP_SERVICE_UNAVAIL, 'Service unavailable']
+
+          # 30700 - 30799  500 Gateway Timeout
+          GATEWAY_TIMEOUT = [30700, HTTP_GATEWAY_TIMEOUT, 'Gateway timeout']
+
+          # 30800 - 30899 500 Lifecycle error
+          OVER_QUOTA = [30800, HTTP_INTERNAL, "Instance %s has %s snapshots. Quota is %s "]
+          JOB_QUEUE_TIMEOUT = [30801, HTTP_INTERNAL, "Job timeout after waiting for %s seconds."]
+          JOB_TIMEOUT = [30802, HTTP_INTERNAL, "Job is killed since it runs longer than ttl: %s seconds."]
+          BAD_SERIALIZED_DATAFILE = [30803, HTTP_INTERNAL, "Invalid serialized data file from: %s"]
+          FILESIZE_TOO_LARGE = [30804, HTTP_BAD_REQUEST, "Size of file from url %s is %s, max allowed %s"]
+          TOO_MANY_REDIRECTS = [30805, HTTP_BAD_REQUEST, "Too many redirects from url:%s, max redirects allowed is %s"]
+          FILE_CORRUPTED = [30806, HTTP_BAD_REQUEST, "Serialized file is corrupted."]
+          REDIS_CONCURRENT_UPDATE = [30807, HTTP_INTERNAL, "Server busy, please try again later."]
+          INVALID_SNAPSHOT_NAME = [30808, HTTP_BAD_REQUEST, "Invalid snapshot name. %s"]
+
+
+          # 31000 - 32000  Service-specific Error
+          # Defined in services directory, for example mongodb/lib/mongodb_service/
+
+          def initialize(code, *args)
+            @http_status = code[1]
+            @error_code  = code[0]
+            @error_msg   = sprintf(code[2], *args)
+          end
+
+          def to_s
+            "Error Code: #{@error_code}, Error Message: #{@error_msg}"
+          end
+
+          def to_hash
+            {
+              'status' => @http_status,
+              'msg' => {
+                'code' => @error_code,
+                'description' => @error_msg,
+                'error' => {
+                  'backtrace' => backtrace,
+                  'types' => self.class.ancestors.map(&:name) - Object.ancestors.map(&:name)
+                }
+              }
+            }
+          end
+        end
+
+
+        def success(response = true)
+          {'success' => true, 'response' => response}
+        end
+
+        def failure(exception)
+          {'success' => false, 'response' => exception.to_hash}
+        end
+
+        def internal_fail()
+          e = ServiceError.new(ServiceError::INTERNAL_ERROR)
+          failure(e)
+        end
+
+        def timeout_fail()
+          e = ServiceError.new(ServiceError::GATEWAY_TIMEOUT)
+          failure(e)
+        end
+
+        def parse_msg(msg)
+          Yajl::Parser.parse(msg)
+        end
+
+      end
+    end
+  end
+end

data/lib/base/service_message.rb

@@ -0,0 +1,94 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+#
+require 'api/message'
+
+module VCAP::Services::Internal
+
+  # Provisioner --> Node
+  class ProvisionRequest < ServiceMessage
+    required :plan, String
+    optional :credentials
+    optional :version
+  end
+
+  # Node --> Provisioner
+  class ProvisionResponse < ServiceMessage
+    required :success
+    optional :credentials
+    optional :error
+  end
+
+  class UnprovisionRequest < ServiceMessage
+    required :name, String
+    required :bindings, [Hash]
+  end
+
+  class BindRequest < ServiceMessage
+    required :name, String
+    optional :bind_opts, Hash
+    optional :credentials
+  end
+
+  class BindResponse < ServiceMessage
+    required :success
+    optional :credentials
+    optional :error
+  end
+
+  class UnbindRequest < ServiceMessage
+    required :credentials
+  end
+
+  class SimpleResponse < ServiceMessage
+    required :success
+    optional :error
+  end
+
+  class RestoreRequest < ServiceMessage
+    required :instance_id
+    required :backup_path
+  end
+
+  class NodeHandlesReport < ServiceMessage
+    required :instances_list
+    required :bindings_list
+    required :node_id
+  end
+
+  class PurgeOrphanRequest < ServiceMessage
+    # A list of orphan instances names
+    required :orphan_ins_list
+    # A list of orphan bindings credentials
+    required :orphan_binding_list
+  end
+
+  class ServiceHandle < ServiceMessage
+    required :service_id,     String
+    required :configuration,  Hash
+    required :credentials,    Hash
+    optional :dashboard_url,  String
+  end
+
+  class ServiceInstanceHandleV2 < ServiceMessage
+    required :gateway_data,         Hash
+    required :credentials,          Hash
+    optional :name,                 String
+    optional :service_plan_guid,    String
+    optional :space_guid,           String
+    optional :service_bindings_url, String
+    optional :space_url,            String
+    optional :service_plan_url,     String
+    optional :dashboard_url,        String
+  end
+
+  class ServiceBindingHandleV2 < ServiceMessage
+    required :credentials,           Hash
+    required :gateway_data,          Hash
+    required :gateway_name,          String
+    optional :app_guid,              String
+    optional :service_instance_guid, String
+    optional :binding_options,       Hash
+    optional :app_url,               String
+    optional :service_instance_url,  String
+  end
+end

data/lib/base/service_plan_change_set.rb

@@ -0,0 +1,11 @@
+module VCAP::Services
+  class ServicePlanChangeSet
+    attr_reader :service, :plans_to_add, :plans_to_update, :service_guid
+    def initialize(service, service_guid, options = {})
+      @service = service
+      @service_guid = service_guid
+      @plans_to_add = options[:plans_to_add] || []
+      @plans_to_update = options[:plans_to_update] || []
+    end
+  end
+end

data/lib/base/simple_aop.rb

@@ -0,0 +1,63 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+# Simple before filter to instance methods. We can chain filters to multiple methods.
+# Filter method can return nil or raise error to terminater the evoke chain.
+#
+# Example:
+# Class MyClass
+#   include Before
+#
+#   def f1(msg) puts msg end
+#
+#   def f2
+#     puts "f2"
+#   end
+#
+#   def before_filter
+#     puts "before"
+#
+#     true  # return true to proceed
+#   end
+#
+#   # use before method after methods are defined
+#   before [:f1, :f2], :before_filter
+# end
+#
+# c = MyClass.new
+# c.f1("hello")
+# c.f2
+# =>
+# before
+# hello
+# before
+# f2
+module Before
+
+  def self.included(base)
+    base.extend ClassMethods
+  end
+
+  module ClassMethods
+    PREFIX = "___".freeze
+
+    def before(methods, callbacks)
+      Array(methods).each do | method|
+        method = method.to_sym
+        callbacks = Array(callbacks).map{|callback| callback.to_sym}
+
+        enhance_method(method, callbacks)
+      end
+    end
+
+    # enhance single method with callbacks
+    def enhance_method(method, callbacks)
+      _method = (PREFIX + method.to_s).to_sym
+      alias_method _method, method
+
+      self.send(:define_method, method) do |*args, &blk|
+        [callbacks, _method].flatten.each do |callback|
+          break unless self.send(callback, *args, &blk)
+        end
+      end
+    end
+  end
+end

data/lib/base/snapshot_v2/snapshot.rb

@@ -0,0 +1,227 @@
+# Copyright (c) 2009-2011 VMware, Inc.
+require "resque-status"
+require "fileutils"
+require "uuid"
+
+require_relative "../service_error"
+require_relative "../../base/job/package.rb"
+require_relative "snapshot_client"
+
+module VCAP::Services::Base::SnapshotV2
+    # common utils for snapshot job
+    class SnapshotJob
+      attr_reader :name, :snapshot_id
+
+      # FIXME: untangle mixins
+      # include Snapshot
+      include Resque::Plugins::Status
+
+      class << self
+        def queue_lookup_key
+          :node_id
+        end
+
+        def select_queue(*args)
+          result = nil
+          args.each do |arg|
+            result = arg[queue_lookup_key]if (arg.is_a? Hash)&& (arg.has_key?(queue_lookup_key))
+          end
+          @logger = Config.logger
+          @logger.info("Select queue #{result} for job #{self.class} with args:#{args.inspect}") if @logger
+          result
+        end
+      end
+
+      def initialize(*args)
+        super(*args)
+        parse_config
+        init_worker_logger
+        Snapshot.redis_connect
+      end
+
+      def fmt_error(e)
+        "#{e}: [#{e.backtrace.join(" | ")}]"
+      end
+
+      def init_worker_logger
+        @logger = Config.logger
+      end
+
+      def required_options(*args)
+        missing_opts = args.select{|arg| !options.has_key? arg.to_s}
+        raise ArgumentError, "Missing #{missing_opts.join(', ')} in options: #{options.inspect}" unless missing_opts.empty?
+      end
+
+      def create_lock
+        lock_name = "lock:lifecycle:#{name}"
+        ttl = @config['job_ttl'] || 600
+        lock = Lock.new(lock_name, :logger => @logger, :ttl => ttl)
+        lock
+      end
+
+      def get_dump_path(name, snapshot_id)
+        snapshot_filepath(@config["snapshots_base_dir"], @config["service_name"], name, snapshot_id)
+      end
+
+      def parse_config
+        @config = Yajl::Parser.parse(ENV['WORKER_CONFIG'])
+        raise "Need environment variable: WORKER_CONFIG" unless @config
+      end
+
+      def cleanup(name, snapshot_id)
+        return unless name && snapshot_id
+        @logger.info("Clean up snapshot and files for #{name}, snapshot id: #{snapshot_id}")
+        delete_snapshot(name, snapshot_id)
+        FileUtils.rm_rf(get_dump_path(name, snapshot_id))
+      end
+
+      def handle_error(e)
+        @logger.error("Error in #{self.class} uuid:#{@uuid}: #{fmt_error(e)}")
+        err = (e.instance_of?(ServiceError)? e : ServiceError.new(ServiceError::INTERNAL_ERROR)).to_hash
+        err_msg = Yajl::Encoder.encode(err["msg"])
+        failed(err_msg)
+      end
+    end
+
+    class BaseCreateSnapshotJob < SnapshotJob
+      # workflow template
+      # Sub class should implement execute method which returns hash represents of snapshot like:
+      # {:snapshot_id => 1,
+      #  :size => 100,
+      #  :files => ["my_snapshot.tgz", "readme.txt"]
+      #  :manifest => {:version => '1', :service => 'mysql'}
+      #  }
+      def perform
+        begin
+          required_options :service_id
+          @name = options["service_id"]
+          @metadata = VCAP.symbolize_keys(options["metadata"])
+          @logger.info("Launch job: #{self.class} for #{name} with metadata: #{@metadata}")
+
+          @snapshot_id = new_snapshot_id
+          lock = create_lock
+
+          @snapshot_files = []
+          lock.lock do
+            quota = @config["snapshot_quota"]
+            if quota
+              current = service_snapshots_count(name)
+              @logger.debug("Current snapshots count for #{name}: #{current}, max: #{quota}")
+              raise ServiceError.new(ServiceError::OVER_QUOTA, name, current, quota) if current >= quota
+            end
+
+            snapshot = execute
+            snapshot = VCAP.symbolize_keys snapshot
+            snapshot[:manifest] ||= {}
+            snapshot[:manifest].merge! @metadata
+            @logger.info("Results of create snapshot: #{snapshot.inspect}")
+
+            # pack snapshot_file into package
+            dump_path = get_dump_path(name, snapshot_id)
+            FileUtils.mkdir_p(dump_path)
+            package_file = "#{snapshot_id}.zip"
+
+            package = Package.new(File.join(dump_path, package_file))
+            package.manifest = snapshot[:manifest]
+            files = Array(snapshot[:files])
+            raise "No snapshot file to package." if files.empty?
+            files.each do |f|
+              full_path = File.join(dump_path, f)
+              @snapshot_files << full_path
+              package.add_files full_path
+            end
+            package.pack(dump_path)
+            @logger.info("Package snapshot file: #{File.join(dump_path, package_file)}")
+
+            # update snapshot metadata for package file
+            snapshot.delete(:files)
+            snapshot[:file] = package_file
+            snapshot[:date] = fmt_time
+            # add default service name
+            snapshot[:name] = "Snapshot #{snapshot[:date]}"
+
+            save_snapshot(name, snapshot)
+
+            completed(Yajl::Encoder.encode(filter_keys(snapshot)))
+            @logger.info("Complete job: #{self.class} for #{name}")
+          end
+        rescue => e
+          cleanup(name, snapshot_id)
+          handle_error(e)
+        ensure
+          set_status({:complete_time => Time.now.to_s})
+          @snapshot_files.each{|f| File.delete(f) if File.exists? f} if @snapshot_files
+        end
+      end
+    end
+
+    class BaseDeleteSnapshotJob < SnapshotJob
+      def perform
+        begin
+          required_options :service_id, :snapshot_id
+          @name = options["service_id"]
+          @snapshot_id = options["snapshot_id"]
+          @logger.info("Launch job: #{self.class} for #{name}")
+
+          lock = create_lock
+
+          lock.lock do
+            result = execute
+            @logger.info("Results of delete snapshot: #{result}")
+
+            delete_snapshot(name, snapshot_id)
+
+            completed(Yajl::Encoder.encode({:result => :ok}))
+            @logger.info("Complete job: #{self.class} for #{name}")
+          end
+        rescue => e
+          handle_error(e)
+        ensure
+          set_status({:complete_time => Time.now.to_s})
+        end
+      end
+
+      def execute
+        cleanup(name, snapshot_id)
+      end
+    end
+
+    class BaseRollbackSnapshotJob < SnapshotJob
+      attr_reader :manifest, :snapshot_files
+      # workflow template
+      # Subclass implement execute method which returns true for a successful rollback
+      def perform
+        begin
+          required_options :service_id, :snapshot_id
+          @name = options["service_id"]
+          @snapshot_id = options["snapshot_id"]
+          @logger.info("Launch job: #{self.class} for #{name}")
+
+          lock = create_lock
+
+          @snapshot_files = []
+          lock.lock do
+            # extract origin files from package
+            dump_path = get_dump_path(name, snapshot_id)
+            package_file = "#{snapshot_id}.zip"
+            package = Package.load(File.join(dump_path, package_file))
+            @manifest = package.manifest
+            @snapshot_files = package.unpack(dump_path)
+            @logger.debug("Unpack files from #{package_file}: #{@snapshot_files}")
+            raise "Package file doesn't contain snapshot file." if @snapshot_files.empty?
+
+            result = execute
+            @logger.info("Results of rollback snapshot: #{result}")
+
+            completed(Yajl::Encoder.encode({:result => :ok}))
+            @logger.info("Complete job: #{self.class} for #{name}")
+          end
+        rescue => e
+          handle_error(e)
+        ensure
+          set_status({:complete_time => Time.now.to_s})
+          @snapshot_files.each{|f| File.delete(f) if File.exists? f} if @snapshot_files
+        end
+      end
+    end
+end