bosh-director 1.5.0.pre.1113

data/lib/bosh/director/download_helper.rb

@@ -0,0 +1,44 @@
+module Bosh::Director
+  module DownloadHelper
+    require 'net/http'
+    require 'timeout'
+    require 'uri'
+
+    ##
+    # Downloads a remote file
+    #
+    # @param [String] resource Resource name to be logged
+    # @param [String] remote_file Remote file to download
+    # @param [String] local_file Local file to store the downloaded file
+    # @raise [Bosh::Director::ResourceNotFound] If remote file is not found
+    # @raise [Bosh::Director::ResourceError] If there's a network problem
+    def download_remote_file(resource, remote_file, local_file)
+      @logger.info("Downloading remote #{resource} from #{remote_file}") if @logger
+      uri = URI.parse(remote_file)
+      Net::HTTP.start(uri.host, uri.port, 
+                      :use_ssl => uri.scheme == 'https',
+                      :verify_mode => OpenSSL::SSL::VERIFY_NONE) do |http|
+        http.request_get(uri.request_uri) do |response|
+          unless response.kind_of? Net::HTTPSuccess
+            @logger.error("Downloading remote #{resource} from #{remote_file} failed: #{response.message}") if @logger
+            if response.kind_of? Net::HTTPNotFound 
+              raise ResourceNotFound, "No #{resource} found at `#{remote_file}'."
+            else
+              raise ResourceError, "Downloading remote #{resource} failed. Check task debug log for details."
+            end
+          end
+            
+          File.open(local_file, 'wb') do |file|
+            response.read_body do |chunk|
+              file.write(chunk)
+            end
+          end
+        end
+      end       
+    rescue URI::Error, SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, Errno::ECONNREFUSED, EOFError,
+           Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError => e
+      @logger.error("Downloading remote #{resource} from #{remote_file} failed: #{e.inspect}") if @logger
+      raise ResourceError, "Downloading remote #{resource} failed. Check task debug log for details."
+    end
+  end
+end

data/lib/bosh/director/duration.rb

@@ -0,0 +1,36 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module Duration
+
+    def duration(delta)
+      seconds = delta % 60
+      delta = (delta / 60).floor
+      minutes = delta % 60
+      delta = (delta / 60).floor
+      hours = delta % 24
+      delta = (delta / 24).floor
+      days = delta
+
+      result = []
+
+      duration_helper(days, result, "day")
+      duration_helper(hours, result, "hour")
+      duration_helper(minutes, result, "minute")
+      duration_helper(seconds, result, "second")
+
+      result << "0 seconds" if result.empty?
+
+      result.join(" ")
+    end
+
+    def duration_helper(value, result, unit)
+      if value > 0
+        result << "#{value} #{value == 1 ? unit : "#{unit}s"}"
+      end
+    end
+
+    module_function :duration, :duration_helper
+
+  end
+end

data/lib/bosh/director/encryption_helper.rb

@@ -0,0 +1,10 @@
+module Bosh::Director
+  module EncryptionHelper
+    def generate_agent_credentials
+      ["crypt_key", "sign_key"].inject({}) do |credentials, key|
+        credentials[key] = SecureRandom.base64(48)
+        credentials
+      end
+    end
+  end
+end

data/lib/bosh/director/errors.rb

@@ -0,0 +1,198 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  include Api::Http
+
+  class ProblemHandlerError < StandardError; end
+
+  # DirectorError is a generic exception for most of the errors originated
+  # in BOSH Director.
+  class DirectorError < StandardError
+    # Wraps an exception to DirectorError, so it can be assigned a generic
+    # error code and properly logged.
+    # @param [Exception] exception
+    # @return [DirectorError] Director error
+    def self.create_from_exception(exception)
+      if exception.kind_of?(DirectorError)
+        exception
+      else
+        DirectorError.new(exception.message)
+      end
+    end
+
+    # Creates a new subclass of DirectorError with
+    # given name, error code and response code
+    # @param [Fixnum] error_code Error code
+    # @param [Fixnum] response_code HTTP response code
+    # @return [Class]
+    def self.define_error(error_code, response_code)
+      Class.new(DirectorError) do
+        define_method(:initialize) do |*args|
+          message = args[0]
+          super(message)
+          @error_code = error_code
+          @response_code = response_code
+        end
+      end
+    end
+
+    attr_reader :response_code
+    attr_reader :error_code
+
+    def initialize(message = nil)
+      super
+      @response_code = 500
+      @error_code = 100
+      @format = "Director error: %s"
+    end
+  end
+
+  def self.err(error_code, response_code = BAD_REQUEST)
+    DirectorError.define_error(error_code, response_code)
+  end
+
+  TaskNotFound = err(10000, NOT_FOUND)
+  TaskCancelled = err(10001, OK)
+
+  UserNotFound = err(20000, NOT_FOUND)
+  UserImmutableUsername = err(20001)
+  UserInvalid = err(20002)
+  UserNameTaken = err(20003)
+
+  ReleaseAlreadyExists = err(30000)
+  ReleaseExistingPackageHashMismatch = err(30001)
+  ReleaseInvalidArchive = err(30002)
+  ReleaseManifestNotFound = err(30003)
+  ReleaseExistingJobHashMismatch = err(30004)
+  ReleaseNotFound = err(30005, NOT_FOUND)
+  ReleaseVersionNotFound = err(30006, NOT_FOUND)
+  ReleaseInUse = err(30007)
+  ReleaseVersionInUse = err(30008)
+  ReleaseDeleteFailed = err(30009)
+
+  ValidationInvalidType = err(40000)
+  ValidationMissingField = err(40001)
+  ValidationViolatedMin = err(40002)
+  ValidationViolatedMax = err(40003)
+
+  StemcellInvalidArchive = err(50000)
+  StemcellImageNotFound = err(50001)
+  StemcellAlreadyExists = err(50002)
+  StemcellNotFound = err(50003, NOT_FOUND)
+  StemcellInUse = err(50004)
+
+  PackageInvalidArchive = err(60000)
+
+  DeploymentNotFound = err(70000, NOT_FOUND)
+  InstanceNotFound = err(70001, NOT_FOUND)
+  InstanceInvalidIndex = err(70002)
+  InstanceDeploymentMissing = err(70003)
+  InstanceVmMissing = err(70004)
+  VmAgentIdMissing = err(70005)
+  VmCloudIdMissing = err(70006)
+  VmInstanceOutOfSync = err(70006)
+  InstanceTargetStateUndefined = err(70007)
+  SnapshotNotFound = err(70008)
+
+  JobInvalidArchive = err(80000)
+  JobMissingManifest = err(80001)
+  JobMissingTemplateFile = err(80002)
+  JobMissingPackage = err(80003)
+  JobMissingMonit = err(80004)
+  JobInvalidLogSpec = err(80005)
+  JobTemplateBindingFailed = err(80006)
+  JobTemplateUnpackFailed = err(80007)
+  JobInvalidPropertySpec = err(80008)
+  JobInvalidPropertyMapping = err(80009)
+  JobIncompatibleSpecs = err(80010)
+
+  ResourceError = err(100001)
+  ResourceNotFound = err(100002, NOT_FOUND)
+
+  PropertyAlreadyExists = err(110001)
+  PropertyInvalid = err(110002)
+  PropertyNotFound = err(110003, NOT_FOUND)
+
+  CompilationConfigUnknownNetwork = err(120001)
+
+  NetworkReservationInvalidIp = err(130001)
+  NetworkReservationMissing = err(130002)
+  NetworkReservationAlreadyExists = err(130003)
+  NetworkReservationInvalidType = err(130004)
+  NetworkReservationIpMissing = err(130005)
+  NetworkReservationIpNotOwned = err(130006)
+  NetworkReservationVipDefaultProvided = err(130007)
+  NetworkReservationAlreadyInUse = err(130008)
+  NetworkReservationWrongType = err(130009)
+  NetworkReservationError = err(130010)
+  NetworkReservationNotEnoughCapacity = err(130010)
+
+  JobMissingRelease = err(140001)
+  JobUnknownRelease = err(140002)
+  JobUnknownResourcePool = err(140003)
+  JobInvalidInstanceIndex = err(140004)
+  JobInvalidInstanceState = err(140005)
+  JobInvalidJobState = err(140006)
+  JobMissingNetwork = err(140007)
+
+  JobUnknownNetwork = err(150001)
+  JobNetworkInstanceIpMismatch = err(150002)
+  JobNetworkInvalidDefault = err(150003)
+  JobNetworkMultipleDefaults = err(150004)
+  JobNetworkMissingDefault = err(150005)
+
+  NetworkOverlappingSubnets = err(160001)
+  NetworkInvalidRange = err(160002)
+  NetworkInvalidGateway = err(160003)
+  NetworkInvalidDns = err(160004)
+  NetworkReservedIpOutOfRange = err(160005)
+  NetworkStaticIpOutOfRange = err(160006)
+
+  ResourcePoolUnknownNetwork = err(170001)
+  ResourcePoolNotEnoughCapacity = err(170002)
+
+  UpdateConfigInvalidWatchTime = err(180001)
+
+  DeploymentAmbiguousReleaseSpec = err(190001)
+  DeploymentDuplicateReleaseName = err(190002)
+  DeploymentDuplicateResourcePoolName = err(190003)
+  DeploymentRenamedJobNameStillUsed = err(190004)
+  DeploymentCanonicalJobNameTaken = err(190005)
+  DeploymentCanonicalNetworkNameTaken = err(190006)
+  DeploymentNoNetworks = err(190007)
+  DeploymentCanonicalNameTaken = err(190008)
+  DeploymentInvalidNetworkType = err(190009)
+  DeploymentUnknownTemplate = err(190012)
+
+  CloudDiskNotAttached = err(390001)
+  CloudDiskMissing = err(390002)
+  CloudNotEnoughDiskSpace = err(390003)
+
+  AgentTaskNoBlobstoreId = err(400001)
+  AgentInvalidStateFormat = err(400002)
+  AgentWrongDeployment = err(400003)
+  AgentUnexpectedJob = err(400004)
+  AgentRenameInProgress = err(400005)
+  AgentJobMismatch = err(400006)
+  AgentJobNotRunning = err(400007)
+  AgentJobNotStopped = err(400008)
+  AgentUnexpectedDisk = err(400009)
+  AgentDiskOutOfSync = err(400010)
+
+  CloudcheckTooManySimilarProblems = err(410001)
+  CloudcheckResolutionNotProvided = err(410002)
+  CloudcheckInvalidResolutionFormat = err(410003)
+
+  DnsInvalidCanonicalName = err(420001)
+
+  PackageCompilationNetworkNotReserved = err(430001)
+  PackageCompilationNotEnoughWorkersForReuse = err(430002)
+
+  BadManifest = err(440001)
+
+  RpcRemoteException = err(450001)
+  RpcTimeout = err(450002)
+
+  SystemError = err(500000, INTERNAL_SERVER_ERROR)
+  NotEnoughDiskSpace = err(500001, INTERNAL_SERVER_ERROR)
+end

data/lib/bosh/director/event_log.rb

@@ -0,0 +1,136 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  class EventLog
+
+    # Event log conventions:
+    # All event log entries having same "stage" logically belong to the same
+    # event group. "tags" is an array of strings that supposed to act as hint
+    # of what is being tracked.
+    # Each "<stage name> <tags>" combination represents a number of steps that
+    # is tracked separately. "task" is just a description of current step that
+    # acts as a hint for event log renderer, i.e. it can be displayed next to
+    # progress bar.
+    # Example: "job_update" stage might have two tasks: "canary update" and
+    # "update" and a separate tag for each job name, so each job will have
+    # a separate progress bar in CLI.
+
+    # Sample rendering for event log entry:
+    # {
+    #   "time":1312233461,"stage":"job_update","task":"update",
+    #   "tags":["mysql_node"], "index":2,"total":4,"state":"finished",
+    #   "progress":50,"data":{"key1" => "value1"}
+    # }
+
+    # Job update (mysql_node):
+    # update |--------        | (2/4) 50%
+
+    attr_reader :stage
+    attr_reader :total
+    attr_reader :counter
+
+    def initialize(io = nil)
+      @logger = EventLogger.new(io || StringIO.new)
+      @lock = Mutex.new
+      @counter = 0
+    end
+
+    def begin_stage(stage, total = nil, tags = [])
+      @lock.synchronize do
+        @stage = stage
+        @tags = tags
+        @counter = 0
+        @total = total
+      end
+    end
+
+    def track(task = nil)
+      index = nil
+      @lock.synchronize do
+        @counter += 1
+        index = @counter
+      end
+
+      ticker = EventTicker.new(self, task, index)
+
+      start_task(task, index)
+      begin
+        yield ticker if block_given?
+      rescue => e
+        task_failed(task, index, 100, e.to_s)
+        raise
+      end
+      finish_task(task, index)
+    end
+
+    # Adds an error entry to the event log.
+    # @param [DirectorError] error Director error
+    # @return [void]
+    def log_error(error)
+      entry = {
+        :time => Time.now.to_i,
+        :error => {
+          :code => error.error_code,
+          :message => error.message
+        }
+      }
+
+      @logger.info(Yajl::Encoder.encode(entry))
+    end
+
+    def start_task(task, index, progress = 0)
+      log_task(task, "started", index, progress)
+    end
+
+    def finish_task(task, index, progress = 100)
+      log_task(task, "finished", index, progress)
+    end
+
+    def task_failed(task, index, progress = 100, error = nil)
+      log_task(task, "failed", index, progress, {"error" => error})
+    end
+
+    def log_task(task, state, index, progress = 0, data = {})
+      entry = {
+        :time => Time.now.to_i,
+        :stage => @stage,
+        :task => task,
+        :tags => @tags,
+        :index => index,
+        :total => @total,
+        :state => state,
+        :progress => progress,
+      }
+
+      if data.size > 0
+        entry[:data] = data
+      end
+
+      @logger.info(Yajl::Encoder.encode(entry))
+    end
+
+  end
+
+  class EventLogger < Logger
+    def format_message(level, time, progname, msg)
+      msg + "\n"
+    end
+  end
+
+  # Sometimes task needs to be split into subtasks so we can track its progress
+  # with more granularity. In that case we can use EventTicker helper class
+  # to advance progress between N and N+1 by small increments.
+  class EventTicker
+    def initialize(event_log, task, index)
+      @event_log = event_log
+      @task = task
+      @index = index
+      @progress = 0
+    end
+
+    def advance(delta, data = {})
+      @progress = [@progress + delta, 100].min
+      @event_log.log_task(@task, "in_progress", @index, @progress.to_i, data)
+    end
+  end
+end

data/lib/bosh/director/ext.rb

@@ -0,0 +1,64 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Resque
+  def redis
+    Bosh::Director::Config.redis
+  end
+end
+
+class Object
+ def to_openstruct
+   self
+ end
+end
+
+class Array
+ def to_openstruct
+   map { |el| el.to_openstruct }
+ end
+end
+
+class Hash
+  def recursive_merge!(other)
+    self.merge!(other) do |_, old_value, new_value|
+      if old_value.class == Hash && new_value.class == Hash
+        old_value.recursive_merge!(new_value)
+      else
+        new_value
+      end
+    end
+    self
+  end
+
+ def to_openstruct
+   mapped = {}
+   each { |key, value| mapped[key] = value.to_openstruct }
+   OpenStruct.new(mapped)
+ end
+end
+
+require "sequel/connection_pool/threaded"
+
+class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
+
+  alias_method :acquire_original, :acquire
+  alias_method :release_original, :release
+
+  def acquire(thread)
+    logger = Bosh::Director::Config.logger
+    result = acquire_original(thread)
+    if logger
+      logger.debug("Acquired connection: #{@allocated[thread].object_id}")
+    end
+    result
+  end
+
+  def release(thread)
+    logger = Bosh::Director::Config.logger
+    if logger
+      logger.debug("Released connection: #{@allocated[thread].object_id}")
+    end
+    release_original(thread)
+  end
+
+end