bosh-director 1.3215.4.0 → 1.3232.0

data/lib/bosh/director/api/instance_lookup.rb

@@ -18,7 +18,7 @@ module Bosh::Director
         instance = Models::Instance.find(deployment: deployment, job: job_name, index: job_index)
         if instance.nil?
           raise InstanceNotFound,
-                "`#{deployment.name}/#{job_name}/#{job_index}' doesn't exist"
+                "'#{deployment.name}/#{job_name}/#{job_index}' doesn't exist"
         end
         instance
       end
@@ -27,7 +27,7 @@ module Bosh::Director
         instance = Models::Instance.find(deployment: deployment, job: job_name, uuid: uuid)
         if instance.nil?
           raise InstanceNotFound,
-                "`#{deployment.name}/#{job_name}/#{uuid}' doesn't exist"
+                "'#{deployment.name}/#{job_name}/#{uuid}' doesn't exist"
         end
         instance
       end

data/lib/bosh/director/api/instance_manager.rb

@@ -38,11 +38,11 @@ module Bosh::Director
       def agent_client_for(instance)
         unless instance.vm_cid
           raise InstanceVmMissing,
-                "`#{instance}' doesn't reference a VM"
+                "'#{instance}' doesn't reference a VM"
         end
 
         unless instance.agent_id
-          raise VmAgentIdMissing, "VM `#{instance.vm_cid}' doesn't have an agent id"
+          raise VmAgentIdMissing, "VM '#{instance.vm_cid}' doesn't have an agent id"
         end
 
         AgentClient.with_vm_credentials_and_agent_id(instance.credentials, instance.agent_id)

data/lib/bosh/director/api/local_identity_provider.rb

@@ -34,6 +34,14 @@ module Bosh
         def scopes
           ['bosh.admin']
         end
+
+        def username_or_client
+          username
+        end
+
+        def client
+          nil
+        end
       end
     end
   end

data/lib/bosh/director/api/property_manager.rb

@@ -12,10 +12,11 @@ module Bosh::Director
         property.save
 
       rescue Sequel::ValidationFailed => e
-        if e.errors[[:name, :deployment_id]].include?(:unique)
+
+        if e.errors[[:name, :deployment_id]] && e.errors[[:name, :deployment_id]].include?(:unique)
           raise PropertyAlreadyExists,
-                "Property `#{property_name}' already exists " +
-                "for deployment `#{deployment.name}'"
+                "Property '#{property_name}' already exists " +
+                "for deployment '#{deployment.name}'"
         end
         invalid_property(e.errors)
       end
@@ -38,8 +39,8 @@ module Bosh::Director
         property = Models::DeploymentProperty.find(filters)
         if property.nil?
           raise PropertyNotFound,
-                "Property `#{property_name}' not found " +
-                "for deployment `#{deployment.name}'"
+                "Property '#{property_name}' not found " +
+                "for deployment '#{deployment.name}'"
         end
         property
       end

data/lib/bosh/director/api/release_manager.rb

@@ -4,7 +4,7 @@ module Bosh::Director
       include ApiHelper
 
       def get_all_releases
-        releases = Models::Release.order_by(:name.asc).map do |release|
+        releases = Models::Release.order_by(Sequel.asc(:name)).map do |release|
           release_versions = sorted_release_versions(release)
           {
             'name' => release.name,
@@ -39,7 +39,7 @@ module Bosh::Director
       def find_by_name(name)
         release = Models::Release[:name => name]
         if release.nil?
-          raise ReleaseNotFound, "Release `#{name}' doesn't exist"
+          raise ReleaseNotFound, "Release '#{name}' doesn't exist"
         end
         release
       end
@@ -64,7 +64,7 @@ module Bosh::Director
           end
           if release_version.nil?
             raise ReleaseVersionNotFound,
-                  "Release version `#{release.name}/#{version}' doesn't exist"
+                  "Release version '#{release.name}/#{version}' doesn't exist"
           end
         end
 

data/lib/bosh/director/api/resource_manager.rb

@@ -76,9 +76,9 @@ module Bosh::Director
       def blobstore_resource(id)
         yield @blobstore_client
       rescue Bosh::Blobstore::NotFound
-        raise ResourceNotFound, "Resource `#{id}' not found in the blobstore"
+        raise ResourceNotFound, "Resource '#{id}' not found in the blobstore"
       rescue Bosh::Blobstore::BlobstoreError => e
-        raise ResourceError, "Blobstore error accessing resource `#{id}': #{e}"
+        raise ResourceError, "Blobstore error accessing resource '#{id}': #{e}"
       end
     end
   end

data/lib/bosh/director/api/resurrector_manager.rb

@@ -15,7 +15,7 @@ module Bosh::Director
       end
 
       def set_pause_for_all(desired_state)
-        Models::Instance.update(resurrection_paused: desired_state)
+        Models::Instance.dataset.update(resurrection_paused: desired_state)
       end
     end
   end

data/lib/bosh/director/api/route_configuration.rb

@@ -30,6 +30,7 @@ module Bosh
           controllers['/task'] = Bosh::Director::Api::Controllers::TaskController.new(@config)
           controllers['/tasks'] = Bosh::Director::Api::Controllers::TasksController.new(@config)
           controllers['/users'] = Bosh::Director::Api::Controllers::UsersController.new(@config)
+          controllers['/events'] = Bosh::Director::Api::Controllers::EventsController.new(@config)
           controllers
         end
       end

data/lib/bosh/director/api/stemcell_manager.rb

@@ -10,13 +10,13 @@ module Bosh::Director
         stemcell = Models::Stemcell[:name => name, :version => version]
         if stemcell.nil?
           raise StemcellNotFound,
-                "Stemcell `#{name}/#{version}' doesn't exist"
+                "Stemcell '#{name}/#{version}' doesn't exist"
         end
         stemcell
       end
 
       def find_all_stemcells
-        Models::Stemcell.order_by(:name.asc).map do |stemcell|
+        Models::Stemcell.order_by(Sequel.asc(:name)).map do |stemcell|
           {
             'name' => stemcell.name,
             'operating_system' => stemcell.operating_system,
@@ -32,7 +32,7 @@ module Bosh::Director
 
         if stemcells.nil? || stemcells.empty?
           raise StemcellNotFound,
-            "Stemcell with Operating System `#{os}' doesn't exist"
+            "Stemcell with Operating System '#{os}' doesn't exist"
         end
 
         find_latest(stemcells)
@@ -43,7 +43,7 @@ module Bosh::Director
 
         if stemcells.nil? || stemcells.empty?
           raise StemcellNotFound,
-            "Stemcell `#{name}' doesn't exist"
+            "Stemcell '#{name}' doesn't exist"
         end
 
         find_latest(stemcells)
@@ -54,7 +54,7 @@ module Bosh::Director
             dataset.order(:name)[:operating_system => os, :version => version]
         if stemcell.nil?
           raise StemcellNotFound,
-                "Stemcell version `#{version}' for OS `#{os}' doesn't exist"
+                "Stemcell version '#{version}' for OS '#{os}' doesn't exist"
         end
         stemcell
       end

data/lib/bosh/director/api/task_manager.rb

@@ -24,6 +24,7 @@ module Bosh::Director
           "state" => task.state,
           "description" => task.description,
           "timestamp" => task.timestamp.to_i,
+          "started_at" => task.started_at ? task.started_at.to_i: nil,
           "result" => task.result,
           "user" => task.username || "admin",
           "deployment" => task.deployment_name

data/lib/bosh/director/api/task_remover.rb

@@ -14,7 +14,7 @@ module Bosh::Director::Api
     private
     def removal_candidates_dataset(type)
       Bosh::Director::Models::Task.filter("state NOT IN ('processing', 'queued') and type='#{type}'").
-        order{id.desc}.limit(2, @max_tasks)
+        order{Sequel.desc(:id)}.limit(2, @max_tasks)
     end
   end
 end

data/lib/bosh/director/api/uaa_identity_provider.rb

@@ -52,10 +52,18 @@ module Bosh
           @token = token
         end
 
-        def username
+        def username_or_client
           @token['user_name'] || @token['client_id']
         end
 
+        def client
+          @token['client_id']
+        end
+
+        def username
+          @token['user_name']
+        end
+
         def scopes
           @token['scope']
         end

data/lib/bosh/director/api/user/database_user_manager.rb

@@ -53,7 +53,7 @@ module Bosh::Director
       def find_by_name(name)
         user = Models::User[:username => name]
         if user.nil?
-          raise UserNotFound, "User `#{name}' doesn't exist"
+          raise UserNotFound, "User '#{name}' doesn't exist"
         end
         user
       end

data/lib/bosh/director/app.rb

@@ -9,7 +9,7 @@ module Bosh::Director
   class App
 
     class << self
-      # Some places (ie, resque jobs) need to reference the authoritative app instance
+      # Some places need to reference the authoritative app instance
       # from class methods.
       def instance
         @@instance

data/lib/bosh/director/arp_flusher.rb

@@ -0,0 +1,23 @@
+module Bosh::Director
+  class ArpFlusher
+    def delete_arp_entries(vm_cid_to_exclude, ip_addresses)
+      filtered_instances = filter_instances(vm_cid_to_exclude)
+
+      ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
+        filtered_instances.each do |instance|
+          pool.process do
+            agent = AgentClient.with_vm_credentials_and_agent_id(instance.credentials, instance.agent_id)
+            agent.delete_arp_entries(ips: ip_addresses)
+          end
+        end
+      end
+    end
+
+    def filter_instances(vm_cid_to_exclude)
+      Models::Instance
+        .exclude(vm_cid: nil)
+        .exclude(vm_cid: vm_cid_to_exclude)
+        .exclude(compilation: true).all
+    end
+  end
+end

data/lib/bosh/director/cloudcheck_helper.rb

@@ -51,7 +51,7 @@ module Bosh::Director
         # One situation where this handler is actually useful is when
         # VM has already been deleted but something failed after that
         # and it is still referenced in DB. In that case it makes sense
-        # to ignore "VM not found" errors in `delete_vm' and let the method
+        # to ignore "VM not found" errors in `delete_vm` and let the method
         # proceed creating a new VM. Other errors are not forgiven.
 
         @logger.warn("VM '#{instance_model.vm_cid}' might have already been deleted from the cloud")
@@ -148,7 +148,7 @@ module Bosh::Director
     def agent_timeout_guard(vm_cid, vm_credentials, agent_id, &block)
       yield agent_client(vm_credentials, agent_id)
     rescue Bosh::Director::RpcTimeout
-      handler_error("VM `#{vm_cid}' is not responding")
+      handler_error("VM '#{vm_cid}' is not responding")
     end
 
     def vm_deleter
@@ -157,8 +157,9 @@ module Bosh::Director
 
     def vm_creator
       disk_manager = DiskManager.new(cloud, @logger)
+      arp_flusher = ArpFlusher.new
       job_renderer = JobRenderer.create
-      @vm_creator ||= VmCreator.new(cloud, @logger, vm_deleter, disk_manager, job_renderer)
+      @vm_creator ||= VmCreator.new(cloud, @logger, vm_deleter, disk_manager, job_renderer, arp_flusher)
     end
 
     def validate_spec(spec)

data/lib/bosh/director/compile_task.rb

@@ -114,8 +114,8 @@ module Bosh::Director
         unless dep_task.compiled?
           raise DirectorError,
                 "Cannot generate package dependency spec " +
-                "for `#{@package.name}', " +
-                "`#{dep_task.package.name}' hasn't been compiled yet"
+                "for '#{@package.name}', " +
+                "'#{dep_task.package.name}' hasn't been compiled yet"
         end
 
         compiled_package = dep_task.compiled_package
@@ -133,7 +133,7 @@ module Bosh::Director
 
     # @param [CompileTask] task
     # @return [Models::CompiledPackage]
-    def find_compiled_package(logger, event_log)
+    def find_compiled_package(logger, event_log_stage)
       # if `package` has source associated with it (blobstore_id and sha1)
       #   then we need an exact match in find_compiled_package
 
@@ -153,7 +153,7 @@ module Bosh::Director
       if compiled_package
         logger.info("Found compiled version of package '#{package.desc}' for stemcell '#{stemcell.desc}'")
       else
-        compiled_package = fetch_from_global_cache(logger, event_log)
+        compiled_package = fetch_from_global_cache(logger, event_log_stage)
       end
 
       logger.info("Package '#{package.desc}' needs to be compiled on '#{stemcell.desc}'") if compiled_package.nil?
@@ -184,9 +184,9 @@ module Bosh::Director
       compiled_package
     end
 
-    def fetch_from_global_cache(logger, event_log)
+    def fetch_from_global_cache(logger, event_log_stage)
       if Config.use_compiled_package_cache? && BlobUtil.exists_in_global_cache?(package, cache_key)
-        event_log.track("Downloading '#{package.desc}' from global cache") do
+        event_log_stage.advance_and_track("Downloading '#{package.desc}' from global cache") do
           # has side effect of putting CompiledPackage model in db
           logger.info("Found compiled version of package '#{package.desc}' for stemcell '#{stemcell.desc}' in global cache")
           return BlobUtil.fetch_from_global_cache(package, stemcell.model, cache_key, dependency_key)

data/lib/bosh/director/compile_task_generator.rb

@@ -2,9 +2,9 @@ require 'bosh/director'
 
 module Bosh::Director
   class CompileTaskGenerator
-    def initialize(logger, event_log)
+    def initialize(logger, event_log_stage)
       @logger = logger
-      @event_log = event_log
+      @event_log_stage = event_log_stage
     end
 
     # The compile_tasks hash passed in by the caller will be populated with CompileTasks objects
@@ -13,7 +13,7 @@ module Bosh::Director
       # has no cycles: this is being enforced on release upload.
       # Other than that it's a vanilla Depth-First Search (DFS).
 
-      @logger.info("Checking whether package `#{package.desc}' needs to be compiled for stemcell `#{stemcell.model.desc}'")
+      @logger.info("Checking whether package '#{package.desc}' needs to be compiled for stemcell '#{stemcell.model.desc}'")
       task_key = [package.id, stemcell.id]
       task = compile_tasks[task_key]
 
@@ -30,15 +30,15 @@ module Bosh::Director
 
       task = CompileTask.new(package, stemcell, job, package_dependency_key, package_cache_key)
 
-      compiled_package = task.find_compiled_package(@logger, @event_log)
+      compiled_package = task.find_compiled_package(@logger, @event_log_stage)
       if compiled_package
         task.use_compiled_package(compiled_package)
       end
 
-      @logger.info("Processing package `#{package.desc}' dependencies")
+      @logger.info("Processing package '#{package.desc}' dependencies")
       dependencies = package_dependency_manager.dependencies(package)
       dependencies.each do |dependency|
-        @logger.info("Package `#{package.desc}' depends on package `#{dependency.desc}'")
+        @logger.info("Package '#{package.desc}' depends on package '#{dependency.desc}'")
         dependency_task = generate!(compile_tasks, job, template, dependency, stemcell)
         task.add_dependency(dependency_task)
       end

data/lib/bosh/director/compiled_release_downloader.rb

@@ -17,11 +17,11 @@ module Bosh::Director
       FileUtils.mkpath(path)
 
       compiled_packages = @compiled_packages_group.compiled_packages
-      event_log.begin_stage("copying packages", compiled_packages.count)
+      event_log_stage = event_log.begin_stage("copying packages", compiled_packages.count)
 
       compiled_packages.each do |compiled_package|
         desc = "#{compiled_package.package.name}/#{compiled_package.package.version}"
-        event_log.track(desc) do
+        event_log_stage.advance_and_track(desc) do
           blobstore_id = compiled_package.blobstore_id
           File.open(File.join(path, "#{compiled_package.package.name}.tgz"), 'w') do |f|
             @blobstore_client.get(blobstore_id, f, sha1: compiled_package.sha1)
@@ -32,10 +32,10 @@ module Bosh::Director
       path = File.join(@download_dir, 'jobs')
       FileUtils.mkpath(path)
 
-      event_log.begin_stage("copying jobs", @templates.count)
+      event_log_stage = event_log.begin_stage("copying jobs", @templates.count)
       @templates.each do |template|
         desc = "#{template.name}/#{template.version}"
-        event_log.track(desc) do
+        event_log_stage.advance_and_track(desc) do
           blobstore_id = template.blobstore_id
           File.open(File.join(path, "#{template.name}.tgz"), 'w') do |f|
             @blobstore_client.get(blobstore_id, f, sha1: template.sha1)

data/lib/bosh/director/config.rb

@@ -31,6 +31,7 @@ module Bosh::Director
         :fix_stateful_nodes,
         :enable_snapshots,
         :max_vm_create_tries,
+        :flush_arp,
         :nats_uri,
         :default_ssh_options,
         :keep_unreachable_vms,
@@ -41,8 +42,8 @@ module Bosh::Director
 
       attr_reader(
         :db_config,
-        :redis_logger_level,
-        :ignore_missing_gateway
+        :ignore_missing_gateway,
+        :record_events,
       )
 
       def clear
@@ -67,6 +68,7 @@ module Bosh::Director
 
       def configure(config)
         @max_vm_create_tries = Integer(config.fetch('max_vm_create_tries', 5))
+        @flush_arp = config.fetch('flush_arp', false)
 
         @base_dir = config['dir']
         FileUtils.mkdir_p(@base_dir)
@@ -94,13 +96,6 @@ module Bosh::Director
         @logger.add_appenders(shared_appender)
         @logger.level = Logging.levelify(logging_config.fetch('level', 'debug'))
 
-        # use a separate logger with the same appender to avoid multiple file writers
-        redis_logger = Logging::Logger.new('DirectorRedis')
-        redis_logger.add_appenders(shared_appender)
-        logging_config = config.fetch('redis', {}).fetch('logging', {})
-        @redis_logger_level = Logging.levelify(logging_config.fetch('level', 'info'))
-        redis_logger.level = @redis_logger_level
-
         # Event logger supposed to be overridden per task,
         # the default one does nothing
         @event_log = EventLog::Log.new
@@ -110,13 +105,6 @@ module Bosh::Director
 
         @max_threads = config.fetch('max_threads', 32).to_i
 
-        self.redis_options = {
-          :host     => config['redis']['host'],
-          :port     => config['redis']['port'],
-          :password => config['redis']['password'],
-          :logger   => redis_logger
-        }
-
         @revision = get_revision
 
         @logger.info("Starting BOSH Director: #{VERSION} (#{@revision})")
@@ -169,6 +157,7 @@ module Bosh::Director
         @enable_post_deploy = config.fetch('enable_post_deploy', false)
         @generate_vm_passwords = config.fetch('generate_vm_passwords', false)
         @remove_dev_tools = config['remove_dev_tools']
+        @record_events = config.fetch('record_events', false)
 
         Bosh::Clouds::Config.configure(self)
 
@@ -192,13 +181,13 @@ module Bosh::Director
       end
 
       def configure_db(db_config)
-        patch_sqlite if db_config['adapter'] == 'sqlite'
+        connection_config = db_config.dup
+        connection_options = connection_config.delete('connection_options') {{}}
+        connection_config.delete_if { |_, v| v.to_s.empty? }
+        connection_config = connection_config.merge(connection_options)
 
-        connection_options = db_config.delete('connection_options') {{}}
-        db_config.delete_if { |_, v| v.to_s.empty? }
-        db_config = db_config.merge(connection_options)
-
-        db = Sequel.connect(db_config)
+        Sequel.default_timezone = :utc
+        db = Sequel.connect(connection_config)
 
         Bosh::Common.retryable(sleep: 0.5, tries: 20, on: [Exception]) do
           db.extension :connection_validator
@@ -254,18 +243,6 @@ module Bosh::Director
 
       alias_method :task_checkpoint, :job_cancelled?
 
-      def redis_options
-        @redis_options ||= {}
-      end
-
-      def redis_logger_level
-        @redis_logger_level || Logger::INFO
-      end
-
-      def redis_options=(options)
-        @redis_options = options
-      end
-
       def cloud_options=(options)
         @lock.synchronize do
           @cloud_options = options
@@ -285,22 +262,6 @@ module Bosh::Director
         @nats_rpc
       end
 
-      def redis
-        threaded[:redis] ||= Redis.new(redis_options)
-      end
-
-      def redis_logger=(logger)
-        if redis?
-          redis.client.logger = logger
-        else
-          redis_options[:logger] = logger
-        end
-      end
-
-      def redis?
-        !threaded[:redis].nil?
-      end
-
       def encryption?
         @encryption
       end
@@ -328,38 +289,6 @@ module Bosh::Director
         temp_dir
       end
 
-      def patch_sqlite
-        return if @patched_sqlite
-        @patched_sqlite = true
-
-        require 'sequel'
-        require 'sequel/adapters/sqlite'
-
-        Sequel::SQLite::Database.class_eval do
-          def connect(server)
-            opts = server_opts(server)
-            opts[:database] = ':memory:' if blank_object?(opts[:database])
-            db = ::SQLite3::Database.new(opts[:database])
-            db.busy_handler do |retries|
-              Bosh::Director::Config.logger.debug "SQLITE BUSY, retry ##{retries}"
-              sleep(0.1)
-              retries < 20
-            end
-
-            connection_pragmas.each { |s| log_yield(s) { db.execute_batch(s) } }
-
-            class << db
-              attr_reader :prepared_statements
-            end
-            db.instance_variable_set(:@prepared_statements, {})
-
-            db
-          end
-        end
-      end
-
-      # Migrates director UUID to database
-      # Currently used by integration tests to set director UUID
       def override_uuid
         new_uuid = nil
         state_file = File.join(base_dir, 'state.json')
@@ -440,18 +369,23 @@ module Bosh::Director
       end
     end
 
-    def resque_logger
+    def worker_logger
       logger = Logging::Logger.new('DirectorWorker')
-      resque_logging = hash.fetch('resque', {}).fetch('logging', {})
-      if resque_logging.has_key?('file')
-        logger.add_appenders(Logging.appenders.file('DirectorWorkerFile', filename: resque_logging.fetch('file'), layout: ThreadFormatter.layout))
+      logging_config = hash.fetch('logging', {})
+      worker_logging = hash.fetch('delayed_job', {}).fetch('logging', {})
+      if worker_logging.has_key?('file')
+        logger.add_appenders(Logging.appenders.file('DirectorWorkerFile', filename: worker_logging.fetch('file'), layout: ThreadFormatter.layout))
       else
         logger.add_appenders(Logging.appenders.stdout('DirectorWorkerIO', layout: ThreadFormatter.layout))
       end
-      logger.level = Logging.levelify(resque_logging.fetch('level', 'info'))
+      logger.level = Logging.levelify(logging_config.fetch('level', 'debug'))
       logger
     end
 
+    def db
+      Config.configure_db(hash['db'])
+    end
+
     def blobstore_config
       hash.fetch('blobstore')
     end
@@ -460,6 +394,10 @@ module Bosh::Director
       hash['backup_destination']
     end
 
+    def log_access_events_to_syslog
+      hash['log_access_events_to_syslog']
+    end
+
     def configure_evil_config_singleton!
       Config.configure(hash)
     end
@@ -468,6 +406,10 @@ module Bosh::Director
       Bosh::Director::Api::DirectorUUIDProvider.new(Config)
     end
 
+    def record_events
+      hash.fetch('record_events', false)
+    end
+
     private
 
     attr_reader :hash