bosh-director 1.3215.4.0 → 1.3232.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: faf6283449ed9e2f917191f99c686b67dca177af
-  data.tar.gz: 53ccfd5747076bf4105787ab11568471ddfbf5cc
+  metadata.gz: 3be5b66fb713f0101c8fdd159b0bc3032a717f1d
+  data.tar.gz: 0429e247b484b8ef9bbefb29282112be66ad596c
 SHA512:
-  metadata.gz: 431bd32c3d55f3f581ea2c068ee69d20c86e49de4645bd758a331fd43e7fc15cd207e854569f2e883e2f82f9cf5e06edfe839141afaa8c1c82ab0b40cc380dc9
-  data.tar.gz: 3ccbba5d34e8b4ba95508c91c13cb741ba241a7aeb25c9faaaef998d70a353182c0ae6fff743c6bb20008d7b3593e8ae37b61e4cd19541aa6a3424342f8f2b0b
+  metadata.gz: d6cbcbff8db5ec4b94178b51753fd51c024064d336d92c65c661fa7cdc5ff3c7a8b52bf9d07301fbccc30ba5c017984e48f6b20ccdbd9b846b8e530f8555ea63
+  data.tar.gz: 7f5c5e5f696bb7d1a8bdfab55631dd198ddcd3d8c7db624722aa4428500319a0857b424301413587ecc0a5ae47d3a99418739995386942a8ae5b4c8255fc7456

data/bin/bosh-director-console

@@ -66,7 +66,7 @@ def config
   BD::Config
 end
 
-[:redis, :blobstore, :nats, :cloud].each do |helper|
+[:blobstore, :nats, :cloud].each do |helper|
   self.class.send(:define_method, helper) do
     config.send(helper)
   end

data/bin/bosh-director-drain-workers

@@ -22,16 +22,22 @@ config = Bosh::Director::Config.load_file(config_file)
 
 Bosh::Director::App.new(config)
 
+Delayed::Worker.logger = config.worker_logger
 logger = Bosh::Director::Config.logger
 
+Delayed::Worker.backend = :sequel
+
 loop do
   logger.info('Checking for any workers that are still processing..')
 
-  worker = Resque::Worker.all.find(&:working?)
-  if worker.nil?
+  locked_jobs = Delayed::Job.exclude(:locked_by => nil)
+
+  if locked_jobs.count == 0
     active_worker = 0
     logger.info('All workers are idle')
     break
+  else
+    worker = locked_jobs.first[:locked_by]
   end
 
   break if report

data/bin/bosh-director-worker

@@ -2,74 +2,55 @@
 
 require 'erb'
 require 'bosh/director'
-require 'resque/failure/backtrace'
 
 config_file = nil
 index = nil
 erb = false
 
-opts = OptionParser.new do |op|
-  op.on('-c', '--config [ARG]', 'Configuration File') do |opt|
-    config_file = opt
-  end
+begin
+  opts = OptionParser.new do |op|
+    op.on('-c', '--config [ARG]', 'Configuration File') do |opt|
+      config_file = opt
+    end
 
-  op.on('-i', '--index [ARG]', Integer, 'Worker Index') do |opt|
-    index = opt
-  end
+    op.on('-i', '--index [ARG]', Integer, 'Worker Index') do |opt|
+      index = opt
+    end
 
-  op.on('-e', '--[no-]erb', 'Treat Configuration as ERB Template') do |opt|
-    erb = opt
+    op.on('-e', '--[no-]erb', 'Treat Configuration as ERB Template') do |opt|
+      erb = opt
+    end
   end
-end
 
-opts.parse!(ARGV.dup)
+  opts.parse!(ARGV.dup)
 
-config_file ||= ::File.expand_path('../../config/bosh-director.yml', __FILE__)
-config = nil
+  config_file ||= ::File.expand_path('../../config/bosh-director.yml', __FILE__)
+  config = nil
 
 # Is this code even used anymore?
-if erb
-  class ConfigBindingHelper
-    attr_reader :index
+  if erb
+    class ConfigBindingHelper
+      attr_reader :index
 
-    def initialize(index)
-      @index = index
-    end
+      def initialize(index)
+        @index = index
+      end
 
-    def get_binding
-      binding
+      def get_binding
+        binding
+      end
     end
-  end
-
-  template = ERB.new(File.read(config_file), 0, '%<>-')
-  contents = template.result(ConfigBindingHelper.new(index).get_binding)
-  config_hash = Psych.load(contents)
-  config = Bosh::Director::Config.load_hash(config_hash)
-else
-  config = Bosh::Director::Config.load_file(config_file)
-end
-
-Bosh::Director::App.new(config)
-
-Resque.logger = config.resque_logger
 
-worker = nil
-queues = (ENV['QUEUES'] || ENV['QUEUE']).to_s.split(',')
-
-begin
-  worker = Resque::Worker.new(*queues)
-  worker.term_child = true
-rescue Resque::NoQueueError
-  abort 'set QUEUE env var, e.g. $ QUEUE=critical,high rake resque:work'
-end
+    template = ERB.new(File.read(config_file), 0, '%<>-')
+    contents = template.result(ConfigBindingHelper.new(index).get_binding)
+    config_hash = Psych.load(contents)
+    config = Bosh::Director::Config.load_hash(config_hash)
+  else
+    config = Bosh::Director::Config.load_file(config_file)
+  end
+  config.db
 
-Resque.after_fork do
-  EM.error_handler { |e| puts "Error raised during event loop: #{e.inspect}, backtrace: #{e.backtrace}" }
-  Thread.new { EM.run }
+  worker = Bosh::Director::Worker.new(config, index)
+  worker.prep
+  worker.start
 end
-
-Resque::Failure.backend = Resque::Failure::Backtrace
-
-worker.log "Starting worker #{worker}"
-
-worker.work(ENV['INTERVAL'] || 5) # interval, will block

data/db/migrations/director/20110617211923_add_deployments_release_versions.rb

@@ -13,13 +13,13 @@ Sequel.migration do
       manifest = Psych.load(deployment[:manifest])
 
       unless manifest.is_a?(Hash) && manifest["release"] && manifest["release"]["version"]
-        raise "Invalid manifest for `#{deployment[:name]}', no version data"
+        raise "Invalid manifest for '#{deployment[:name]}', no version data"
       end
 
       release_version = self[:release_versions].filter(:release_id => deployment[:release_id], :version => manifest["release"]["version"].to_s).first
 
       if release_version.nil?
-        raise "Release version #{manifest["release"]["version"]} referenced by `#{deployment[:name]}' manifest not found"
+        raise "Release version #{manifest["release"]["version"]} referenced by '#{deployment[:name]}' manifest not found"
       end
 
       self[:deployments_release_versions].insert(:release_version_id => release_version[:id], :deployment_id => deployment[:id])

data/db/migrations/director/20120524175805_add_task_type.rb

@@ -7,28 +7,28 @@ Sequel.migration do
       add_index :description
     end
 
-    self[:tasks].filter(:description => "create deployment").
-        update(:type => "update_deployment")
-    self[:tasks].filter(:description.like("delete deployment:%")).
-        update(:type => "delete_deployment")
-    self[:tasks].filter(:description => "fetch logs").
-        update(:type => "fetch_logs")
-    self[:tasks].filter(:description.like("ssh:%")).
-        update(:type => "ssh")
-    self[:tasks].filter(:description => "scan cloud").
-        update(:type => "cck_scan")
-    self[:tasks].filter(:description => "apply resolutions").
-        update(:type => "cck_apply")
-    self[:tasks].filter(:description => "retrieve vm-stats").
-        update(:type => "vms")
-    self[:tasks].filter(:description => "create release").
-        update(:type => "update_release")
-    self[:tasks].filter(:description.like("delete release:%")).
-        update(:type => "delete_release")
-    self[:tasks].filter(:description => "create stemcell").
-        update(:type => "update_stemcell")
-    self[:tasks].filter(:description.like("delete stemcell:%")).
-        update(:type => "delete_stemcell")
+    self[:tasks].filter(description: 'create deployment').
+        update(:type => 'update_deployment')
+    self[:tasks].grep(:description, 'delete deployment:%').
+        update(:type => 'delete_deployment')
+    self[:tasks].filter(:description => 'fetch logs').
+        update(:type => 'fetch_logs')
+    self[:tasks].grep(:description, 'ssh:%').
+        update(:type => 'ssh')
+    self[:tasks].filter(:description => 'scan cloud').
+        update(:type => 'cck_scan')
+    self[:tasks].filter(:description => 'apply resolutions').
+        update(:type => 'cck_apply')
+    self[:tasks].filter(:description => 'retrieve vm-stats').
+        update(:type => 'vms')
+    self[:tasks].filter(:description => 'create release').
+        update(:type => 'update_release')
+    self[:tasks].grep(:description, 'delete release:%').
+        update(:type => 'delete_release')
+    self[:tasks].filter(:description => 'create stemcell').
+        update(:type => 'update_stemcell')
+    self[:tasks].grep(:description, 'delete stemcell:%').
+        update(:type => 'delete_stemcell')
 
     alter_table :tasks do
       set_column_allow_null :type, false
@@ -41,4 +41,4 @@ Sequel.migration do
       drop_index :description
     end
   end
-end
+end

data/db/migrations/director/20130531172604_add_director_attributes.rb

@@ -1,8 +1,7 @@
 Sequel.migration do
   up do
     create_table(:director_attributes) do
-      primary_key :uuid
-      String :uuid, unique: true, null: false
+      String :uuid, unique: true, null: false, primary_key: true
     end
   end
 

data/db/migrations/director/20140116002324_pivot_director_attributes.rb

@@ -3,8 +3,7 @@ Sequel.migration do
     rename_table(:director_attributes, :old_director_attributes)
 
     create_table(:director_attributes) do
-      primary_key :name
-      String :name, unique: true, null: false
+      String :name, unique: true, null: false, primary_key: true
       String :value
     end
 

data/db/migrations/director/20160210201838_denormalize_compiled_package_stemcell_id_to_stemcell_name_and_version.rb

@@ -23,25 +23,23 @@ Sequel.migration do
       stemcell_foreign_key_name = stemcell_foreign_keys.first.fetch(:name)
 
       build_indexes = indexes(:compiled_packages).select { |_, value| value.fetch(:columns) == [:package_id, :stemcell_id, :build] }
-      raise 'Failed to run migration, found more than 1 build index' if build_indexes.size != 1
-      # build_indexes is an array, where each element is an array with first element being the name of index
-      build_index = build_indexes.first.first
-
-      stemcell_indexes = indexes(:compiled_packages).select { |_, value| value.fetch(:columns) == [:package_id, :stemcell_id, :dependency_key_sha1]}
-      raise 'Failed to run migration, found more than 1 stemcell_id index' if stemcell_indexes.size != 1
-      # stemcell_indexes is an array, where each element is an array with first element being the name of index
-      stemcell_index = stemcell_indexes.first.first
-
       if [:mysql2, :mysql].include?(adapter_scheme)
+        raise 'Failed to run migration, found more than 1 build index' if build_indexes.size != 1
+        # build_indexes is an array, where each element is an array with first element being the name of index
+        build_index = build_indexes.first.first
         alter_table(:compiled_packages) do
           drop_index(nil, name: build_index)
         end
       elsif [:postgres].include?(adapter_scheme)
+        build_index = build_indexes.empty? ? 'compiled_packages_package_id_stemcell_id_build_key' : build_indexes.first.first
         alter_table(:compiled_packages) do
           drop_constraint(build_index)
         end
       end
 
+      stemcell_indexes = indexes(:compiled_packages).select { |_, value| value.fetch(:columns) == [:package_id, :stemcell_id, :dependency_key_sha1]}
+      stemcell_index = stemcell_indexes.empty? ? 'package_stemcell_dependency_key_sha1_idx' : stemcell_indexes.first.first
+
       alter_table(:compiled_packages) do
         drop_constraint(stemcell_foreign_key_name, :type => :foreign_key)
         add_index [:package_id, :stemcell_os, :stemcell_version, :build], unique: true, name: 'package_stemcell_build_idx'

data/db/migrations/director/20160211174110_add_events.rb

@@ -0,0 +1,22 @@
+Sequel.migration do
+  up do
+    create_table :events do
+      primary_key :id
+      Integer :parent_id
+      String :user, :null => false
+      Time :timestamp, :index => true, :null => false
+      String :action, :null => false
+      String :object_type, :null => false
+      String :object_name
+      String :error, :text => true
+      String :task
+      String :deployment
+      String :instance
+      String :context_json, :text => true
+    end
+  end
+
+  down do
+    drop_table :events
+  end
+end

data/db/migrations/director/20160324181932_create_delayed_jobs.rb

@@ -0,0 +1,22 @@
+Sequel.migration do
+  up do
+    create_table :delayed_jobs, force: true do |table|
+      primary_key :id
+      table.Integer :priority, default: 0, null: false # Allows some jobs to jump to the front of the queue
+      table.Integer :attempts, default: 0, null: false # Provides for retries, but still fail eventually.
+      table.String :handler, :text => true, null: false # YAML-encoded string of the object that will do work
+      table.String :last_error, :text => true           # reason for last failure (See Note below)
+      table.Time :run_at                           # When to run. Could be Time.zone.now for immediately, or sometime in the future.
+      table.Time :locked_at                        # Set when a client is working on this object
+      table.Time :failed_at                        # Set when all retries have failed (actually, by default, the record is deleted instead)
+      table.String :locked_by                          # Who is working on this object (if locked)
+      table.String :queue                              # The name of the queue this job is in
+    end
+
+    add_index :delayed_jobs, [:priority, :run_at], name: "delayed_jobs_priority"
+  end
+
+  down do
+    drop_table :delayed_jobs
+  end
+end

data/db/migrations/director/20160324182211_add_locks.rb

@@ -0,0 +1,15 @@
+Sequel.migration do
+  up do
+    create_table :locks do
+      primary_key :id
+      Time :expired_at, :null => false
+      String :name, :unique => true, :null => false
+      String :uid, :unique => true, :null => false
+      index :name, :unique => true
+    end
+  end
+
+  down do
+    drop_table :locks
+  end
+end

data/db/migrations/director/20160329201256_set_instances_with_nil_serial_to_false.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+Sequel.migration do
+  up do
+    self[:instances].all do |instance|
+      unless instance[:spec_json].nil?
+        spec = JSON.parse(instance[:spec_json])
+        if !spec['update'].nil? && spec['update'].has_key?('serial') && spec['update']['serial'].nil?
+          spec['update']['serial'] = false
+          self[:instances].where(id: instance[:id]).update(spec_json: JSON.generate(spec))
+        end
+      end
+    end
+  end
+end

data/db/migrations/director/20160331225404_backfill_stemcell_os.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  up do
+    self[:stemcells].all do |stemcell|
+      if stemcell[:operating_system].nil? || stemcell[:operating_system] == ''
+        self[:stemcells].where(id: stemcell[:id]).update(operating_system: stemcell[:name])
+      end
+    end
+  end
+end

data/db/migrations/director/20160411104407_add_task_started_at.rb

@@ -0,0 +1,9 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+Sequel.migration do
+  change do
+    alter_table(:tasks) do
+      add_column(:started_at, Time)
+    end
+  end
+end

data/lib/bosh/director.rb

@@ -28,13 +28,14 @@ require 'bcrypt'
 require 'blobstore_client'
 require 'eventmachine'
 require 'netaddr'
-require 'resque'
+require 'delayed_job'
 require 'sequel'
 require 'sinatra/base'
 require 'securerandom'
 require 'yajl'
 require 'nats/client'
 require 'securerandom'
+require 'delayed_job_sequel'
 
 require 'common/thread_formatter'
 require 'bosh/core/encryption_handler'
@@ -65,6 +66,7 @@ require 'bosh/director/package_dependencies_manager'
 require 'bosh/director/job_renderer'
 require 'bosh/director/cycle_helper'
 require 'bosh/director/encryption_helper'
+require 'bosh/director/worker'
 require 'bosh/director/password_helper'
 require 'bosh/director/vm_creator'
 require 'bosh/director/vm_recreator'
@@ -98,10 +100,10 @@ require 'bosh/director/deployment_deleter'
 require 'bosh/director/permission_authorizer'
 require 'bosh/director/transactor'
 require 'bosh/director/sequel'
+require 'bosh/director/arp_flusher'
 require 'common/thread_pool'
 
 require 'bosh/director/manifest/manifest'
-require 'bosh/director/manifest/redactor'
 require 'bosh/director/manifest/changeset'
 require 'bosh/director/manifest/diff_lines'
 
@@ -121,6 +123,7 @@ require 'bosh/director/jobs/base_job'
 require 'bosh/director/jobs/backup'
 require 'bosh/director/jobs/scheduled_backup'
 require 'bosh/director/jobs/scheduled_orphan_cleanup'
+require 'bosh/director/jobs/scheduled_events_cleanup'
 require 'bosh/director/jobs/create_snapshot'
 require 'bosh/director/jobs/snapshot_deployment'
 require 'bosh/director/jobs/snapshot_deployments'
@@ -146,6 +149,7 @@ require 'bosh/director/jobs/release/release_job'
 require 'bosh/director/jobs/ssh'
 require 'bosh/director/jobs/attach_disk'
 require 'bosh/director/jobs/helpers'
+require 'bosh/director/jobs/db_job'
 
 require 'bosh/director/models/helpers/model_helper'
 
@@ -154,6 +158,7 @@ require 'bosh/director/blobstores'
 require 'bosh/director/api/director_uuid_provider'
 require 'bosh/director/api/local_identity_provider'
 require 'bosh/director/api/uaa_identity_provider'
+require 'bosh/director/api/event_manager'
 require 'bosh/director/app'
 
 module Bosh::Director
@@ -162,6 +167,7 @@ end
 
 require 'bosh/director/thread_pool'
 require 'bosh/director/api/extensions/scoping'
+require 'bosh/director/api/extensions/syslog_request_logger'
 require 'bosh/director/api/controllers/backups_controller'
 require 'bosh/director/api/controllers/cleanup_controller'
 require 'bosh/director/api/controllers/deployments_controller'
@@ -179,4 +185,5 @@ require 'bosh/director/api/controllers/cloud_configs_controller'
 require 'bosh/director/api/controllers/runtime_configs_controller'
 require 'bosh/director/api/controllers/locks_controller'
 require 'bosh/director/api/controllers/restore_controller'
+require 'bosh/director/api/controllers/events_controller'
 require 'bosh/director/api/route_configuration'