bosh-director 1.5.0.pre.1113

data/CHANGELOG

@@ -0,0 +1,34 @@
+== 0.7 - 12/07/12
+* Gem bumps:
+ * Upgrade openstack_cpi gem to v0.0.4 and blobstore_client to v0.5.0 in director
+ * Bump vSphere CPI versions in deployer and director.
+ * Bump director vsphere_cpi
+ * Bump vcloud_cpi version
+ * Bump aws_cpi to 0.6.3
+* Rename soap log to cpi log
+* `bosh releases` shows which releases are in use
+* Improved DNS support
+* add director feature list to /info
+* Save properties from job spec on release upload
+* Filter job properties
+* Use common property helpers everywhere
+* Release Rebase improvements
+* Don't update task state when checkpointing
+* Persist VM env for cloudcheck
+* Trigger update when resource pool env changes
+* Fix InstanceDeleter unit test for 1.9.3.
+* Removed Lock.try_lock
+* CPI log exposed to all providers.
+* Reduce bosh vms timeout from 30 seconds to 5.
+* Add LockHelper.
+* Make release lock more granular in UpdateDeployment job.
+* Use new set_vm_metadata call.
+* Fix metadata_helper to use the deployment name.
+* Use migrations from the CPI gems.
+* Injecting vcloud_cpi dependencies into director
+
+== 0.6 - 08/29/12
+* release rebase support
+* fingerprint matching
+* updated VSphere CPI to 0.4.10 (vCPU fix)
+* removed compiled package sharing

data/bin/bosh-director

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'bosh/director'
+require 'thin'
+
+config_file = nil
+
+opts = OptionParser.new do |opts|
+  opts.on('-c', '--config [ARG]', 'Configuration File') do |opt|
+    config_file = opt
+  end
+end
+
+opts.parse!(ARGV.dup)
+
+config_file ||= ::File.expand_path('../../config/bosh-director.yml', __FILE__)
+config = Bosh::Director::Config.load_file(config_file)
+
+Bosh::Director::App.new(config)
+
+thin_server = Thin::Server.new('127.0.0.1', config.hash['port'], :signals => false) do
+  use Rack::CommonLogger
+  map "/" do
+    run Bosh::Director::Api::Controller.new
+  end
+end
+
+%w(TERM INT QUIT).each do |signal|
+  trap(signal) do
+    Bosh::Director::Config.logger.info('Shutting down Director')
+    thin_server.stop!
+  end
+end
+
+Bosh::Director::Config.logger.info("Listening on port #{config.hash['port']}")
+thin_server.start!

data/bin/bosh-director-console

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+APP_DIR = File.expand_path(File.join("..", ".."), __FILE__)
+
+DEFAULT_DIRECTOR_CONFIG = File.join(APP_DIR, "config", "bosh-director.yml")
+
+require 'bosh/director'
+require 'irb'
+require 'irb/completion'
+require 'rack/test'
+
+module Bosh
+  module Director
+
+    class TestApplication
+      include Rack::Test::Methods
+
+      def app
+        Bosh::Director::Api::Controller.new
+      end
+    end
+
+    class Console
+      def self.start(context)
+        new.start(context)
+      end
+
+      def start(context)
+        options = {}
+
+        OptionParser.new do |opt|
+          opt.banner = "Usage: director_console [options]"
+          opt.on('-c', '--config [ARG]', 'configuration file') { |v| options[:config] = v }
+          opt.parse!(ARGV)
+        end
+
+        if options[:config]
+          config_file = File.expand_path(options[:config])
+        else
+          config_file = DEFAULT_DIRECTOR_CONFIG
+        end
+
+        puts "=> Loading #{config_file}"
+        Bosh::Director::App.new(Bosh::Director::Config.load_file(config_file))
+
+        context.instance_eval do
+          include BD::Models
+        end
+
+        begin
+          require 'ruby-debug'
+          puts "=> Debugger enabled"
+        rescue LoadError
+          puts "=> ruby-debug not found, debugger disabled"
+        end
+
+        puts "=> Welcome to BOSH Director console\n=> You can use 'app' to access REST API"
+        puts "=> You can also use 'cloud', 'blobstore', 'nats' helpers to query these services"
+
+        IRB.start
+      end
+
+    end
+  end
+end
+
+BD = Bosh::Director
+
+def app
+  BD::TestApplication.new
+end
+
+def config
+  BD::Config
+end
+
+[:redis, :blobstore, :nats, :cloud].each do |helper|
+  self.class.send(:define_method, helper) do
+    config.send(helper)
+  end
+end
+
+BD::Console.start(self)
+

data/bin/bosh-director-drain-workers

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+require "bosh/director"
+
+config_file = nil
+report = false
+active_worker = 1
+
+opts = OptionParser.new do |opts|
+  opts.on("-c", "--config [ARG]", "Configuration File") do |opt|
+    config_file = opt
+  end
+  opts.on("-r", "--report", "Report 1 if any active worker, 0 otherwise") do |opt|
+    report = true
+  end
+end
+
+opts.parse!(ARGV.dup)
+
+config_file ||= ::File.expand_path("../../config/bosh-director.yml", __FILE__)
+config = Bosh::Director::Config.load_file(config_file)
+
+Bosh::Director::App.new(config)
+
+logger = Bosh::Director::Config.logger
+
+loop do
+  logger.info("Checking for any workers that are still processing..")
+  worker = Resque::Worker.all.find { |worker| worker.working? }
+  if worker.nil?
+    active_worker = 0
+    logger.info("All workers are idle")
+    break
+  end
+  if report
+    break
+  end
+  logger.info("Found worker: #{worker}, sleeping for 60 seconds.")
+  sleep 60
+end
+
+puts "#{active_worker}" if report

data/bin/bosh-director-migrate

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+require "logger"
+require "sequel"
+require "bosh/director"
+require "optparse"
+
+config_file = nil
+
+opts = OptionParser.new do |opts|
+  opts.on("-c", "--config FILE", "configuration file") do |opt|
+    config_file = opt
+  end
+end
+
+opts.parse!(ARGV.dup)
+
+if config_file.nil?
+  puts opts
+  exit 1
+end
+
+config = Psych.load_file(config_file)
+unless config["db"] && config["db"]["database"]
+  abort ("Director database config missing from config file")
+end
+
+db = Bosh::Director::Config.configure_db(config["db"])
+
+def migrate(database, schema_table, dir, target = nil)
+  options = {}
+  options[:table] = schema_table if schema_table
+  options[:target] = target.to_i if target
+
+  Sequel.extension :migration
+  Sequel::TimestampMigrator.new(database, dir, options).run
+end
+
+migrations_dir = File.expand_path("../../db/migrations/director", __FILE__)
+
+migrate(db, nil, migrations_dir)
+
+if config["dns"] && config["dns"]["db"]
+  dns_migrations_dir = File.expand_path("../../db/migrations/dns", __FILE__)
+  dns_db = Bosh::Director::Config.configure_db(config["dns"]["db"])
+  migrate(dns_db, "dns_schema",
+          dns_migrations_dir)
+end
+
+cpi = config["cloud"]["plugin"]
+require_path = File.join("cloud", cpi)
+cpi_path = $LOAD_PATH.find { |p| File.exist?(File.join(p, require_path)) }
+migrations = File.expand_path("../db/migrations", cpi_path)
+
+if File.directory?(migrations)
+  migrate(db, "#{cpi}_cpi_schema", migrations)
+end
+

data/bin/bosh-director-scheduler

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'bosh/director'
+require 'bosh/director/scheduler'
+
+config_file = ::File.expand_path('../../config/bosh-director.yml', __FILE__)
+
+opts = OptionParser.new do |opts|
+  opts.on('-c', '--config [ARG]', 'Configuration File') do |opt|
+    config_file = opt
+  end
+end
+
+opts.parse!(ARGV.dup)
+
+config = Bosh::Director::Config.load_file(config_file)
+Bosh::Director::App.new(config)
+
+scheduler = Bosh::Director::Scheduler.new(config.hash.fetch('scheduled_jobs', []))
+
+%w(TERM INT QUIT).each do |signal|
+  trap(signal) do
+    scheduler.stop!
+  end
+end
+
+scheduler.start!

data/bin/bosh-director-worker

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+
+require "erb"
+require "bosh/director"
+
+config_file = nil
+index = nil
+erb = false
+
+opts = OptionParser.new do |opts|
+  opts.on("-c", "--config [ARG]", "Configuration File") do |opt|
+    config_file = opt
+  end
+
+  opts.on("-i", "--index [ARG]", Integer, "Worker Index") do |opt|
+    index = opt
+  end
+
+  opts.on("-e", "--[no-]erb", "Treat Configuration as ERB Template") do |opt|
+    erb = opt
+  end
+end
+
+opts.parse!(ARGV.dup)
+
+config_file ||= ::File.expand_path("../../config/bosh-director.yml", __FILE__)
+config = nil
+
+# Is this code even used anymore?
+if erb
+  class ConfigBindingHelper
+    attr_reader :index
+
+    def initialize(index)
+      @index = index
+    end
+
+    def get_binding
+      binding
+    end
+  end
+
+  template = ERB.new(File.read(config_file), 0, "%<>-")
+  contents = template.result(ConfigBindingHelper.new(index).get_binding)
+  config_hash = Psych.load(contents)
+  config = Bosh::Director::Config.load_hash(config_hash)
+else
+  config = Bosh::Director::Config.load_file(config_file)
+end
+
+Bosh::Director::App.new(config)
+
+resque_logging = config.hash.fetch('resque', {}).fetch('logging', {})
+resque_log_device = Logger::LogDevice.new(resque_logging.fetch('file', STDOUT))
+resque_logger_level = resque_logging.fetch('level', 'info').upcase
+Resque.logger = Logger.new(resque_log_device)
+Resque.logger.level = Logger.const_get(resque_logger_level)
+
+worker = nil
+queues = (ENV['QUEUES'] || ENV['QUEUE']).to_s.split(',')
+
+begin
+  worker = Resque::Worker.new(*queues)
+  worker.term_child = true
+rescue Resque::NoQueueError
+  abort "set QUEUE env var, e.g. $ QUEUE=critical,high rake resque:work"
+end
+
+Resque.after_fork do
+  EM.error_handler{ |e| puts "Error raised during event loop: #{e.message}" }
+  Thread.new { EM.run }
+end
+
+worker.log "Starting worker #{worker}"
+
+worker.work(ENV['INTERVAL'] || 5) # interval, will block

data/db/migrations/README

@@ -0,0 +1 @@
+Generate migrations using rake new_migration[<type>,<name>]

data/db/migrations/director/20110209010747_initial.rb

@@ -0,0 +1,118 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+Sequel.migration do
+  change do
+    create_table :releases do
+      primary_key :id
+      String :name, :null => false, :unique => true
+    end
+
+    create_table :release_versions do
+      primary_key :id
+      String :version, :null => false
+      foreign_key :release_id, :releases, :null => false
+    end
+
+    create_table :packages do
+      primary_key :id
+      String :name, :null => false
+      String :version, :null => false
+      String :blobstore_id, :null => false
+      String :sha1, :null => false
+      String :dependency_set_json, :null => false, :text => true
+      foreign_key :release_id, :releases, :null => false
+      unique [:release_id, :name, :version]
+    end
+
+    create_table :templates do
+      primary_key :id
+      String :name, :null => false
+      String :version, :null => false
+      String :blobstore_id, :null => false
+      String :sha1, :null => false
+      String :package_names_json, :null => false, :text => true
+      foreign_key :release_id, :releases, :null => false
+      unique [:release_id, :name, :version]
+    end
+
+    create_table :stemcells do
+      primary_key :id
+      String :name, :null => false
+      String :version, :null => false
+      String :cid, :null => false
+      unique [:name, :version]
+    end
+
+    create_table :compiled_packages do
+      primary_key :id
+      String :blobstore_id, :null => false
+      String :sha1, :null => false
+      String :dependency_key, :null => false
+      Integer :build, :unsigned => true, :null => false
+      foreign_key :package_id, :packages, :null => false
+      foreign_key :stemcell_id, :stemcells, :null => false
+      unique [:package_id, :stemcell_id, :dependency_key]
+      unique [:package_id, :stemcell_id, :build]
+    end
+
+    create_table :deployments do
+      primary_key :id
+      String :name, :null => false, :unique => true
+      String :manifest, :null => true, :text => true
+      foreign_key :release_id, :releases, :null => true
+    end
+
+    create_table :vms do
+      primary_key :id
+      String :agent_id, :null => false, :unique => true
+      String :cid, :null => false
+      foreign_key :deployment_id, :deployments, :null => false
+    end
+
+    create_table :instances do
+      primary_key :id
+      String :job, :null => false
+      Integer :index, :unsigned => true, :null => false
+      String :disk_cid, :unique => true, :null => true
+      foreign_key :deployment_id, :deployments, :null => false
+      foreign_key :vm_id, :vms, :unique => true, :null => true
+    end
+
+    create_table :tasks do
+      primary_key :id
+      String :state, :index => true, :null => false
+      Time :timestamp, :index => true, :null => false
+      String :description, :null => false
+      String :result, :text => true, :null => true
+      String :output, :null => true
+    end
+
+    create_table :users do
+      primary_key :id
+      String :username, :unique => true, :null => false
+      String :password, :null => false
+    end
+
+    create_table :packages_release_versions do
+      primary_key :id
+      foreign_key :package_id, :packages, :null => false
+      foreign_key :release_version_id, :release_versions, :null => false
+      unique [:package_id, :release_version_id]
+    end
+
+    create_table :release_versions_templates do
+      primary_key :id
+      foreign_key :release_version_id, :release_versions, :null => false
+      foreign_key :template_id, :templates, :null => false
+      unique [:release_version_id, :template_id]
+    end
+
+    create_table :deployments_stemcells do
+      primary_key :id
+      foreign_key :deployment_id, :deployments, :null => false
+      foreign_key :stemcell_id, :stemcells, :null => false
+      unique [:deployment_id, :stemcell_id]
+    end
+
+  end
+end