bosh-director 1.3232.24.0 → 1.3262.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 16c44a4ba321cd6f42757087c868401728f78195
-  data.tar.gz: 39c92fd8253fa376bee4ac3dead05a58d93dde2c
+  metadata.gz: 576adc0d9f6febe9f333d2f9323ae9d8c7a31f87
+  data.tar.gz: 01311d5a771cd111a100a5866d7cd38007dde982
 SHA512:
-  metadata.gz: 4104f9bfd037b661b69a8a379368978fa023b7a1356bfbe6dac4aac82d87e335801dca5eda2d45a632ab9b1c6e688bf89c7b7bb2752756c7f87b145fb5ac80a2
-  data.tar.gz: 136679263e45817087f19658bc5093583c726b4163ea881b0c8cafd8b209fc12c91a4e0a7a12ceaaaa4e3fcd60dad5de9fbd0445b0119877da95bbdb1829b0b8
+  metadata.gz: addb243b3fe331ae2783f7799af05991645e5529b205bf968756f2bc4dd1e7d41d797c0130657706d8f98b86366d344f874156d0fdf25e38e88fb3d2f65f93b9
+  data.tar.gz: 0f4ca4cd3d286d8a904202380647faa9e5f4e62de7b1abab550a615a69311a14ae38ca2eaa957874b794357d6b862c080140471300d55d461078a219b15ce5d3

data/db/migrations/director/20160414183654_set_teams_on_task.rb

@@ -0,0 +1,7 @@
+Sequel.migration do
+  change do
+    alter_table(:tasks) do
+      add_column(:teams, String)
+    end
+  end
+end

data/db/migrations/director/20160427164345_add_teams.rb

@@ -0,0 +1,48 @@
+Sequel.migration do
+  up do
+    create_table :teams do
+      primary_key :id
+      String :name, :unique => true, :null => false
+    end
+
+    create_table :deployments_teams do
+      foreign_key :deployment_id, :deployments, :null => false, :on_delete => :cascade
+      foreign_key :team_id, :teams, :null => false, :on_delete => :cascade
+      unique [:deployment_id, :team_id]
+    end
+
+    deployments_with_teams = self[:deployments].reject { |d| d[:teams].nil? }
+    team_names = deployments_with_teams
+                   .map { |d| d[:teams].split(',') }
+                   .flatten
+                   .uniq
+                   .map { |team| [team] }
+
+    self[:teams].import([:name], team_names)
+
+    deployments_with_teams.each do |d|
+      deployment_teams = d[:teams]
+                           .split(',')
+                           .map do |team_name|
+        team = self[:teams].filter({name: team_name}).first
+        [d[:id], team[:id]]
+      end
+
+      self[:deployments_teams].import([:deployment_id, :team_id], deployment_teams)
+    end
+
+    alter_table(:deployments) do
+      drop_column :teams
+    end
+
+    create_table :tasks_teams do
+      foreign_key :task_id, :tasks, :null => false, :on_delete => :cascade
+      foreign_key :team_id, :teams, :null => false, :on_delete => :cascade
+      unique [:task_id, :team_id]
+    end
+
+    alter_table(:tasks) do
+      drop_column :teams
+    end
+  end
+end

data/db/migrations/director/20160511191928_ephemeral_blobs.rb

@@ -0,0 +1,10 @@
+Sequel.migration do
+  change do
+    create_table :ephemeral_blobs do
+      primary_key :id
+      String :blobstore_id, :null => false
+      String :sha1, :null => false
+      Time :created_at, null: false
+    end
+  end
+end

data/db/migrations/director/20160513102035_add_tracking_to_instance.rb

@@ -0,0 +1,7 @@
+Sequel.migration do
+  change do
+    alter_table(:instances) do
+      add_column(:ignore, TrueClass, :default => false)
+    end
+  end
+end

data/db/migrations/director/20160531164756_add_local_dns_blobs.rb

@@ -0,0 +1,14 @@
+Sequel.migration do
+  up do
+    create_table :local_dns_blobs do
+      primary_key :id
+      String :blobstore_id, :null => false
+      String :sha1, :null => false
+      Time :created_at, null: false
+    end
+  end
+
+  down do
+    drop_table :local_dns_blobs
+  end
+end

data/lib/bosh/director.rb

@@ -32,7 +32,6 @@ require 'delayed_job'
 require 'sequel'
 require 'sinatra/base'
 require 'securerandom'
-require 'yajl'
 require 'nats/client'
 require 'securerandom'
 require 'delayed_job_sequel'
@@ -40,16 +39,19 @@ require 'delayed_job_sequel'
 require 'common/thread_formatter'
 require 'bosh/core/encryption_handler'
 require 'bosh/director/api'
+require 'bosh/director/dns/blobstore_dns_publisher'
 require 'bosh/director/dns/canonicalizer'
 require 'bosh/director/dns/dns_manager'
 require 'bosh/director/dns/local_dns_repo'
 require 'bosh/director/errors'
 require 'bosh/director/ext'
 require 'bosh/director/ip_util'
+require 'bosh/director/cidr_range_combiner'
 require 'bosh/director/lock_helper'
 require 'bosh/director/validation_helper'
 require 'bosh/director/download_helper'
 require 'bosh/director/tagged_logger'
+require 'bosh/director/legacy_deployment_helper'
 
 require 'bosh/director/version'
 require 'bosh/director/config'
@@ -148,6 +150,7 @@ require 'bosh/director/jobs/cloud_check/apply_resolutions'
 require 'bosh/director/jobs/release/release_job'
 require 'bosh/director/jobs/ssh'
 require 'bosh/director/jobs/attach_disk'
+require 'bosh/director/jobs/delete_vm'
 require 'bosh/director/jobs/helpers'
 require 'bosh/director/jobs/db_job'
 
@@ -186,4 +189,5 @@ require 'bosh/director/api/controllers/runtime_configs_controller'
 require 'bosh/director/api/controllers/locks_controller'
 require 'bosh/director/api/controllers/restore_controller'
 require 'bosh/director/api/controllers/events_controller'
+require 'bosh/director/api/controllers/vms_controller'
 require 'bosh/director/api/route_configuration'

data/lib/bosh/director/api.rb

@@ -27,6 +27,7 @@ require 'bosh/director/api/resurrector_manager'
 require 'bosh/director/api/restore_manager'
 require 'bosh/director/api/cloud_config_manager'
 require 'bosh/director/api/runtime_config_manager'
+require 'bosh/director/api/instance_ignore_manager'
 
 require 'bosh/director/api/instance_lookup'
 require 'bosh/director/api/deployment_lookup'

data/lib/bosh/director/api/api_helper.rb

@@ -43,11 +43,11 @@ module Bosh::Director
       end
 
       def json_encode(payload)
-        Yajl::Encoder.encode(payload)
+        JSON.generate(payload)
       end
 
       def json_decode(payload)
-        Yajl::Parser.parse(payload)
+        JSON.parse(payload)
       end
 
       def start_task

data/lib/bosh/director/api/controllers/cleanup_controller.rb

@@ -5,7 +5,7 @@ module Bosh::Director
     class CleanupController < BaseController
       post '/', :consumes => :json do
         job_queue = JobQueue.new
-        payload = json_decode(request.body)
+        payload = json_decode(request.body.read)
         task = Bosh::Director::Jobs::CleanupArtifacts.enqueue(current_user, payload['config'], job_queue)
 
         redirect "/tasks/#{task.id}"

data/lib/bosh/director/api/controllers/deployments_controller.rb

@@ -38,6 +38,7 @@ module Bosh::Director
 
     class DeploymentsController < BaseController
       register DeploymentsSecurity
+      include LegacyDeploymentHelper
 
       def initialize(config)
         super(config)
@@ -46,6 +47,7 @@ module Bosh::Director
         @property_manager = Api::PropertyManager.new
         @instance_manager = Api::InstanceManager.new
         @deployments_repo = DeploymentPlan::DeploymentRepo.new
+        @instance_ignore_manager = Api::InstanceIgnoreManager.new
       end
 
       get '/:deployment/jobs/:job/:index_or_id' do
@@ -73,7 +75,10 @@ module Bosh::Director
             }
           }
         }
+
         options['skip_drain'] = params[:job] if params['skip_drain'] == 'true'
+        options['canaries'] = params[:canaries] if !!params['canaries']
+        options['max_in_flight'] = params[:max_in_flight] if !!params['max_in_flight']
 
         if (request.content_length.nil?  || request.content_length.to_i == 0) && (params['state'])
           manifest_file_path = prepare_yml_file(StringIO.new(deployment.manifest), 'deployment', true)
@@ -83,7 +88,7 @@ module Bosh::Director
 
         latest_cloud_config = Bosh::Director::Api::CloudConfigManager.new.latest
         latest_runtime_config = Bosh::Director::Api::RuntimeConfigManager.new.latest
-        task = @deployment_manager.create_deployment(current_user, manifest_file_path, latest_cloud_config, latest_runtime_config, deployment.name, options)
+        task = @deployment_manager.create_deployment(current_user, manifest_file_path, latest_cloud_config, latest_runtime_config, deployment, options)
         redirect "/tasks/#{task.id}"
       end
 
@@ -113,7 +118,7 @@ module Bosh::Director
 
         latest_cloud_config = Bosh::Director::Api::CloudConfigManager.new.latest
         latest_runtime_config = Bosh::Director::Api::RuntimeConfigManager.new.latest
-        task = @deployment_manager.create_deployment(current_user, manifest_file_path, latest_cloud_config, latest_runtime_config, deployment.name, options)
+        task = @deployment_manager.create_deployment(current_user, manifest_file_path, latest_cloud_config, latest_runtime_config, deployment, options)
         redirect "/tasks/#{task.id}"
       end
 
@@ -149,10 +154,15 @@ module Bosh::Director
 
       put '/:deployment/jobs/:job/:index_or_id/resurrection', consumes: :json do
 
-        payload = json_decode(request.body)
+        payload = json_decode(request.body.read)
         @resurrector_manager.set_pause_for_instance(deployment, params[:job], params[:index_or_id], payload['resurrection_paused'])
       end
 
+      put '/:deployment/instance_groups/:instancegroup/:id/ignore', consumes: :json do
+        payload = json_decode(request.body.read)
+        @instance_ignore_manager.set_ignore_state_for_instance(deployment, params[:instancegroup], params[:id], payload['ignore'])
+      end
+
       post '/:deployment/jobs/:job/:index_or_id/snapshots' do
         if params[:index_or_id].to_s =~ /^\d+$/
           instance = @instance_manager.find_by_name(deployment, params[:job], params[:index_or_id])
@@ -213,7 +223,7 @@ module Bosh::Director
       end
 
       get '/:deployment', authorization: :read do
-        Yajl::Encoder.encode({'manifest' => deployment.manifest})
+        JSON.generate({'manifest' => deployment.manifest})
       end
 
       get '/:deployment/vms', authorization: :read do
@@ -223,7 +233,7 @@ module Bosh::Director
           redirect "/tasks/#{task.id}"
         else
           instances = @deployment_manager.deployment_instances_with_vms(deployment)
-          Yajl::Encoder.encode(create_instances_response(instances))
+          JSON.generate(create_instances_response(instances))
         end
       end
 
@@ -234,7 +244,7 @@ module Bosh::Director
           redirect "/tasks/#{task.id}"
         else
           instances = @instance_manager.find_instances_by_deployment(deployment)
-          Yajl::Encoder.encode(create_instances_response(instances))
+          JSON.generate(create_instances_response(instances))
         end
       end
 
@@ -247,7 +257,7 @@ module Bosh::Director
       end
 
       post '/:deployment/ssh', :consumes => [:json] do
-        payload = json_decode(request.body)
+        payload = json_decode(request.body.read)
         task = @instance_manager.ssh(current_user, deployment, payload)
         redirect "/tasks/#{task.id}"
       end
@@ -266,13 +276,13 @@ module Bosh::Director
       end
 
       post '/:deployment/properties', :consumes => [:json] do
-        payload = json_decode(request.body)
+        payload = json_decode(request.body.read)
         @property_manager.create_property(deployment, payload['name'], payload['value'])
         status(204)
       end
 
       put '/:deployment/properties/:property', :consumes => [:json] do
-        payload = json_decode(request.body)
+        payload = json_decode(request.body.read)
         @property_manager.update_property(deployment, params[:property], payload['value'])
         status(204)
       end
@@ -305,12 +315,12 @@ module Bosh::Director
       end
 
       put '/:deployment/problems', :consumes => [:json] do
-        payload = json_decode(request.body)
+        payload = json_decode(request.body.read)
         start_task { @problem_manager.apply_resolutions(current_user, deployment, payload['resolutions']) }
       end
 
       put '/:deployment/scan_and_fix', :consumes => :json do
-        jobs_json = json_decode(request.body)['jobs']
+        jobs_json = json_decode(request.body.read)['jobs']
         payload = convert_job_instance_hash(jobs_json)
         if deployment_has_instance_to_resurrect?(deployment)
           start_task { @problem_manager.scan_and_fix(current_user, deployment, payload) }
@@ -340,33 +350,36 @@ module Bosh::Director
         if deployment
           deployment_name = deployment['name']
           if deployment_name
-            @deployments_repo.find_or_create_by_name(deployment_name, options)
+            options['new'] = Models::Deployment[name: deployment_name].nil? ? true : false
+            deployment_model = @deployments_repo.find_or_create_by_name(deployment_name, options)
           end
         end
 
-        task = @deployment_manager.create_deployment(current_user, manifest_file_path, cloud_config, runtime_config, deployment_name, options)
+        task = @deployment_manager.create_deployment(current_user, manifest_file_path, cloud_config, runtime_config, deployment_model, options)
 
         redirect "/tasks/#{task.id}"
       end
 
       post '/:deployment/diff', authorization: :diff, :consumes => :yaml do
+
         manifest_text = request.body.read
         validate_manifest_yml(manifest_text)
 
+        manifest_hash = Psych.load(manifest_text)
+
+        ignore_cc = ignore_cloud_config?(manifest_hash)
+
         if deployment
-          before_manifest = Manifest.load_from_text(deployment.manifest, deployment.cloud_config, deployment.runtime_config)
+          before_manifest = Manifest.load_from_text(deployment.manifest, ignore_cc ? nil : deployment.cloud_config, deployment.runtime_config)
           before_manifest.resolve_aliases
         else
           before_manifest = Manifest.load_from_text(nil, nil, nil)
         end
 
-        after_cloud_config = Bosh::Director::Api::CloudConfigManager.new.latest
+        after_cloud_config = ignore_cc ? nil : Bosh::Director::Api::CloudConfigManager.new.latest
         after_runtime_config = Bosh::Director::Api::RuntimeConfigManager.new.latest
-        after_manifest = Manifest.load_from_text(
-          manifest_text,
-          after_cloud_config,
-          after_runtime_config
-        )
+
+        after_manifest = Manifest.load_from_hash(manifest_hash, after_cloud_config, after_runtime_config)
         after_manifest.resolve_aliases
 
         redact =  params['redact'] != 'false'
@@ -391,14 +404,14 @@ module Bosh::Director
 
       post '/:deployment/errands/:errand_name/runs' do
         errand_name = params[:errand_name]
-        keep_alive = json_decode(request.body)['keep-alive'] || FALSE
+        keep_alive = json_decode(request.body.read)['keep-alive'] || FALSE
 
         task = JobQueue.new.enqueue(
           current_user,
           Jobs::RunErrand,
           "run errand #{errand_name} from deployment #{deployment.name}",
           [deployment.name, errand_name, keep_alive],
-          deployment.name
+          deployment
         )
 
         redirect "/tasks/#{task.id}"
@@ -407,7 +420,7 @@ module Bosh::Director
       get '/:deployment/errands', authorization: :read do
         deployment_plan = load_deployment_plan
 
-        errands = deployment_plan.jobs.select(&:is_errand?)
+        errands = deployment_plan.instance_groups.select(&:is_errand?)
 
         errand_data = errands.map do |errand|
           {"name" => errand.name}
@@ -433,8 +446,9 @@ module Bosh::Director
       end
 
       def deployment_has_instance_to_resurrect?(deployment)
-        false if deployment.nil?
-        instances = @instance_manager.filter_by(deployment, resurrection_paused: false)
+        return false if deployment.nil?
+        return false if @resurrector_manager.pause_for_all?
+        instances = @instance_manager.filter_by(deployment, resurrection_paused: false, ignore: false)
         instances.any?
       end
 

data/lib/bosh/director/api/controllers/disks_controller.rb

@@ -11,8 +11,9 @@ module Bosh::Director
 
       # PUT /disks/disk_cid/attachments?deployment=foo&job=dea&instance_id=17f01a35-bf9c-4949-bcf2-c07a95e4df33
       put '/:disk_cid/attachments' do
+        deployment = Api::DeploymentManager.new.find_by_name(params[:deployment])
         job_queue = JobQueue.new
-        task = Bosh::Director::Jobs::AttachDisk.enqueue(current_user, params[:deployment], params[:job], params[:instance_id], params[:disk_cid], job_queue)
+        task = Bosh::Director::Jobs::AttachDisk.enqueue(current_user, deployment, params[:job], params[:instance_id], params[:disk_cid], job_queue)
 
         redirect "/tasks/#{task.id}"
       end

data/lib/bosh/director/api/controllers/events_controller.rb

@@ -1,4 +1,5 @@
 require 'bosh/director/api/controllers/base_controller'
+require 'time'
 
 module Bosh::Director
   module Api::Controllers
@@ -15,6 +16,28 @@ module Bosh::Director
           events = events.filter("id < ?", before_id)
         end
 
+        if params['before_time']
+          begin
+            before_datetime = timestamp_filter_value(params['before_time'])
+          rescue ArgumentError
+            status(400)
+            body("Invalid before parameter: '#{params['before_time']}' ")
+            return
+          end
+          events = events.filter("timestamp < ?", before_datetime)
+        end
+
+        if params['after_time']
+          begin
+            after_datetime = timestamp_filter_value(params['after_time']) + 1
+          rescue ArgumentError
+            status(400)
+            body("Invalid after parameter: '#{params['after_time']}' ")
+            return
+          end
+          events = events.filter("timestamp >= ?", after_datetime)
+        end
+
         if params['task']
           events = events.where(task: params['task'])
         end
@@ -32,6 +55,17 @@ module Bosh::Director
         end
         json_encode(events)
       end
+
+      private
+
+      def timestamp_filter_value(value)
+        return Time.at(value.to_i).utc if integer?(value)
+        Time.parse(value)
+      end
+
+      def integer?(string)
+        string =~ /\A[-+]?\d+\z/
+      end
     end
   end
 end