bosh-director 1.3202.0 → 1.3213.0

data/lib/bosh/director/jobs/update_stemcell.rb

@@ -61,7 +61,7 @@ module Bosh::Director
           stemcell_manifest = Psych.load_file(stemcell_manifest_file)
 
           @name = safe_property(stemcell_manifest, "name", :class => String)
-          @operating_system = safe_property(stemcell_manifest, "operating_system", :class => String, :optional => true)
+          @operating_system = safe_property(stemcell_manifest, "operating_system", :class => String, :optional => true, :default => @name)
           @version = safe_property(stemcell_manifest, "version", :class => String)
           @cloud_properties = safe_property(stemcell_manifest, "cloud_properties", :class => Hash, :optional => true)
           @sha1 = safe_property(stemcell_manifest, "sha1", :class => String)

data/lib/bosh/director/jobs/vm_state.rb

@@ -9,13 +9,15 @@ module Bosh::Director
         :vms
       end
 
-      def initialize(deployment_id, format)
+      def initialize(deployment_id, format, state_for_missing_vms = false)
         @deployment_id = deployment_id
         @format = format
+        @state_for_missing_vms = state_for_missing_vms
       end
 
       def perform
-        instances = Models::Instance.filter(:deployment_id => @deployment_id).exclude(vm_cid: nil)
+        instances = Models::Instance.filter(:deployment_id => @deployment_id)
+        instances = instances.exclude(vm_cid: nil) unless @state_for_missing_vms
         ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
           instances.each do |instance|
             pool.process do
@@ -30,27 +32,9 @@ module Bosh::Director
       end
 
       def process_instance(instance)
-        ips = []
         dns_records = []
-        job_state = nil
-        job_vitals = nil
-        processes = []
 
-        begin
-          agent = AgentClient.with_vm_credentials_and_agent_id(instance.credentials, instance.agent_id, :timeout => TIMEOUT)
-          agent_state = agent.get_state(@format)
-          agent_state['networks'].each_value do |network|
-            ips << network['ip']
-          end
-
-          job_state = agent_state['job_state']
-          if agent_state['vitals']
-            job_vitals = agent_state['vitals']
-          end
-          processes = agent_state['processes'] if agent_state['processes']
-        rescue Bosh::Director::RpcTimeout
-          job_state = 'unresponsive agent'
-        end
+        job_state, job_vitals, processes, ips = vm_details(instance)
 
         if dns_manager.dns_enabled?
           dns_records = dns_manager.find_dns_record_names_by_instance(instance)
@@ -81,6 +65,35 @@ module Bosh::Director
 
       private
 
+      def vm_details(instance)
+        ips = []
+        processes = []
+        job_vitals = nil
+        job_state = nil
+
+        if instance.vm_cid
+          begin
+            agent = AgentClient.with_vm_credentials_and_agent_id(instance.credentials, instance.agent_id, :timeout => TIMEOUT)
+            agent_state = agent.get_state(@format)
+            agent_state['networks'].each_value do |network|
+              ips << network['ip']
+            end
+
+            job_state = agent_state['job_state']
+            if agent_state['vitals']
+              job_vitals = agent_state['vitals']
+            end
+            processes = agent_state['processes'] if agent_state['processes']
+          rescue Bosh::Director::RpcTimeout
+            job_state = 'unresponsive agent'
+          end
+        else
+          job_state = 'missing vm'
+        end
+
+        return job_state, job_vitals, processes, ips
+      end
+
       def get_index(agent_state)
         index = agent_state['index']
 

data/lib/bosh/director/key_generator.rb

@@ -0,0 +1,54 @@
+module Bosh
+  module Director
+    class KeyGenerator
+      def dependency_key_from_models(package, release_version)
+        @all_packages = release_version.packages.map do |p|
+          {
+            'name' => p.name,
+            'version' => p.version,
+            'dependencies' => p.dependency_set
+          }
+        end
+
+        root_package_hash = {'name' => package.name, 'version' => package.version, 'dependencies' => package.dependency_set}
+        package_hashes = PackageDependenciesManager.new(release_version).transitive_dependencies(package)
+
+        root_package_hash['dependencies'].sort.map do |dependency_name|
+          arrayify(find_package_hash(dependency_name), package_hashes.dup)
+        end.to_json
+      end
+
+      def dependency_key_from_manifest(package_name, compiled_packages)
+        @all_packages = compiled_packages
+        package = compiled_packages.find { |package| package['name'] == package_name }
+        raise ReleaseExistingPackageHashMismatch, "Package '#{package_name}' not found in the release manifest." if package.nil?
+
+        package['dependencies'].sort.map do |dependency_name|
+          arrayify(find_package_hash(dependency_name), all_packages.dup)
+        end.to_json
+      end
+
+      private
+
+      attr_reader :all_packages
+
+      def arrayify(package, remaining_packages)
+        remaining_packages.delete(package)
+
+        [
+          package['name'],
+          package['version']
+        ].tap do |output|
+          if package['dependencies'] && package['dependencies'].length > 0
+            output << package['dependencies'].sort.map { |sub_dep| arrayify(find_package_hash(sub_dep), remaining_packages) }
+          end
+          output
+        end
+      end
+
+      def find_package_hash(name)
+        all_packages.find { |package| package['name'] == name }
+      end
+    end
+  end
+end

data/lib/bosh/director/lock.rb

@@ -34,7 +34,7 @@ module Bosh::Director
         sleep_interval = [1.0, @expiration/2].max
         begin
           loop do
-            @logger.debug("Renewing lock: #@name")
+            @logger.debug("Renewing lock: #@name, id: #@id")
             redis.watch(@name)
             existing_lock = redis.get(@name)
             lock_id = existing_lock.split(":")[1]
@@ -82,7 +82,7 @@ module Bosh::Director
                           "else: #{existing_lock}")
         if lock_expired?(existing_lock)
           @logger.debug("Lock #@name is already expired, " +
-                            "trying to take it back")
+                            "trying to take it back, id: #@id")
           replaced_lock = redis.getset(@name, "#{lock_expiration}:#@id")
           if replaced_lock == existing_lock
             @logger.debug("Lock #@name was revoked and relocked")

data/lib/bosh/director/log_bundles_cleaner.rb

@@ -1,5 +1,6 @@
 module Bosh::Director
   class LogBundlesCleaner
+
     def initialize(blobstore, log_bundle_ttl, logger)
       @blobstore = blobstore
       @bundle_lifetime = log_bundle_ttl

data/lib/bosh/director/manifest/changeset.rb

@@ -9,9 +9,14 @@ module Bosh::Director
   class Changeset
     KEY_NAME = 'name'
 
-    def initialize(before, after)
+    def initialize(before, after, redact = true, redacted_before = nil, redacted_after = nil)
+      @redact = redact
+      @redacted_before = redacted_before.nil? ? Redactor.redact_properties(Bosh::Common::DeepCopy.copy(before), redact) : redacted_before
+      @redacted_after = redacted_after.nil? ? Redactor.redact_properties(Bosh::Common::DeepCopy.copy(after), redact) : redacted_after
+
       @before = before
       @after = after
+
       if @before && @after
         @merged = @before.deep_merge(@after)
       elsif @before
@@ -26,16 +31,16 @@ module Bosh::Director
 
       @merged.each_pair do |key, value|
         if @before.nil? || @before[key].nil?
-          lines.concat(yaml_lines({key => value}, indent, 'added'))
+          lines.concat(yaml_lines({key => @redacted_after[key]}, indent, 'added'))
 
         elsif @after.nil? || @after[key].nil?
-          lines.concat(yaml_lines({key => value}, indent, 'removed'))
+          lines.concat(yaml_lines({key => @redacted_before[key]}, indent, 'removed'))
 
         elsif @before[key].is_a?(Array) && @after[key].is_a?(Array)
-          lines.concat(compare_arrays(@before[key], @after[key], key, indent))
+          lines.concat(compare_arrays(@before[key], @after[key], @redacted_before[key], @redacted_after[key], key, indent))
 
         elsif value.is_a?(Hash)
-          changeset = Changeset.new(@before[key], @after[key])
+          changeset = Changeset.new(@before[key], @after[key],@redact, @redacted_before[key], @redacted_after[key])
           diff_lines = changeset.diff(indent+1)
           unless diff_lines.empty?
             lines << Line.new(indent, "#{key}:", nil)
@@ -43,8 +48,8 @@ module Bosh::Director
           end
 
         elsif @before[key] != @after[key]
-          lines << Line.new(indent, "#{key}: #{@before[key]}", 'removed')
-          lines << Line.new(indent, "#{key}: #{@after[key]}", 'added')
+          lines.concat(yaml_lines({key => @redacted_before[key]}, indent, 'removed'))
+          lines.concat(yaml_lines({key => @redacted_after[key]}, indent, 'added'))
         end
       end
       lines
@@ -58,51 +63,63 @@ module Bosh::Director
       lines
     end
 
-    def compare_arrays(old_value, new_value, parent_name, indent)
-      added = new_value - old_value
-      removed = old_value - new_value
+    def compare_arrays(old_value, new_value, redacted_old_value, redacted_new_value, parent_name, indent)
+      # combine arrays of redacted and unredacted values. unredacted arrays for diff logic, and redacted arrays for output
+      combined_old_value = old_value.zip redacted_old_value
+      combined_new_value = new_value.zip redacted_new_value
+
+      added = combined_new_value - combined_old_value
+      removed = combined_old_value - combined_new_value
 
       lines = DiffLines.new
 
-      added.each do |elem|
+      added.each do |pair|
+
+        elem = pair.first
+        redacted_elem = pair.last
+
         if elem.is_a?(Hash)
-          using_names = (added+removed).all? { |e| e['name'] }
-          using_ranges = (added+removed).all? { |e| e['range'] }
+          using_names = (added+removed).all? { |e| e.first['name'] }
+
+          using_ranges = (added+removed).all? { |e| e.first['range'] }
           if using_names || using_ranges
+            #clean up duplicate values
             if using_names
-              removed_same_name_element = removed.find { |e| e['name'] == elem['name'] }
+              removed_same_name_element = removed.find { |e| e.first['name'] == elem['name'] }
             elsif using_ranges
-              removed_same_name_element = removed.find { |e| e['range'] == elem['range'] }
+              removed_same_name_element = removed.find { |e| e.first['range'] == elem['range'] }
             end
             removed.delete(removed_same_name_element)
 
             if removed_same_name_element
-              changeset = Changeset.new(removed_same_name_element, elem)
+              changeset = Changeset.new(removed_same_name_element.first, elem, @redact, removed_same_name_element.last, redacted_elem)
               diff_lines = changeset.diff(indent+1)
 
               unless diff_lines.empty?
                 # write name if elem has been changed
                 if using_names
-                  lines.concat(yaml_lines([{'name' => elem['name']}], indent, nil))
+                  lines.concat(yaml_lines([{'name' => redacted_elem['name']}], indent, nil))
                 elsif using_ranges
-                  lines.concat(yaml_lines([{'range' => elem['range']}], indent, nil))
+                  lines.concat(yaml_lines([{'range' => redacted_elem['range']}], indent, nil))
                 end
                 lines.concat(diff_lines)
               end
             else
-              lines.concat(yaml_lines([elem], indent, 'added'))
+              lines.concat(yaml_lines([redacted_elem], indent, 'added'))
             end
 
           else
-            lines.concat(yaml_lines([elem], indent, 'added'))
+            lines.concat(yaml_lines([redacted_elem], indent, 'added'))
           end
         else
-          lines.concat(yaml_lines([elem], indent, 'added'))
+          lines.concat(yaml_lines([redacted_elem], indent, 'added'))
         end
       end
 
       unless removed.empty?
-        lines.concat(yaml_lines(removed, indent, 'removed'))
+        redacted_removed = []
+        removed.each do |pair| redacted_removed.push(pair.last) end
+        lines.concat(yaml_lines(redacted_removed, indent, 'removed'))
       end
 
       unless lines.empty?

data/lib/bosh/director/manifest/diff_lines.rb

@@ -26,11 +26,7 @@ module Bosh::Director
       releases
       update
       jobs
-    )
-
-    REDACT_KEY_NAMES = %w(
-      properties
-      bosh
+      addons
     )
 
     def order
@@ -62,27 +58,5 @@ module Bosh::Director
 
       self.replace(ordered_lines)
     end
-
-    def redact_properties
-      i = 0
-      while i < self.size
-        line = self[i]
-
-        if REDACT_KEY_NAMES.any? { |key_name| line.text =~ /\b#{key_name}:/ }
-          properties_indent = line.full_indent
-          i += 1
-          line = self[i]
-
-          while line && line.full_indent > properties_indent
-            line.text.gsub!(/: .+/, ': <redacted>') # readact hash values
-            line.text.gsub!(/- [^:]+$/, '- <redacted>') # redact array values
-            i += 1
-            line = self[i]
-          end
-        end
-        i += 1
-      end
-      self
-    end
   end
 end

data/lib/bosh/director/manifest/manifest.rb

@@ -1,16 +1,20 @@
+require 'set'
+
 module Bosh::Director
   class Manifest
-    def self.load_from_text(manifest_text, cloud_config)
+    def self.load_from_text(manifest_text, cloud_config, runtime_config)
       cloud_config_hash =  cloud_config.nil? ? nil : cloud_config.manifest
+      runtime_config_hash = runtime_config.nil? ? nil : runtime_config.manifest
       manifest_hash = manifest_text.nil? ? {} : Psych.load(manifest_text)
-      new(manifest_hash, cloud_config_hash)
+      new(manifest_hash, cloud_config_hash, runtime_config_hash)
     end
 
-    attr_reader :manifest_hash, :cloud_config_hash
+    attr_reader :manifest_hash, :cloud_config_hash, :runtime_config_hash
 
-    def initialize(manifest_hash, cloud_config_hash)
+    def initialize(manifest_hash, cloud_config_hash, runtime_config_hash)
       @manifest_hash = manifest_hash
       @cloud_config_hash = cloud_config_hash
+      @runtime_config_hash = runtime_config_hash
     end
 
     def resolve_aliases
@@ -28,12 +32,23 @@ module Bosh::Director
       end
     end
 
-    def diff(other_manifest)
-      Changeset.new(to_hash, other_manifest.to_hash).diff.order.redact_properties
+    def diff(other_manifest, redact)
+      Changeset.new(to_hash, other_manifest.to_hash, redact).diff.order
     end
 
     def to_hash
-      @manifest_hash.merge(@cloud_config_hash || {})
+      hash = @manifest_hash.merge(@cloud_config_hash || {})
+      hash.merge(@runtime_config_hash || {}) do |key, old, new|
+        if key == 'releases'
+          if old && new
+            old.to_set.merge(new.to_set).to_a
+          else
+            old.nil? ? new : old
+          end
+        else
+          new
+        end
+      end
     end
 
     private

data/lib/bosh/director/manifest/redactor.rb

@@ -0,0 +1,44 @@
+module Bosh::Director
+  class Redactor
+    REDACT_KEY_NAMES = %w(
+      properties
+      bosh
+    )
+
+    def self.redact_properties(obj, redact = true ,redact_key_is_ancestor = false)
+      return obj unless redact
+      if redact_key_is_ancestor
+        if obj.respond_to?(:key?)
+          obj.keys.each{ |key|
+            if obj[key].respond_to?(:each)
+              redact_properties(obj[key], redact, true)
+            else
+              obj[key] = '<redacted>'
+            end
+          }
+        elsif obj.respond_to?(:each_index)
+          obj.each_index { |i|
+            if obj[i].respond_to?(:each)
+              redact_properties(obj[i], redact, true)
+            else
+              obj[i] = '<redacted>'
+            end
+          }
+        end
+      else
+        if obj.respond_to?(:each)
+          obj.each{ |a|
+            if obj.respond_to?(:key?) && REDACT_KEY_NAMES.any? { |key| key == a.first } && a.last.respond_to?(:key?)
+              redact_properties(a.last, redact, true)
+            else
+              redact_properties(a.respond_to?(:last) ? a.last : a, redact)
+            end
+
+          }
+        end
+      end
+
+      obj
+    end
+  end
+end

data/lib/bosh/director/models.rb

@@ -2,6 +2,7 @@
 #
 
 require 'bosh/director/models/cloud_config'
+require 'bosh/director/models/runtime_config'
 require 'bosh/director/models/compiled_package'
 require 'bosh/director/models/deployment'
 require 'bosh/director/models/deployment_problem'