bosh-director 1.3215.4.0 → 1.3232.0

data/lib/bosh/director/jobs/update_stemcell.rb

@@ -38,7 +38,7 @@ module Bosh::Director
       def perform
         logger.info("Processing update stemcell")
 
-        event_log.begin_stage("Update stemcell", @stemcell_url ? UPDATE_STEPS + 1 : UPDATE_STEPS)
+        begin_stage("Update stemcell", @stemcell_url ? UPDATE_STEPS + 1 : UPDATE_STEPS)
 
         track_and_log("Downloading remote stemcell") { download_remote_stemcell } if @stemcell_url
 
@@ -66,7 +66,7 @@ module Bosh::Director
           @cloud_properties = safe_property(stemcell_manifest, "cloud_properties", :class => Hash, :optional => true)
           @sha1 = safe_property(stemcell_manifest, "sha1", :class => String)
 
-          logger.info("Found stemcell image `#{@name}/#{@version}', " +
+          logger.info("Found stemcell image '#{@name}/#{@version}', " +
                       "cloud properties are #{@cloud_properties.inspect}")
 
           logger.info("Verifying stemcell image")
@@ -80,7 +80,7 @@ module Bosh::Director
         track_and_log("Checking if this stemcell already exists") do
           begin
             stemcell = @stemcell_manager.find_by_name_and_version @name, @version
-            raise StemcellAlreadyExists, "Stemcell `#{@name}/#{@version}' already exists" unless @fix
+            raise StemcellAlreadyExists, "Stemcell '#{@name}/#{@version}' already exists" unless @fix
           rescue StemcellNotFound => e
             stemcell = Models::Stemcell.new
             stemcell.name = @name
@@ -110,7 +110,7 @@ module Bosh::Director
       def verify_sha1
         stemcell_hash = Digest::SHA1.file(@stemcell_path).hexdigest
         if stemcell_hash != @stemcell_sha1
-          raise StemcellSha1DoesNotMatch, "Stemcell SHA1 `#{stemcell_hash}' does not match the expected SHA1 `#{@stemcell_sha1}'"
+          raise StemcellSha1DoesNotMatch, "Stemcell SHA1 '#{stemcell_hash}' does not match the expected SHA1 '#{@stemcell_sha1}'"
         end
       end
 

data/lib/bosh/director/jobs/vm_state.rb

@@ -52,6 +52,7 @@ module Bosh::Director
           :job_name => instance.job,
           :index => instance.index,
           :job_state => job_state,
+          :state => instance.state,
           :resource_pool => vm_type_name,
           :vm_type => vm_type_name,
           :vitals => job_vitals,
@@ -87,8 +88,6 @@ module Bosh::Director
           rescue Bosh::Director::RpcTimeout
             job_state = 'unresponsive agent'
           end
-        else
-          job_state = 'missing vm'
         end
 
         return job_state, job_vitals, processes, ips

data/lib/bosh/director/lock.rb

@@ -1,6 +1,6 @@
 module Bosh::Director
 
-  # Distributed lock backed by Redis.
+  # Distributed lock backed by DB.
   class Lock
 
     # Error returned when Lock could not be acquired.
@@ -14,7 +14,7 @@ module Bosh::Director
     #   lock
     def initialize(name, opts = {})
       @name = name
-      @id = SecureRandom.uuid
+      @uid = SecureRandom.uuid
       @timeout = opts[:timeout] || 1.0
       @expiration = opts[:expiration] || 10.0
       @logger = Config.logger
@@ -30,24 +30,21 @@ module Bosh::Director
       acquire
 
       @refresh_thread = Thread.new do
-        redis = Config.redis
         sleep_interval = [1.0, @expiration/2].max
         begin
-          loop do
-            @logger.debug("Renewing lock: #@name, id: #@id")
-            redis.watch(@name)
-            existing_lock = redis.get(@name)
-            lock_id = existing_lock.split(":")[1]
-            break if lock_id != @id
+          stopped = false
+          until stopped
+            @logger.debug("Renewing lock: #@name")
             lock_expiration = Time.now.to_f + @expiration + 1
-            redis.multi do
-              redis.set(@name, "#{lock_expiration}:#@id")
+
+            if Models::Lock.where(name: @name, uid: @uid).update(expired_at: Time.at(lock_expiration)) == 0
+              stopped = true
             end
-            sleep(sleep_interval)
+
+            sleep(sleep_interval) unless stopped
           end
         ensure
           @logger.debug("Lock renewal thread exiting")
-          redis.quit
         end
       end
 
@@ -72,65 +69,43 @@ module Bosh::Director
 
     def acquire
       @logger.debug("Acquiring lock: #@name")
-      redis = Config.redis
       started = Time.now
 
       lock_expiration = Time.now.to_f + @expiration + 1
-      until redis.setnx(@name, "#{lock_expiration}:#@id")
-        existing_lock = redis.get(@name)
-        @logger.debug("Lock #@name is already locked by someone " +
-                          "else: #{existing_lock}")
-        if lock_expired?(existing_lock)
-          @logger.debug("Lock #@name is already expired, " +
-                            "trying to take it back, id: #@id")
-          replaced_lock = redis.getset(@name, "#{lock_expiration}:#@id")
-          if replaced_lock == existing_lock
-            @logger.debug("Lock #@name was revoked and relocked")
-            break
-          else
-            @logger.debug("Lock #@name was acquired by someone else, " +
-                              "trying again")
+      acquired = false
+      until acquired
+        begin
+          Models::Lock.create(name: @name, uid: @uid, expired_at: Time.at(lock_expiration))
+          acquired = true
+        rescue Sequel::DatabaseError
+          affected_locks = Models::Lock.where(name: @name).where{ expired_at < Time.now }.update(uid: @uid, expired_at: Time.at(lock_expiration))
+          if affected_locks == 1
+            acquired = true
           end
         end
 
-        raise TimeoutError, "Redis lock #{@name} is acquired by another thread" if Time.now - started > @timeout
-
-        sleep(0.5)
-
-        lock_expiration = Time.now.to_f + @expiration + 1
+        unless acquired
+          raise TimeoutError, "Failed to acquire lock for #{@name} uid: #{@uid}" if Time.now - started > @timeout
+          sleep(0.5)
+          lock_expiration = Time.now.to_f + @expiration + 1
+        end
       end
 
       @lock_expiration = lock_expiration
       @logger.debug("Acquired lock: #@name")
     end
 
+
     def delete
-      @logger.debug("Deleting lock: #@name")
-      redis = Config.redis
-
-      redis.watch(@name)
-      existing_lock = redis.get(@name)
-      if existing_lock.nil?
-        @logger.debug("Lost lock #@name")
-        redis.unwatch
+      if Models::Lock.where(name: @name, uid: @uid).delete > 0
+        @logger.debug("Deleted lock: #{@name} uid: #{@uid}")
       else
-        lock_id = existing_lock.split(":")[1]
-        if lock_id == @id
-          redis.multi do
-            redis.del(@name)
-          end
-        else
-          redis.unwatch
-        end
-        @logger.debug("Deleted lock: #@name")
+        @logger.debug("Can not find lock: #{@name} - uid: #{@uid}")
       end
     end
 
-    def lock_expired?(lock)
-      existing_lock_expiration = lock.split(":")[0].to_f
-      lock_time_left = existing_lock_expiration - Time.now.to_f
-      @logger.info("Lock: #{lock} expires in #{lock_time_left} seconds")
-      lock_time_left < 0
+    def lock_expired?(lock_record)
+      lock_record.expired_at < Time.now
     end
   end
 end

data/lib/bosh/director/logs_fetcher.rb

@@ -3,8 +3,7 @@ module Bosh::Director
     # @param [Bosh::Director::EventLog::Log] event_log
     # @param [Bosh::Director::Api::InstanceManager] instance_manager
     # @param [Bosh::Director::LogBundlesCleaner] log_bundles_cleaner
-    def initialize(event_log, instance_manager, log_bundles_cleaner, logger)
-      @event_log = event_log
+    def initialize(instance_manager, log_bundles_cleaner, logger)
       @instance_manager = instance_manager
       @log_bundles_cleaner = log_bundles_cleaner
       @logger = logger
@@ -21,7 +20,7 @@ module Bosh::Director
       agent = @instance_manager.agent_client_for(instance)
       blobstore_id = nil
 
-      stage = @event_log.begin_stage("Fetching logs for #{instance.job}/#{instance.uuid} (#{instance.index})", 1)
+      stage = Config.event_log.begin_stage("Fetching logs for #{instance.job}/#{instance.uuid} (#{instance.index})", 1)
       stage.advance_and_track('Finding and packing log files') do
         fetch_logs_result = agent.fetch_logs(log_type, filters)
         blobstore_id = fetch_logs_result['blobstore_id']

data/lib/bosh/director/manifest/changeset.rb

@@ -1,24 +1,21 @@
-class ::Hash
-  def deep_merge(second)
-    merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
-    self.merge(second, &merger)
-  end
-end
-
 module Bosh::Director
   class Changeset
     KEY_NAME = 'name'
 
-    def initialize(before, after, redact = true, redacted_before = nil, redacted_after = nil)
-      @redact = redact
-      @redacted_before = redacted_before.nil? ? Redactor.redact_properties(Bosh::Common::DeepCopy.copy(before), redact) : redacted_before
-      @redacted_after = redacted_after.nil? ? Redactor.redact_properties(Bosh::Common::DeepCopy.copy(after), redact) : redacted_after
+    REDACT_KEY_NAMES = %w(
+      properties
+      bosh
+    )
+
+    def initialize(before, after, redacted_before = nil, redacted_after = nil)
+      @redacted_before = redacted_before.nil? ? Changeset.redact_properties!(Bosh::Common::DeepCopy.copy(before)) : redacted_before
+      @redacted_after = redacted_after.nil? ? Changeset.redact_properties!(Bosh::Common::DeepCopy.copy(after)) : redacted_after
 
       @before = before
       @after = after
 
       if @before && @after
-        @merged = @before.deep_merge(@after)
+        @merged = deep_merge(@before, @after)
       elsif @before
         @merged = @before
       else
@@ -26,64 +23,113 @@ module Bosh::Director
       end
     end
 
-    def diff(indent = 0)
-      lines = DiffLines.new
+    # redacts properties from ruby object to avoid having to use a regex to redact properties from diff output
+    # please do not use regexes for diffing/redacting
+    def self.redact_properties!(obj, redact_key_is_ancestor = false)
+      if redact_key_is_ancestor
+        if obj.respond_to?(:key?)
+          obj.keys.each{ |key|
+            if obj[key].respond_to?(:each)
+              redact_properties!(obj[key], true)
+            else
+              obj[key] = '<redacted>'
+            end
+          }
+        elsif obj.respond_to?(:each_index)
+          obj.each_index { |i|
+            if obj[i].respond_to?(:each)
+              redact_properties!(obj[i], true)
+            else
+              obj[i] = '<redacted>'
+            end
+          }
+        end
+      else
+        if obj.respond_to?(:each)
+          obj.each{ |a|
+            if obj.respond_to?(:key?) && REDACT_KEY_NAMES.any? { |key| key == a.first } && a.last.respond_to?(:key?)
+              redact_properties!(a.last, true)
+            else
+              redact_properties!(a.respond_to?(:last) ? a.last : a)
+            end
 
-      @merged.each_pair do |key, value|
-        if @before.nil? || @before[key].nil?
-          lines.concat(yaml_lines({key => @redacted_after[key]}, indent, 'added'))
+          }
+        end
+      end
 
-        elsif @after.nil? || @after[key].nil?
-          lines.concat(yaml_lines({key => @redacted_before[key]}, indent, 'removed'))
+      obj
+    end
 
-        elsif @before[key].is_a?(Array) && @after[key].is_a?(Array)
-          lines.concat(compare_arrays(@before[key], @after[key], @redacted_before[key], @redacted_after[key], key, indent))
+    def diff(redact=true, indent = 0)
+      lines = DiffLines.new
 
-        elsif value.is_a?(Hash)
-          changeset = Changeset.new(@before[key], @after[key],@redact, @redacted_before[key], @redacted_after[key])
-          diff_lines = changeset.diff(indent+1)
-          unless diff_lines.empty?
-            lines << Line.new(indent, "#{key}:", nil)
-            lines.concat(diff_lines)
-          end
+      if redact
+        output_values_before = @redacted_before
+        output_values_after = @redacted_after
+      else
+        output_values_before = @before
+        output_values_after = @after
+      end
 
-        elsif @before[key] != @after[key]
-          lines.concat(yaml_lines({key => @redacted_before[key]}, indent, 'removed'))
-          lines.concat(yaml_lines({key => @redacted_after[key]}, indent, 'added'))
+      @merged.each_pair do |key, value|
+        if @before[key] != @after[key]
+          if @before.nil? || @before[key].nil?
+            lines.concat(yaml_lines({key => output_values_after[key]}, indent, 'added'))
+
+          elsif @after.nil? || @after[key].nil?
+            lines.concat(yaml_lines({key => output_values_before[key]}, indent, 'removed'))
+
+          elsif @before[key].is_a?(Array) && @after[key].is_a?(Array)
+            lines.concat(compare_arrays(@before[key], @after[key], output_values_before[key], output_values_after[key], key, redact, indent))
+
+          elsif @before[key].is_a?(Hash) && @after[key].is_a?(Hash)
+            changeset = Changeset.new(@before[key], @after[key], output_values_before[key], output_values_after[key])
+            diff_lines = changeset.diff(redact, indent+1)
+            unless diff_lines.empty?
+              lines << Line.new(indent, "#{key}:", nil)
+              lines.concat(diff_lines)
+            end
+
+          else
+            lines.concat(yaml_lines({key => output_values_before[key]}, indent, 'removed'))
+            lines.concat(yaml_lines({key => output_values_after[key]}, indent, 'added'))
+          end
         end
       end
       lines
     end
 
+    private
+
+    def deep_merge(first, second)
+      merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
+      first.merge(second, &merger)
+    end
+
     def yaml_lines(value, indent, state)
       lines = DiffLines.new
-      value.to_yaml(indent: Line::INDENT).gsub("---\n", '').split("\n").each do |line|
+      value.to_yaml(indent: Line::INDENT).gsub(/^---\n/, '').split("\n").each do |line|
         lines << Line.new(indent, line, state)
       end
       lines
     end
 
-    def compare_arrays(old_value, new_value, redacted_old_value, redacted_new_value, parent_name, indent)
+    def compare_arrays(old_value, new_value, output_old_value, output_new_value, parent_name, redact, indent)
       # combine arrays of redacted and unredacted values. unredacted arrays for diff logic, and redacted arrays for output
-      combined_old_value = old_value.zip redacted_old_value
-      combined_new_value = new_value.zip redacted_new_value
-
+      combined_old_value = old_value.zip output_old_value
+      combined_new_value = new_value.zip output_new_value
       added = combined_new_value - combined_old_value
       removed = combined_old_value - combined_new_value
 
       lines = DiffLines.new
 
       added.each do |pair|
-
         elem = pair.first
         redacted_elem = pair.last
-
         if elem.is_a?(Hash)
           using_names = (added+removed).all? { |e| e.first['name'] }
-
           using_ranges = (added+removed).all? { |e| e.first['range'] }
           if using_names || using_ranges
-            #clean up duplicate values
             if using_names
               removed_same_name_element = removed.find { |e| e.first['name'] == elem['name'] }
             elsif using_ranges
@@ -92,8 +138,8 @@ module Bosh::Director
             removed.delete(removed_same_name_element)
 
             if removed_same_name_element
-              changeset = Changeset.new(removed_same_name_element.first, elem, @redact, removed_same_name_element.last, redacted_elem)
-              diff_lines = changeset.diff(indent+1)
+              changeset = Changeset.new(removed_same_name_element.first, elem, removed_same_name_element.last, redacted_elem)
+              diff_lines = changeset.diff(redact, indent+1)
 
               unless diff_lines.empty?
                 # write name if elem has been changed

data/lib/bosh/director/manifest/manifest.rb

@@ -33,7 +33,7 @@ module Bosh::Director
     end
 
     def diff(other_manifest, redact)
-      Changeset.new(to_hash, other_manifest.to_hash, redact).diff.order
+      Changeset.new(to_hash, other_manifest.to_hash).diff(redact).order
     end
 
     def to_hash

data/lib/bosh/director/models.rb

@@ -23,6 +23,9 @@ require 'bosh/director/models/template'
 require 'bosh/director/models/user'
 require 'bosh/director/models/persistent_disk'
 require 'bosh/director/models/rendered_templates_archive'
+require 'bosh/director/models/lock'
+require 'delayed_job_sequel'
+require 'bosh/director/models/event'
 
 module Bosh::Director
   module Models

data/lib/bosh/director/models/event.rb

@@ -0,0 +1,18 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director::Models
+  class Event < Sequel::Model(Bosh::Director::Config.db)
+    def validate
+      validates_presence [:timestamp, :action, :object_type]
+    end
+
+    def context
+      return {} if context_json.nil?
+      Yajl::Parser.parse(context_json)
+    end
+
+    def context=(data)
+      self.context_json = Yajl::Encoder.encode(data)
+    end
+  end
+end

data/lib/bosh/director/models/lock.rb

@@ -0,0 +1,9 @@
+module Bosh::Director::Models
+  class Lock < Sequel::Model(Bosh::Director::Config.db)
+    def validate
+      validates_presence [:name,:expired_at, :uid]
+      validates_unique [:name, :uid]
+    end
+
+  end
+end

data/lib/bosh/director/models/release_version.rb

@@ -12,7 +12,9 @@ module Bosh::Director::Models
     end
 
     def package_by_name(package_name)
-      packages_by_name.fetch(package_name)
+      packages_by_name.fetch(package_name) do
+        raise "Package name '#{package_name}' not found in release '#{release.name}/#{version}'"
+      end
     end
 
     private