bosh-director 1.5.0.pre.1113

data/lib/bosh/director/jobs/delete_release.rb

@@ -0,0 +1,219 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module Jobs
+    class DeleteRelease < BaseJob
+      include LockHelper
+
+      @queue = :normal
+
+      def self.job_type
+        :delete_release
+      end
+
+      def initialize(name, options = {})
+        @name = name
+        @blobstore = options.fetch(:blobstore) { App.instance.blobstores.blobstore }
+        @errors = []
+        @force = !!options["force"]
+        @version = options["version"]
+        @release_manager = Api::ReleaseManager.new
+      end
+
+      def delete_release_version(release_version)
+        release = release_version.release
+
+        packages_to_keep = []
+        packages_to_delete = []
+        templates_to_keep = []
+        templates_to_delete = []
+
+        # We don't delete packages inside this loop b/c Sequel will also delete
+        # them from packages collection we're iterating on which will lead to
+        # skipping some packages
+        release_version.packages.each do |package|
+          if package.release_versions == [ release_version ]
+            packages_to_delete << package
+          else
+            packages_to_keep << package
+          end
+        end
+
+        release_version.templates.each do |template|
+          if template.release_versions == [ release_version ]
+            templates_to_delete << template
+          else
+            templates_to_keep << template
+          end
+        end
+
+        event_log.begin_stage("Deleting packages", packages_to_delete.count)
+        packages_to_delete.each do |package|
+          track_and_log("#{package.name}/#{package.version}") do
+            logger.info("Package #{package.name}/#{package.version} " +
+                        "is only used by this release version " +
+                        "and will be deleted")
+            delete_package(package)
+          end
+        end
+
+        packages_to_keep.each do |package|
+          logger.info("Keeping package #{package.name}/#{package.version} " +
+                      "as it is used by other release versions")
+          package.remove_release_version(release_version)
+        end
+
+        event_log.begin_stage("Deleting templates", templates_to_delete.count)
+        templates_to_delete.each do |template|
+          track_and_log("#{template.name}/#{template.version}") do
+            logger.info("Template #{template.name}/#{template.version} " +
+                        "is only used by this release version " +
+                        "and will be deleted")
+            delete_template(template)
+          end
+        end
+
+        templates_to_keep.each do |template|
+          logger.info("Keeping template " +
+                      "#{template.name}/#{template.version} as it is used " +
+                      "by other release versions")
+          template.remove_release_version(release_version)
+        end
+
+        logger.info("Remove all deployments in release version")
+        release_version.remove_all_deployments
+
+        if @errors.empty? || @force
+          release_version.destroy
+        end
+
+        if release.versions.empty?
+          delete_release(release)
+        end
+      end
+
+      def delete_release(release)
+        event_log.begin_stage("Deleting packages", release.packages.count)
+        release.packages.each do |package|
+          track_and_log("#{package.name}/#{package.version}") do
+            delete_package(package)
+          end
+        end
+
+        event_log.begin_stage("Deleting templates", release.templates.count)
+        release.templates.each do |template|
+          track_and_log("#{template.name}/#{template.version}") do
+            delete_template(template)
+          end
+        end
+
+        if @errors.empty? || @force
+          event_log.begin_stage("Deleting release versions",
+                                release.versions.count)
+
+          release.versions.each do |release_version|
+            track_and_log("#{release.name}/#{release_version.version}") do
+              release_version.destroy
+            end
+          end
+
+          release.destroy
+        end
+      end
+
+      def delete_package(package)
+        compiled_packages = package.compiled_packages
+
+        logger.info("Deleting package #{package.name}/#{package.version}")
+
+        compiled_packages.each do |compiled_package|
+          stemcell = compiled_package.stemcell
+          logger.info("Deleting compiled package " +
+                      "(#{compiled_package.blobstore_id}) " +
+                      "#{package.name}/#{package.version} " +
+                      "for #{stemcell.name}/#{stemcell.version}")
+          delete_blobstore_id(compiled_package.blobstore_id) do
+            compiled_package.destroy
+          end
+        end
+
+        delete_blobstore_id(package.blobstore_id) do
+          package.remove_all_release_versions
+          package.destroy
+        end
+      end
+
+      def delete_template(template)
+        logger.info("Deleting template: #{template.name}/#{template.version}")
+
+        delete_blobstore_id(template.blobstore_id) do
+          template.remove_all_release_versions
+          template.destroy
+        end
+      end
+
+      def perform
+        logger.info("Processing delete release")
+
+        with_release_lock(@name, :timeout => 10) do
+          logger.info("Looking up release: #{@name}")
+          release = @release_manager.find_by_name(@name)
+          desc = "#{@name}/#{@version}"
+          logger.info("Found: #{release.name}")
+
+          if @version
+            logger.info("Looking up release version `#{desc}'")
+            release_version = @release_manager.find_version(release, @version)
+            logger.info("Found release version `#{desc}'")
+            logger.info("Checking for any deployments still using " +
+                         "this particular release version")
+
+            deployments = release_version.deployments
+
+            unless deployments.empty?
+              names = deployments.map{ |d| d.name }.join(", ")
+              raise ReleaseVersionInUse,
+                    "Release version `#{desc}' is still in use by: #{names}"
+            end
+
+            delete_release_version(release_version)
+          else
+            logger.info("Checking for any deployments still using the release")
+            deployments = release.versions.map { |version|
+              version.deployments
+            }.flatten.uniq
+
+            unless deployments.empty?
+              names = deployments.map { |d| d.name }.join(", ")
+              raise ReleaseInUse,
+                    "Release `#{@name}' is still in use by: #{names}"
+            end
+
+            delete_release(release)
+          end
+        end
+
+        unless @errors.empty?
+          errors = @errors.map { |e| e.to_s }.join(", ")
+          raise ReleaseDeleteFailed, "Can't delete release: #{errors}"
+        end
+
+        "/release/#{@name}"
+      end
+
+      def delete_blobstore_id(blobstore_id)
+        deleted = false
+        begin
+          @blobstore.delete(blobstore_id)
+          deleted = true
+        rescue Exception => e
+          logger.warn("Could not delete from blobstore: #{e}\n " +
+                      e.backtrace.join("\n"))
+          @errors << e
+        end
+        yield if deleted || @force
+      end
+
+    end
+  end
+end

data/lib/bosh/director/jobs/delete_snapshots.rb

@@ -0,0 +1,23 @@
+module Bosh::Director
+  module Jobs
+    class DeleteSnapshots < BaseJob
+
+      @queue = :normal
+
+      def self.job_type
+        :delete_snapshot
+      end
+
+      def initialize(snapshots_cids)
+        @snapshot_cids = snapshots_cids
+      end
+
+      def perform
+        logger.info("deleting snapshots: #{@snapshot_cids.join(', ')}")
+        snapshots = Bosh::Director::Models::Snapshot.where(snapshot_cid: @snapshot_cids).to_a
+        Bosh::Director::Api::SnapshotManager.delete_snapshots(snapshots)
+        "snapshot(s) #{@snapshot_cids.join(', ')} deleted"
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/delete_stemcell.rb

@@ -0,0 +1,102 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module Jobs
+    class DeleteStemcell < BaseJob
+      include LockHelper
+
+      @queue = :normal
+
+      def self.job_type
+        :delete_stemcell
+      end
+
+      # @param [String] name Stemcell name
+      # @param [String] version Stemcell version
+      def initialize(name, version, options = {})
+        @name = name
+        @version = version
+        @options = options
+        @cloud = Config.cloud
+        @blobstore = options.fetch(:blobstore) { App.instance.blobstores.blobstore }
+        @stemcell_manager = Api::StemcellManager.new
+      end
+
+      def perform
+        logger.info("Processing delete stemcell")
+        with_stemcell_lock(@name, @version) do
+          logger.info("Looking up stemcell: #{desc}")
+          @stemcell = @stemcell_manager.find_by_name_and_version(@name, @version)
+          logger.info("Found: #{@stemcell.pretty_inspect}")
+
+          validate_deletion
+          delete_from_cloud
+          cleanup_compiled_packages
+          delete_stemcell_metadata
+        end
+
+        "/stemcells/#{@name}/#{@version}"
+      end
+
+      def validate_deletion
+        logger.info("Checking for any deployments still using the stemcell")
+        deployments = @stemcell.deployments
+        unless deployments.empty?
+          names = deployments.map { |d| d.name }.join(", ")
+          raise StemcellInUse,
+                "Stemcell `#{desc}' is still in use by: #{names}"
+        end
+      end
+
+      def delete_from_cloud
+        event_log.begin_stage("Deleting stemcell from cloud", 1)
+
+        event_log.track("Delete stemcell") do
+          @cloud.delete_stemcell(@stemcell.cid)
+        end
+      rescue => e
+        raise unless force?
+        logger.warn(e.backtrace.join("\n"))
+        logger.info("Force deleting is set, ignoring exception: #{e.message}")
+      end
+
+      def cleanup_compiled_packages
+        logger.info("Looking for any compiled packages on this stemcell")
+        compiled_packages =
+            Models::CompiledPackage.filter(:stemcell_id => @stemcell.id)
+
+        event_log.begin_stage("Deleting compiled packages",
+                              compiled_packages.count, [@name, @version])
+        logger.info("Deleting compiled packages " +
+                        "(#{compiled_packages.count}) for `#{desc}'")
+
+        compiled_packages.each do |compiled_package|
+          next unless compiled_package
+
+          package = compiled_package.package
+          track_and_log("#{package.name}/#{package.version}") do
+            logger.info("Deleting compiled package: " +
+                            "#{package.name}/#{package.version}")
+            @blobstore.delete(compiled_package.blobstore_id)
+            compiled_package.destroy
+          end
+        end
+      end
+
+      def delete_stemcell_metadata
+        event_log.begin_stage("Deleting stemcell metadata", 1)
+        event_log.track("Deleting stemcell metadata") do
+          @stemcell.destroy
+        end
+      end
+
+      def desc
+        "#@name/#@version"
+      end
+
+      def force?
+        @options["force"]
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/fetch_logs.rb

@@ -0,0 +1,99 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module Jobs
+    class FetchLogs < BaseJob
+      include LockHelper
+
+      DEFAULT_BUNDLE_LIFETIME = 86400 * 10 # 10 days
+
+      @queue = :normal
+
+      attr_accessor :bundle_lifetime
+
+      def self.job_type
+        :fetch_logs
+      end
+
+      def initialize(instance_id, options = {})
+        @blobstore = options.fetch(:blobstore) { App.instance.blobstores.blobstore }
+        @instance_id = instance_id
+        @log_type = options["type"] || "job"
+        @filters = options["filters"]
+        @instance_manager = Api::InstanceManager.new
+        @bundle_lifetime = DEFAULT_BUNDLE_LIFETIME
+      end
+
+      def perform
+        instance = @instance_manager.find_instance(@instance_id)
+        desc = "#{instance.job}/#{instance.index}"
+
+        deployment = instance.deployment
+
+        if deployment.nil?
+          raise InstanceDeploymentMissing,
+                "`#{desc}' doesn't belong to any deployment"
+        end
+
+        with_deployment_lock(deployment) do
+          cleanup_old_bundles
+
+          event_log.begin_stage("Fetching logs for #{desc}", 1)
+
+          agent = @instance_manager.agent_client_for(instance)
+          blobstore_id = nil
+
+          track_and_log("Finding and packing log files") do
+            logger.info("Fetching logs from agent: " +
+                         "log_type='#{@log_type}', filters='#{@filters}'")
+
+            task = agent.fetch_logs(@log_type, @filters)
+            blobstore_id = task["blobstore_id"]
+          end
+
+          if blobstore_id.nil?
+            raise AgentTaskNoBlobstoreId,
+                  "Agent didn't return a blobstore object id for packaged logs"
+          end
+
+          Models::LogBundle.create(:blobstore_id => blobstore_id,
+                                   :timestamp => Time.now)
+
+          # The returned value of this method is used as task result
+          # and gets extracted by CLI as a tarball blobstore id
+          blobstore_id
+        end
+      end
+
+      def cleanup_old_bundles
+        old_bundles = Models::LogBundle.filter("timestamp <= ?",
+                                               Time.now - bundle_lifetime)
+        count = old_bundles.count
+
+        if count == 0
+          logger.info("No old bundles to delete")
+          return
+        end
+
+        logger.info("Deleting #{count} old log bundle#{count > 1 ? "s" : ""}")
+
+        old_bundles.each do |bundle|
+          begin
+            logger.info("Deleting log bundle #{bundle.id}: " +
+                        "#{bundle.blobstore_id}")
+            @blobstore.delete(bundle.blobstore_id)
+            bundle.delete
+          rescue Bosh::Blobstore::BlobstoreError => e
+            logger.warn("Could not delete #{bundle.blobstore_id}: #{e}")
+            # Assuming object has been deleted from blobstore by someone else,
+            # cleaning up DB record accordingly
+            if e.kind_of?(Bosh::Blobstore::NotFound)
+              bundle.delete
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/bosh/director/jobs/scheduled_backup.rb

@@ -0,0 +1,38 @@
+module Bosh::Director
+  module Jobs
+    class ScheduledBackup < BaseJob
+      @queue = :normal
+
+      attr_reader :backup_job
+
+      def self.job_type
+        :scheduled_backup
+      end
+
+      def initialize(options={})
+        @backup_job = options.fetch(:backup_job) { Backup.new(backup_file) }
+        @backup_destination = options.fetch(:backup_destination) { App.instance.blobstores.backup_destination }
+      end
+
+      def perform
+        @backup_job.perform
+
+        blobstore_path = "backup-#{Time.now.utc.iso8601}.tgz"
+
+        File.open(@backup_job.backup_file) do |f|
+          @backup_destination.create(f, blobstore_path)
+        end
+
+        "Stored `#{blobstore_path}' in backup blobstore"
+      ensure
+        FileUtils.rm_f(@backup_job.backup_file)
+      end
+
+      private
+
+      def backup_file
+        File.join(Dir.tmpdir, "backup-#{task_id}.tgz")
+      end
+    end
+  end
+end