xolo-server 2.1.0 → 2.2.2

data/lib/xolo/server/helpers/maintenance.rb

@@ -12,7 +12,11 @@ module Xolo
 
     module Helpers
 
-      # Nightly cleanup of deprecated and skipped packages.
+      # Nightly maintenance tasks
+      # - accept lingering TEd EAs, if configured to
+      # - autorelease versions
+      # - clean up old versions
+      # - notify title maintainers of old unreleased pilots
       #
       # Also, alerts will be posted, and Emails will be sent to the
       # admins who added versions that have been in pilot for more than
@@ -29,8 +33,27 @@ module Xolo
         # Constants
         #####################################
 
-        # At what hour should the nightly cleanup run?
-        CLEANUP_HOUR = 2
+        # TODO: ? use the next 3 values as defaults, and
+        # set them in the server config
+
+        # how often does our maint timer check to see if it should run maintenance?
+        # Slightly less than an hour means there will always be at least one check every
+        # hour, sometimes two.
+        MAINT_CHECK_INTERVAL = 3500
+        # MAINT_CHECK_INTERVAL = 600
+
+        # At what hour should the nightly maintenance run?
+        MAINT_HOUR = 2
+        # MAINT_HOUR = 10
+
+        # Maint won't run unless the last run was at least this many
+        # seconds ago.
+        # 23 hrs ago means that even if we check twice within the
+        # MAINT_HOUR, only the first one will run.
+        MAINT_MAX_FREQ_SECS = 23 * 3600
+
+        # the route we POST to, to start the nightly process.
+        TIMED_MAINT_TRIGGER_ROUTE = '/maint/maint-internal'
 
         # on which day of the month should we send the unreleased pilot notifications?
         UNRELEASED_PILOTS_NOTIFICATION_DAY = 1
@@ -39,7 +62,7 @@ module Xolo
         # be automatically deleted this many days later.
         # If not set in the server config, this is
         # the default value.
-        # use 0 or less to disable cleanup of deprecated versions
+        # use 0 or less to disable maintenance of deprecated versions
         DFT_DEPRECATED_LIFETIME_DAYS = 30
 
         # If a pilot has not been released in this many
@@ -55,71 +78,78 @@ module Xolo
         # Module Methods
         #####################################
 
-        # A mutex for the cleanup process
-        #
-        # TODO: use Concrrent Ruby instead of Mutex
+        # A mutex for the maint process
         #
         # @return [Mutex] the mutex
         #####################
-        def self.cleanup_mutex
-          @cleanup_mutex ||= Mutex.new
+        def self.maint_mutex
+          @maint_mutex ||= Mutex.new
         end
 
-        # nightly cleanup is done by a Concurrent::TimerTask, which checks every
+        # nightly maint is done by a Concurrent::TimerTask, which checks every
         # hour to see if it should do anything.
         #
-        # It will only do the cleanup if the current time is in the 2am hour
-        # (02:00 - 02:59)
+        # It will only do the maint if the current time is in the MAINT_HOUR hour
+        # (02:00 - 02:59 if MAINT_HOUR = 2)
         #
-        # We trigger the cleanup by POSTing to /cleanup, so that it runs
+        # We trigger the maint by POSTing to TIMED_MAINT_TRIGGER_ROUTE, so that it runs
         # in the context of a request, having access to Title and Version instantiation.
         #
         # @return [Concurrent::TimerTask] the timed task to do log rotation
         ######################################
-        def self.cleanup_timer_task
-          return @cleanup_timer_task if @cleanup_timer_task
+        def self.maint_timer_task
+          return @maint_timer_task if @maint_timer_task
+
+          @maint_timer_task =
+            Concurrent::TimerTask.new(execution_interval: MAINT_CHECK_INTERVAL) { post_to_start_maint }
 
-          @cleanup_timer_task =
-            Concurrent::TimerTask.new(execution_interval: 3600) { post_to_start_cleanup }
+          Xolo::Server.logger.info "Created Concurrent::TimerTask for nightly maintenance. Will check every #{MAINT_CHECK_INTERVAL} secs, and run during the hour of: #{MAINT_HOUR}"
 
-          Xolo::Server.logger.info 'Created Concurrent::TimerTask for nightly cleanup.'
-          @cleanup_timer_task
+          @maint_timer_task
         end
 
-        # When was our last cleanup?
-        # @return [Time] the time of the last cleanup, or the epoch if never
+        # When was our last maint?
+        # @return [Time] the time of the last maint, or the epoch if never
         ######################################
-        def self.last_cleanup
-          @last_cleanup ||= Time.at(0)
+        def self.last_maint
+          @last_maint ||= Time.at(0)
         end
 
-        # Set the time of the last cleanup
-        # @param time [Time] the time of the last cleanup
-        # @return [Time] the time of the last cleanup
+        # Set the time of the last maint
+        # @param time [Time] the time of the last maint
+        # @return [Time] the time of the last maint
         ######################################
-        def self.last_cleanup=(time)
-          @last_cleanup = time
+        def self.last_maint=(time)
+          @last_maint = time
         end
 
-        # post to the server to start the cleanup process
-        # This is done so that the cleanup can run in the context of a request,
+        # post to the server to start the maint process
+        # This is done so that the maint can run in the context of a request,
         # having access to Title and Version instantiation.
         #
-        # @param force [Boolean] force the cleanup to run now
+        # @param force [Boolean] force the maint to run now
         # @return [void]
         ######################################
-        def self.post_to_start_cleanup(force: false)
+        def self.post_to_start_maint(force: false)
           if Xolo::Server.shutting_down?
-            Xolo::Server.logger.info 'Not starting cleanup, server is shutting down'
+            Xolo::Server.logger.info 'Not starting maintenance, server is shutting down'
             return
           end
 
-          # only run the cleanup if it's between 2am and 3am
-          # and the last one was more than 23 hrs ago
-          last_cleanup_hrs_ago = (Time.now - last_cleanup) / 3600
-          return unless force || (Time.now.hour == CLEANUP_HOUR && last_cleanup_hrs_ago > 23)
+          # only run the maintenance if it's during the MAINT_HOUR
+          # and the last one was more than MAINT_MAX_FREQ_SECS ago
+          if force
+            Xolo::Server.logger.info 'Maint: Starting now due to force'
+
+          elsif Time.now.hour == MAINT_HOUR && (Time.now - last_maint) > MAINT_MAX_FREQ_SECS
+            Xolo::Server.logger.info 'Maint: Starting nightly run now'
+
+          else
+            Xolo::Server.logger.info "Maint: Not starting, it isn't time"
+            return
+          end
 
-          uri = URI.parse "https://#{Xolo::Server::Helpers::Auth::IPV4_LOOPBACK}/maint/cleanup-internal"
+          uri = URI.parse "https://#{Xolo::Server::Helpers::Auth::IPV4_LOOPBACK}#{TIMED_MAINT_TRIGGER_ROUTE}"
           https = Net::HTTP.new(uri.host, uri.port)
           https.use_ssl = true
           # The server cert may be self-signed and/or doesn't
@@ -130,32 +160,38 @@ module Xolo
           request['Authorization'] = Xolo::Server::Helpers::Auth.internal_auth_token_header
 
           response = https.request(request)
-          Xolo::Server.logger.info "Cleanup request response: #{response.code} #{response.body}"
+          Xolo::Server.logger.info "Maintenance request response: #{response.code} #{response.body}"
         end
 
-        # Cleanup things that need to be cleaned up
+        # Perform maintenance tasks
         # @return [void]
         ################################
-        def run_cleanup
+        def run_maint
           if Xolo::Server.shutting_down?
-            log_info 'Cleanup: Not starting cleanup, server is shutting down'
+            log_info 'Maint: Not starting maint, server is shutting down'
             return
           end
-          # TODO: Use Concurrent ruby rather than this instance variable
-          mutex = Xolo::Server::Helpers::Maintenance.cleanup_mutex
+
+          mutex = Xolo::Server::Helpers::Maintenance.maint_mutex
 
           if mutex.locked?
-            log_warn 'Cleanup: already running, skipping this run'
+            log_warn 'Maint: already running, skipping this run'
             return
           end
           mutex.lock
-          log_info 'Cleanup: starting'
+          log_info 'Maint: starting'
+
+          # instantiate all the titles once now, rather than in all the task methods
+          title_objects_for_maint = Xolo::Server::Title.all_titles.map { |t| instantiate_title t }
 
-          # add new cleanup tasks/methods here
-          accept_title_editor_eas
-          cleanup_versions
+          # add new maintenance tasks/methods here
+          accept_title_editor_eas title_objects_for_maint
+          auto_release_versions title_objects_for_maint
+          cleanup_versions title_objects_for_maint
+          notify_admins_of_unreleased_pilots title_objects_for_maint
 
-          log_info 'Cleanup: complete'
+          Xolo::Server::Helpers::Maintenance.last_maint = Time.now
+          log_info 'Maint: complete'
         ensure
           mutex&.unlock
         end
@@ -164,38 +200,98 @@ module Xolo
         # and auto accept them if we need to
         # @return [void]
         ######################################
-        def accept_title_editor_eas
+        def accept_title_editor_eas(title_objects)
           unless Xolo::Server.config.jamf_auto_accept_xolo_eas
-            log_info 'Cleanup: The xolo server is not configured to auto-accept Title Editor EAs'
+            log_debug 'Maint: The xolo server is not configured to auto-accept Title Editor EAs'
             return
           end
 
-          log_info 'Cleanup: Looking for Title Editor EAs to auto-accept'
+          log_info 'Maint: Looking for Title Editor EAs to auto-accept'
 
-          # TODO: Be DRY with this stuff and similar in title_jamf_access.rb
-          Xolo::Server::Title.all_titles.each do |title|
-            title_obj = instantiate_title title
+          title_objects.each do |title_obj|
+            title = title_obj.title
             next if title_obj.subscribed?
             next unless title_obj.jamf_patch_ea_awaiting_acceptance?
 
-            log_info "Cleanup: Auto-accepting Title Editor EA for title '#{title}'"
+            log_info "Maint: Auto-accepting Title Editor EA for title '#{title}'"
             title_obj.accept_jamf_patch_ea_via_api
           rescue => e
-            log_error "Cleanup: Error auto-accepting Title Editor EA for title '#{title}': #{e}"
+            log_error "Maint: Error auto-accepting Title Editor EA for title '#{title}': #{e}"
           end # Xolo::Server::Title.all_titles.each
 
-          log_info 'Cleanup: Done with Title Editor EAs to auto-accept'
+          log_info 'Maint: Done with Title Editor EAs to auto-accept'
+        end
+
+        # Do any pending auto-releases
+        # @return [void]
+        ##############################
+        def auto_release_versions(title_objects)
+          log_info 'Maint: Auto-releasing appropriate versions'
+
+          today = Date.today
+
+          # Loop thru the titles
+          title_objects.each do |title_obj|
+            title = title_obj.title
+
+            # nothing to do if its nil or 'none'
+            next if title_obj.auto_release_delay.nil? || title_obj.auto_release_delay == Xolo::NONE
+
+            # title must be subscribed and autopkg
+            # (enforced in xadm)
+            next unless title_obj.subscribed? && !title_obj.autopkg_recipe.pix_empty?
+
+            # title must have a non-negative integer for auto_release_delay,
+            # (number of days to wait before release)
+            # could be zero. (enforced in xadm)
+            # NOTE: it is stored as a string because it might be 'none'
+            # TODO: store none as nil, so we don't have to do the pix_integer check??
+            delay = title_obj.auto_release_delay.to_i
+            next unless title_obj.auto_release_delay&.pix_integer? && !delay.negative?
+
+            # Any version with this creation date should be released now, skip if not
+            # So if the auto_release_delay is 7, its today - 7 days.
+            # if the auto_release_delay is 0, its today - 0 days, aka today
+            creation_date_to_be_released_today = today - delay
+
+            # Find the newest pilot version whose creation date is
+            # creation_date_to_be_released_today or earlier
+            # That one should be released now.
+            vobj_to_release = nil
+
+            # version objects are in order of newest to oldest
+            title_obj.version_objects.each do |vobj|
+              next unless vobj.pilot?
+              next unless vobj.creation_date.to_date <= creation_date_to_be_released_today
+
+              vobj_to_release = vobj
+              break
+            end
+
+            # no versions to release today
+            next unless vobj_to_release
+
+            # release it
+            log_debug "Maint: About to auto-release version '#{vobj_to_release.version}' of '#{title}' which came out #{vobj_to_release.creation_date}"
+
+            # releasing a version is done via the title obj, since it affects the status
+            # of all versions.
+            title_obj.release vobj_to_release.version
+
+            log_info "Maint: Auto-released version '#{vobj_to_release.version}' of '#{title}' which came out #{vobj_to_release.creation_date}", alert: true
+          end # each title
+
+          log_debug 'Maint: Done auto-releasing'
         end
 
         # Cleanup versions.
         # @return [void]
         ################################
-        def cleanup_versions
-          log_info 'Cleanup: cleaning up deprecated and skipped versions'
-
-          Xolo::Server::Title.all_titles.each do |title|
-            title_obj = instantiate_title title
+        def cleanup_versions(title_objects)
+          log_info 'Maint: Cleaning up deprecated and skipped versions'
 
+          # Loop thru the titles
+          title_objects.each do |title_obj|
             title_obj.version_objects.each do |version|
               if version.deprecated?
                 cleanup_deprecated_version version
@@ -203,12 +299,8 @@ module Xolo
                 cleanup_skipped_version version
               end # case
             end # each version
-
-            notify_admins_of_unreleased_pilots(title_obj)
           end # each title
-
-          Xolo::Server::Helpers::Maintenance.last_cleanup = Time.now
-          log_info 'Cleanup: versions cleanup complete'
+          log_info 'Maint: Version cleanup complete'
         end
 
         # Cleanup a deprecated version.
@@ -223,7 +315,7 @@ module Xolo
           days_deprecated = (Time.now - version.deprecation_date) / 86_400
           return unless days_deprecated > deprecated_lifetime_days
 
-          log_info "Cleanup: Deleting deprecated version '#{version.version}' of title '#{version.title}'"
+          log_info "Maint: Deleting deprecated version '#{version.version}' of title '#{version.title}'"
           version.delete
         end
 
@@ -234,38 +326,50 @@ module Xolo
         def cleanup_skipped_version(version)
           return if Xolo::Server.config.keep_skipped_versions
 
-          log_info "Cleanup: Deleting skipped version '#{version.version}' of title '#{version.title}'"
+          log_info "Maint: Deleting skipped version '#{version.version}' of title '#{version.title}'"
           version.delete
         end
 
         # Notify the admins about unreleased pilots if needed
         # @return [void]
         ################################
-        def notify_admins_of_unreleased_pilots(title_obj)
-          return unless Time.now.day == UNRELEASED_PILOTS_NOTIFICATION_DAY
-          return unless unreleased_pilots_notification_days.positive?
-          return unless title_obj.latest_version
+        def notify_admins_of_unreleased_pilots(title_objects)
+          unless Time.now.day == UNRELEASED_PILOTS_NOTIFICATION_DAY
+            log_debug 'Maint: Not notifying admins of old pilots, wrong day of the month.'
+            return
+          end
+          unless unreleased_pilots_notification_days&.positive?
+            log_debug 'Maint: Not notifying admins of old pilots, unreleased_pilots_notification_days is not a positive integer'
+            return
+          end
+
+          log_info 'Maint: Notifying admins about old unreleased pilots'
 
-          latest_vers_obj = instantiate_version title: title_obj, version: title_obj.latest_version
-          return unless latest_vers_obj.pilot?
+          title_objects.each do |title_obj|
+            next unless title_obj.latest_version
 
-          days_in_pilot = ((Time.now - latest_vers_obj.creation_date) / 86_400).to_i
+            latest_vers_obj = instantiate_version title: title_obj, version: title_obj.latest_version
+            next unless latest_vers_obj.pilot?
 
-          return unless days_in_pilot > unreleased_pilots_notification_days
+            days_in_pilot = ((Time.now - latest_vers_obj.creation_date) / 86_400).to_i
 
-          alert_msg = "Cleanup: Notifying #{title_obj.contact_email} about unreleased pilot '#{latest_vers}' of title '#{title_obj.title}', in pilot for #{days_in_pilot} days"
+            next unless days_in_pilot > unreleased_pilots_notification_days
 
-          log_info alert_msg
-          send_alert alert_msg
+            alert_msg = "Maint: Notifying #{title_obj.contact_email} about unreleased pilot '#{latest_vers}' of title '#{title_obj.title}', in pilot for #{days_in_pilot} days"
 
-          email_msg = <<~MSG
+            log_info alert_msg
+            send_alert alert_msg
+
+            email_msg = <<~MSG
             The newest version '#{latest_vers_obj.version}' of title '#{title_obj.title}' has been in pilot for #{days_in_pilot} days, which makes it seem like it's not going to be released.
 
             To reduce clutter, please consider releasing it, deleting it, or deleting the whole title if it's no longer needed.
 
             If this is intentional, you can ignore this monthly message.
-          MSG
-          send_email to: title_obj.contact_email, subject: 'Unreleased Pilot Notification', msg: email_msg
+            MSG
+            send_email to: title_obj.contact_email, subject: 'Unreleased Pilot Notification', msg: email_msg
+          end # each title
+          log_info 'Maint: Done notifying admins about old unreleased pilots'
         end
 
         # how many days can a version be deprecated?
@@ -291,7 +395,7 @@ module Xolo
 
           progress "Server Shutdown by #{session[:admin]}", log: :info
 
-          stop_cleanup_timer_task
+          stop_maint_timer_task
           stop_log_rotation_timer_task
           shutdown_pkg_deletion_pool
           wait_for_object_locks
@@ -316,12 +420,12 @@ module Xolo
           system "/bin/launchctl unload #{SERVER_LAUNCHD_PLIST}"
         end
 
-        # Stop the cleanup timer task
+        # Stop the maintenance timer task
         # @return [void]
         ################################
-        def stop_cleanup_timer_task
-          progress 'Stopping the cleanup timer task', log: :info
-          Xolo::Server::Helpers::Maintenance.cleanup_timer_task.shutdown
+        def stop_maint_timer_task
+          progress 'Stopping the maint timer task', log: :info
+          Xolo::Server::Helpers::Maintenance.maint_timer_task.shutdown
         end
 
         # Stop the log rotation timer task

data/lib/xolo/server/helpers/progress_streaming.rb

@@ -174,16 +174,49 @@ module Xolo
         def stream_progress(stream_file:, stream:)
           log_debug "About to tail: usr/bin/tail -f -c +1 #{Shellwords.escape stream_file.to_s}"
 
-          stdin, stdouterr, wait_thr = Open3.popen2e("/usr/bin/tail -f -c +1 #{Shellwords.escape stream_file.to_s}")
-          stdin.close
-
-          while line = stdouterr.gets
-            break if line.chomp == Xolo::Server::PROGRESS_COMPLETE
-
-            stream << line
-          end
-          stdouterr.close
-          wait_thr.exit
+          begin
+            File.open(stream_file, 'r') do |file|
+              # Go straight to the end of the file if you only want new lines
+              file.seek(0, IO::SEEK_END)
+
+              heartbeat_counter = 0
+              loop do
+                # Check if the stream was closed by the user
+                break if stream.closed?
+
+                # Try to read the next line
+                # This won't block but will return nil if no line is ready
+                if line = file.gets
+                  break if line.chomp == Xolo::Server::PROGRESS_COMPLETE
+
+                  # Send the line to the browser immediately
+                  stream << line
+                else
+                  # If no new line, sleep for a fraction of a second
+                  # This saves your computer's CPU from working too hard
+                  sleep 0.1
+                  heartbeat_counter += 1
+                  if heartbeat_counter == 100 # every 10 secs
+                    stream << ':heartbeat'
+                    heartbeat_counter = 0
+                  end
+                end # if
+              end # loop
+            end # file open
+          rescue IOError, Errno::ECONNRESET
+            nil
+          end # begin
+
+          # stdin, stdouterr, wait_thr = Open3.popen2e("/usr/bin/tail -f -c +1 #{Shellwords.escape stream_file.to_s}")
+          # stdin.close
+
+          # while line = stdouterr.gets
+          #   break if line.chomp == Xolo::Server::PROGRESS_COMPLETE
+
+          #   stream << line
+          # end
+          # stdouterr.close
+          # wait_thr.exit
           # TODO: deal with wait_thr.value.exitstatus > 0 ?
         end