web47core 2.0.0 → 3.0.0

data/lib/app/jobs/cron/restart_orphaned_delayed_jobs.rb

@@ -0,0 +1,44 @@
+#
+# Run in cron
+#
+module Cron
+  #
+  # Cycle through all members and tell them to sync with with switchboard
+  #
+  class RestartOrphanedDelayedJobs < Job
+    cron_tab_entry :hourly
+
+    #
+    # Only run when we have background jobs and we have the time keeper plugin installed
+    #
+    def self.valid_environment?
+      Delayed::Worker.plugins.include?(Delayed::Plugins::TimeKeeper) &&
+        SystemConfiguration.delayed_job_restart_orphaned? &&
+        Delayed::Backend::Mongoid::Job.count.positive?
+    rescue StandardError
+      false
+    end
+
+    #
+    # Cycle through delayed jobs, looking for running jobs
+    # skip if we don't have any records or low sample set
+    # skip if the allowed time is less than allowed by the job
+    # look to see if have a worker associated with the delayed job
+    # If we dont have a worker or the worker is dead, restart the job
+    #
+    def execute
+      Delayed::Backend::Mongoid::Job.each do |delayed_job|
+        next unless delayed_job.running?
+
+        metric = Delayed::Jobs::Metric.where(name: delayed_job.display_name).first
+        next if metric.blank? || metric.count < 30 # not enough data to make a call
+
+        run_time = Time.now.utc - delayed_job.locked_at
+        next if run_time < metric.max_allowed_seconds # still within parameters
+
+        worker = delayed_job.worker
+        delayed_job.resubmit if worker.blank? || worker.dead?
+      end
+    end
+  end
+end

data/lib/app/jobs/cron/server.rb

@@ -6,6 +6,7 @@ module Cron
   #
   class Server
     include StandardModel
+    include ServerProcessAble
     #
     # Constants
     #
@@ -15,20 +16,19 @@ module Cron
     #
     # Fields
     #
-    field :host_name, type: String
-    field :pid, type: Integer
     field :desired_server_count, type: Integer, default: 0
     field :current_server_count, type: Integer, default: 0
-    field :last_check_in_at, type: Time, default: Time.now.utc
     field :state, type: String, default: STATE_SECONDARY
     #
     # Validations
     #
-    validates :host_name, presence: true
-    validates :pid, presence: true
-    validates :last_check_in_at, presence: true
     validates :state, inclusion: { in: ALL_STATES }
     validate :high_lander
+    #
+    # Callback
+    #
+    before_validation :ensure_last_check_in
+
     #
     # Go through the logic once a minute
     #
@@ -89,7 +89,7 @@ module Cron
     # Warm up a server on the next evaluation
     #
     def self.warm_up_server
-      return unless SystemConfiguration.aws_auto_scaling_configured?
+      return unless auto_scaling_configured?
 
       primary_server.auto_scale([primary_server.desired_server_count + 1, 10].min)
     end
@@ -148,18 +148,11 @@ module Cron
       !alive?
     end
 
-    #
-    # Perform a check in for the server
-    #
-    def check_in
-      set({ last_check_in_at: Time.now.utc })
-    end
-
     #
     # Auto scale environment
     #
     def check_auto_scale
-      return unless SystemConfiguration.aws_auto_scaling_configured?
+      return unless auto_scaling_configured?
 
       if delayed_jobs_count.eql?(0)
         handle_zero_job_count
@@ -181,6 +174,24 @@ module Cron
       @sys_config ||= SystemConfiguration.configuration
     end
 
+    #
+    # Test if autoscaling is configured, return false if there is an error
+    #
+    def auto_scaling_configured?
+      @auto_scaling_configured ||= sys_config.aws_auto_scaling_configured?
+    rescue StandardError
+      false
+    end
+
+    #
+    # Test if autoscaling is configured, return false if there is an error
+    #
+    def self.auto_scaling_configured?
+      SystemConfiguration.aws_auto_scaling_configured?
+    rescue StandardError
+      false
+    end
+
     #
     # Returns the AutoScalingGroup associated with the account
     #
@@ -258,6 +269,8 @@ module Cron
       client.update_auto_scaling_group(auto_scaling_group_name: sys_config.aws_auto_scaling_group_name,
                                        min_size: desired_count,
                                        desired_capacity: desired_count)
+    rescue StandardError => error
+      App47Logger.log_error "Unable to set auto scaler to #{desired_count}", error
     end
 
     #
@@ -283,5 +296,9 @@ module Cron
     def inactive_count
       desired_server_count
     end
+
+    def ensure_last_check_in
+      self.last_check_in_at ||= Time.now.utc
+    end
   end
 end

data/lib/app/jobs/cron/switchboard_sync_configuration.rb

@@ -13,6 +13,8 @@ module Cron
     #
     def self.valid_environment?
       SystemConfiguration.switchboard_configured?
+    rescue StandardError
+      false
     end
 
     #

data/lib/app/jobs/cron/switchboard_sync_models.rb

@@ -13,6 +13,8 @@ module Cron
     #
     def self.valid_environment?
       SystemConfiguration.switchboard_configured? && Web47core::Config.switchboard_able_models.present?
+    rescue StandardError
+      false
     end
 
     #

data/lib/app/jobs/cron/tab.rb

@@ -18,7 +18,7 @@ module Cron
     # Fields
     #
     field :name, type: String
-    field :enabled, type: Boolean, default: true
+    field :enabled, type: Mongoid::Boolean, default: true
     field :min, type: String, default: 0
     field :hour, type: String, default: 0
     field :wday, type: String, default: WILDCARD

data/lib/app/jobs/cron/trim_collection.rb

@@ -57,7 +57,7 @@ module Cron
     # Allowed time the amount of time allowed to exists before deleting
     #
     def allowed_time
-      30.days.ago
+      30.days.ago.utc
     end
   end
 end

data/lib/app/jobs/cron/trim_command_jobs.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Cron
+  #
+  # Clean up Jobs
+  #
+  class TrimCommandJobs < TrimCollection
+    #
+    # Fetch each Audit Log and delete it if hasn't been updated in 90 days
+    #
+    def collection
+      CommandJob.all
+    end
+
+    #
+    # Check which audit logs we wanted deleted
+    #
+    # Should be older than 90 days and either not a user model audit log or the model associated with
+    # the UserModelAuditLog has been deleted
+    #
+    def allowed_time_for_item(job)
+      job.ttl.days.ago.utc
+    rescue StandardError => error
+      App47Logger.log_warn "Unable to determine if job should be archived: #{job.inspect}", error
+      30.days.ago.utc
+    end
+  end
+end

data/lib/app/jobs/cron/trim_delayed_job_metrics.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Cron
+  #
+  # Clean up Audit Logs that have not been updated in 90 days
+  #
+  class TrimDelayedJobMetrics < TrimCollection
+    #
+    # Fetch each Audit Log and delete it if hasn't been updated in 90 days
+    #
+    def collection
+      Delayed::Jobs::Metric.all
+    end
+
+    #
+    # Return which field to use for comparison when trimming objects
+    #
+    def comparison_field
+      :last_run_at
+    end
+
+    #
+    # Allowed time the amount of time allowed to exists before deleting
+    #
+    def allowed_time
+      12.months.ago.utc
+    end
+  end
+end

data/lib/app/jobs/cron/trim_delayed_job_workers.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Cron
+  #
+  # Clean up Audit Logs that have not been updated in 90 days
+  #
+  class TrimDelayedJobWorkers < TrimCollection
+    #
+    # Fetch each Audit Log and delete it if hasn't been updated in 90 days
+    #
+    def collection
+      Delayed::Jobs::Worker.all
+    end
+
+    #
+    # Check if the cron job server hasn't reported in a while
+    #
+    def archive?(worker)
+      super && worker.runs.blank?
+    rescue StandardError => error
+      App47Logger.log_warn "Unable to archive item #{worker.inspect}", error
+      false
+    end
+
+    #
+    # Return which field to use for comparison when trimming objects
+    #
+    def comparison_field
+      :last_check_in_at
+    end
+
+    #
+    # Allowed time the amount of time allowed to exists before deleting
+    #
+    def allowed_time
+      2.days.ago.utc
+    end
+  end
+end

data/lib/app/jobs/cron/trim_failed_delayed_jobs.rb

@@ -35,6 +35,8 @@ module Cron
     #
     def self.valid_environment?
       SystemConfiguration.slack_api_url.present?
+    rescue StandardError
+      false
     end
   end
 end

data/lib/app/models/api_token.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+#
+# A specific API token to be used as part of a collection of tokens
+#
+class ApiToken
+  include StandardModel
+  include ApiTokenable
+end

data/lib/app/models/command_job.rb

@@ -0,0 +1,375 @@
+# frozen_string_literal: true
+
+#
+# Base class for all jobs that will be run on builds
+#
+class CommandJob
+  include StandardModel
+  #
+  # Constants
+  #
+  STATE_NEW = 'new' unless defined? STATE_NEW
+  STATE_WIP = 'working' unless defined? STATE_WIP
+  STATE_RETRYING = 'retrying' unless defined? STATE_RETRYING
+  STATE_SUCCESS = 'success' unless defined? STATE_SUCCESS
+  STATE_FAIL = 'failure' unless defined? STATE_FAIL
+  STATE_CANCELLED = 'cancelled' unless defined? STATE_CANCELLED
+  unless defined? ALL_STATES
+    ALL_STATES = [STATE_NEW, STATE_WIP, STATE_RETRYING, STATE_CANCELLED, STATE_SUCCESS, STATE_FAIL].freeze
+  end
+  #
+  # Fields
+  #
+  field :state, type: String, default: STATE_NEW
+  field :retries, type: Integer, default: 0
+  field :max_retries, type: Integer, default: 5
+  field :result, type: String
+  field :error_message, type: String
+  field :started_at, type: Time
+  field :finished_at, type: Time
+  #
+  # Relationships
+  #
+  has_many :logs, class_name: 'CommandJobLog', dependent: :destroy
+  belongs_to :started_by, class_name: 'User', optional: true
+  #
+  # Validations
+  #
+  validates :state, inclusion: { in: ALL_STATES }
+
+  #
+  # Who started this job
+  #
+  def display_started_by
+    started_by.present? ? started_by.name : 'System'
+  end
+
+  #
+  # Default time to keep a job before auto archiving it
+  #
+  def ttl
+    30
+  end
+
+  #
+  # Return the name of this job
+  #
+  def name
+    self.class.to_s.underscore.humanize
+  end
+
+  #
+  # True if in new status
+  #
+  def new_job?
+    job_state?(STATE_NEW)
+  end
+
+  #
+  # True if in WIP status
+  #
+  def work_in_progress?
+    job_state?([STATE_WIP, STATE_RETRYING])
+  end
+
+  #
+  # True if in success status
+  #
+  def succeeded?
+    job_state?(STATE_SUCCESS)
+  end
+
+  #
+  # True if in fail status
+  #
+  def failure?
+    job_state?(STATE_FAIL)
+  end
+
+  #
+  # If we is finished, failed or success
+  #
+  def completed?
+    job_state?([STATE_CANCELLED, STATE_FAIL, STATE_SUCCESS])
+  end
+
+  #
+  # Job has not finished, failure or success
+  #
+  def running?
+    !completed?
+  end
+
+  alias incomplete? running?
+
+  #
+  # If we are cancelled
+  #
+  def cancelled?
+    job_state?(STATE_CANCELLED)
+  end
+
+  def failure_or_cancelled?
+    job_state?([STATE_FAIL, STATE_CANCELLED], default_state: true)
+  end
+
+  #
+  # Fetch the latest version of this instance from the database and check the state against the required
+  # state. If there is a match, then return true, otherwise return false.
+  # If there is an error, return the default.
+  #
+  def job_state?(states, default_state: false)
+    states.is_a?(Array) ? states.include?(state) : states.eql?(state)
+  rescue StandardError => error
+    App47Logger.log_warn "Unable to check job failed or cancelled #{inspect}", error
+    default_state
+  end
+
+  #
+  # Return the job's status and information in a hash that could be used to return to a calling
+  # api
+  #
+  def current_status
+    status = { state: state }
+    status[:message] = error_message if error_message.present?
+    status
+  end
+
+  #
+  # Perform this job in the background
+  #
+  def perform_later
+    perform
+  end
+
+  handle_asynchronously :perform_later
+
+  #
+  # Steps to execute before a run
+  #
+  def before_run
+    case state
+    when STATE_NEW
+      set retries: 0,
+          started_at: Time.now.utc,
+          finished_at: nil,
+          error_message: nil,
+          result: nil,
+          state: STATE_WIP
+    when STATE_RETRYING
+      set retries: 0,
+          started_at: Time.now.utc,
+          finished_at: nil,
+          error_message: nil,
+          result: nil
+    when STATE_FAIL
+      set retries: 0,
+          started_at: Time.now.utc,
+          finished_at: nil,
+          error_message: nil,
+          state: STATE_RETRYING,
+          result: nil
+    else
+      set retries: 0, started_at: Time.now.utc, finished_at: nil, result: nil
+    end
+  end
+
+  #
+  # Steps to execute after a run
+  #
+  def after_run
+    case state
+    when STATE_RETRYING, STATE_WIP
+      set finished_at: Time.now.utc, error_message: nil, state: STATE_SUCCESS
+    when STATE_SUCCESS
+      set finished_at: Time.now.utc, error_message: nil
+    else
+      set finished_at: Time.now.utc
+    end
+  end
+
+  #
+  #
+  # Perform the command job
+  #
+  def perform
+    before_run
+    run
+    after_run
+  rescue StandardError => error
+    log_error 'Unable to start job', error
+    set state: STATE_FAIL, error_message: error.message
+  end
+
+  alias perform_now perform
+
+  #
+  # Run the job, handling any failures that might happen
+  #
+  def run
+    run! unless cancelled?
+  rescue StandardError => error
+    if (retries + 1) >= max_retries
+      log_error "Unable to run job id: #{id}, done retrying", error
+      set state: STATE_FAIL, error_message: "Failed final attempt: #{error.message}"
+    else
+      log_error "Unable to run job id: #{id}, retrying!!", error
+      add_log "Unable to run job: #{error.message}, retrying!!"
+      set error_message: "Failed attempt # #{retries}: #{error.message}", retries: retries + 1, state: STATE_RETRYING
+      run
+    end
+  end
+
+  #
+  # Determine the correct action to take and get it started
+  #
+  def run!
+    raise 'Incomplete class, concrete implementation should implement #run!'
+  end
+
+  #
+  # Write out the contents to the file
+  #
+  def write_file(path, contents)
+    File.open(path, 'w') { |f| f.write(contents) }
+    add_log "Saving:\n #{contents}\nto: #{path}"
+  end
+
+  #
+  # Download a file to the given path
+  #
+  def download_file(file_url, file_path)
+    download = URI.parse(file_url).open
+    IO.copy_stream(download, file_path)
+    add_log "Downloaded file: #{file_url} to #{file_path}"
+  rescue StandardError => error
+    raise "Unable to download file from #{file_url} to #{file_path}, error: ##{error.message}"
+  end
+
+  #
+  # Copy a given file to a new location and record the log
+  #
+  def copy_file(from_path, to_path)
+    if File.exist? from_path
+      FileUtils.cp(from_path, to_path)
+      add_log "Copy file from: #{from_path} to: #{to_path}"
+    else
+      add_log "File not found: #{from_path}, copy not performed"
+    end
+  rescue StandardError => error
+    raise "Unable to copy file from #{from_path} to #{to_path}, error: ##{error.message}"
+  end
+
+  #
+  # Copy a given directory to a new location and record the log
+  #
+  def copy_dir(dir, to_path)
+    FileUtils.cp_r dir, to_path
+    add_log "Copy directory from: #{dir} to: #{to_path}"
+  end
+
+  #
+  # Remove the given file name
+  #
+  def remove_file(file_path)
+    return unless File.exist?(file_path)
+
+    FileUtils.remove_file file_path
+    add_log "Removing file: #{file_path}"
+  end
+
+  #
+  # Remove the given file name
+  #
+  def remove_dir(dir_path)
+    return unless File.exist?(dir_path)
+
+    FileUtils.remove_dir dir_path
+    add_log "Removing dir: #{dir_path}"
+  end
+
+  #
+  # Create a directory and record it
+  #
+  def mkdir(dir)
+    return if File.exist?(dir)
+
+    FileUtils.mkdir dir
+    add_log "Created directory: #{dir}"
+  end
+
+  alias make_dir mkdir
+
+  #
+  # Unzip a given file
+  #
+  def unzip_file(file_path, to_dir)
+    run_command "unzip #{file_path}", to_dir, error_texts: 'unzip:'
+  end
+
+  #
+  # Run the command capturing the command output and any standard error to the log.
+  #
+  def run_command(command, dir = '/tmp', options = {})
+    command = command.join(' ') if command.is_a?(Array)
+    output = Tempfile.open('run-command-', '/tmp') do |f|
+      Dir.chdir(dir) { `#{command} > #{f.path} 2>&1` }
+      mask_keywords(f.open.read, options[:mask_texts])
+    end
+    output = 'Success' if output.blank?
+    command = mask_keywords(command, options[:mask_texts])
+    if block_given?
+      yield output
+    else
+      logs.create!(dir: dir, command: command, message: output)
+    end
+    options[:output_limit] ||= -1
+    check_for_text(output, options[:error_texts], output_limit: options[:output_limit])
+    check_for_text(output, options[:required_texts], inclusive_check: false, output_limit: options[:output_limit])
+    output
+  end
+
+  #
+  # Mask keywords if given in the command
+  #
+  def mask_keywords(output, keywords = [])
+    return output if keywords.blank?
+
+    keywords = [keywords] if keywords.is_a?(String)
+    keywords.each do |keyword|
+      output = output.gsub(keyword, '***********')
+    end
+    output
+  end
+
+  #
+  # Check if any occurrences were found (or not found)
+  # For most command jobs, we want to see the full output. -1 accomplishes this
+  #
+  def check_for_text(output, texts = [], inclusive_check: true, output_limit: -1)
+    return if texts.blank?
+
+    texts = [texts] if texts.is_a?(String)
+    texts.each do |text|
+      if inclusive_check
+        raise "Error: found text (#{text}) - #{output[0...output_limit]}" if output.match?(/#{text}/)
+      else
+        raise "Error: missing text (#{text}) - #{output[0...output_limit]}" unless output.match?(/#{text}/)
+      end
+    end
+  end
+
+  #
+  # Add a job log message
+  #
+  def add_log(message)
+    logs.create!(message: message)
+  end
+
+  #
+  # Which to sort by
+  #
+  def sort_fields
+    %i[created_at]
+  end
+end