web47core 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2eee94f84ba4767b90179b2572ed33e3ba7f60112524fb7b7cd2af8613c42524
-  data.tar.gz: 3ae3b9cedbadabbb98db834d7a4c3fe8e071dd3ccf9f2b18d19f8ec5086fd055
+  metadata.gz: dbf707fe805b12f85070027a7f991404b327d1abf62b633b2b0489740e902e5b
+  data.tar.gz: c0fef2df3709e4319297a1f55c0c0b0968c83a89451f264511e2663f236925f0
 SHA512:
-  metadata.gz: 4cf27f3ef39691a4c1f13c37381bb574bff398895740018fae76215e22532ed8d9f78dd69f05286fa140a15831f580d1c353924f00c01bc5feb4aeb3e14af64e
-  data.tar.gz: a336720edc072530a94a748715f5c37cf179743354636af54a773599892e6a3dc804fe9bc0ceb0274078dc1f9d2a4b7281e9379aa9176a8617c65dedb6629cba
+  metadata.gz: d15d90c313d88d4ad813b98a764508484554b47b790d55b688f1bb156a8b1a0b350ba709fee7be2f26f6db3e83954c461b3f274fedcddb7756c6de9336df7f44
+  data.tar.gz: feeac4239229604c458fb02a292e820f42ff789c361afa9b1a1b958e1057e3b1b99cff995b0bd12b2c6e5621df092f88eed179b3027d02c83d883fd79ed19249

data/lib/app/jobs/cron/trim_command_jobs.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Cron
+  #
+  # Clean up Jobs
+  #
+  class TrimCommandJobs < TrimCollection
+    #
+    # Fetch each Audit Log and delete it if hasn't been updated in 90 days
+    #
+    def collection
+      CommandJob.all
+    end
+
+    #
+    # Check which audit logs we wanted deleted
+    #
+    # Should be older than 90 days and either not a user model audit log or the model associated with
+    # the UserModelAuditLog has been deleted
+    #
+    def allowed_time_for_item(job)
+      job.ttl.days.ago.utc
+    rescue StandardError => error
+      App47Logger.log_warn "Unable to determine if job should be archived: #{job.inspect}", error
+      30.days.ago.utc
+    end
+  end
+end

data/lib/app/models/command_job.rb

@@ -0,0 +1,374 @@
+# frozen_string_literal: true
+
+#
+# Base class for all jobs that will be run on builds
+#
+class CommandJob
+  include StandardModel
+  #
+  # Constants
+  #
+  STATE_NEW = 'new' unless defined? STATE_NEW
+  STATE_WIP = 'working' unless defined? STATE_WIP
+  STATE_RETRYING = 'retrying' unless defined? STATE_RETRYING
+  STATE_SUCCESS = 'success' unless defined? STATE_SUCCESS
+  STATE_FAIL = 'failure' unless defined? STATE_FAIL
+  STATE_CANCELLED = 'cancelled' unless defined? STATE_CANCELLED
+  unless defined? ALL_STATES
+    ALL_STATES = [STATE_NEW, STATE_WIP, STATE_RETRYING, STATE_CANCELLED, STATE_SUCCESS, STATE_FAIL].freeze
+  end
+  #
+  # Fields
+  #
+  field :state, type: String, default: STATE_NEW
+  field :retries, type: Integer, default: 0
+  field :max_retries, type: Integer, default: 5
+  field :result, type: String
+  field :error_message, type: String
+  field :started_at, type: Time
+  field :finished_at, type: Time
+  #
+  # Relationships
+  #
+  has_many :logs, class_name: 'CommandJobLog', dependent: :destroy
+  belongs_to :started_by, class_name: 'User', optional: true
+  #
+  # Validations
+  #
+  validates :state, inclusion: { in: ALL_STATES }
+
+  #
+  # Who started this job
+  #
+  def display_started_by
+    started_by.present? ? started_by.name : 'System'
+  end
+
+  #
+  # Default time to keep a job before auto archiving it
+  #
+  def ttl
+    30
+  end
+
+  #
+  # Return the name of this job
+  #
+  def name
+    self.class.to_s.underscore.humanize
+  end
+
+  #
+  # True if in new status
+  #
+  def new_job?
+    job_state?(STATE_NEW)
+  end
+
+  #
+  # True if in WIP status
+  #
+  def work_in_progress?
+    job_state?([STATE_WIP, STATE_RETRYING])
+  end
+
+  #
+  # True if in success status
+  #
+  def succeeded?
+    job_state?(STATE_SUCCESS)
+  end
+
+  #
+  # True if in fail status
+  #
+  def failure?
+    job_state?(STATE_FAIL)
+  end
+
+  #
+  # If we is finished, failed or success
+  #
+  def completed?
+    job_state?([STATE_CANCELLED, STATE_FAIL, STATE_SUCCESS])
+  end
+
+  #
+  # Job has not finished, failure or success
+  #
+  def running?
+    !completed?
+  end
+
+  alias incomplete? running?
+
+  #
+  # If we are cancelled
+  #
+  def cancelled?
+    job_state?(STATE_CANCELLED)
+  end
+
+  def failure_or_cancelled?
+    job_state?([STATE_FAIL, STATE_CANCELLED], true)
+  end
+
+  #
+  # Fetch the latest version of this instance from the database and check the state against the required
+  # state. If there is a match, then return true, otherwise return false.
+  # If there is an error, return the default.
+  #
+  def job_state?(states, default = false)
+    states.is_a?(Array) ? states.include?(state) : states.eql?(state)
+  rescue StandardError => error
+    App47Logger.log_warn "Unable to check job failed or cancelled #{self.inspect}", error
+    default
+  end
+
+  #
+  # Return the job's status and information in a hash that could be used to return to a calling
+  # api
+  #
+  def current_status
+    status = { state: state }
+    status[:message] = error_message if error_message.present?
+    status
+  end
+
+  #
+  # Perform this job in the background
+  #
+  def perform_later
+    perform
+  end
+
+  handle_asynchronously :perform_later
+
+  #
+  # Steps to execute before a run
+  #
+  def before_run
+    case state
+    when STATE_NEW
+      set retries: 0,
+          started_at: Time.now.utc,
+          finished_at: nil,
+          error_message: nil,
+          result: nil,
+          state: STATE_WIP
+    when STATE_RETRYING
+      set retries: 0,
+          started_at: Time.now.utc,
+          finished_at: nil,
+          error_message: nil,
+          result: nil
+    when STATE_FAIL
+      set retries: 0,
+          started_at: Time.now.utc,
+          finished_at: nil,
+          error_message: nil,
+          state: STATE_RETRYING,
+          result: nil
+    else
+      set retries: 0, started_at: Time.now.utc, finished_at: nil, result: nil
+    end
+  end
+
+  #
+  # Steps to execute after a run
+  #
+  def after_run
+    case state
+    when STATE_RETRYING, STATE_WIP
+      set finished_at: Time.now.utc, error_message: nil, state: STATE_SUCCESS
+    when STATE_SUCCESS
+      set finished_at: Time.now.utc, error_message: nil
+    else
+      set finished_at: Time.now.utc
+    end
+  end
+
+  #
+  #
+  # Perform the command job
+  #
+  def perform
+    before_run
+    run
+    after_run
+  rescue StandardError => error
+    log_error 'Unable to start job', error
+    set state: STATE_FAIL, error_message: error.message
+  end
+
+  alias perform_now perform
+
+  #
+  # Run the job, handling any failures that might happen
+  #
+  def run
+    run! unless cancelled?
+  rescue StandardError => error
+    if (retries + 1) >= max_retries
+      log_error "Unable to run job id: #{id}, done retrying", error
+      set state: STATE_FAIL, error_message: "Failed final attempt: #{error.message}"
+    else
+      log_error "Unable to run job id: #{id}, retrying!!", error
+      add_log "Unable to run job: #{error.message}, retrying!!"
+      set error_message: "Failed attempt # #{retries}: #{error.message}", retries: retries + 1, state: STATE_RETRYING
+      run
+    end
+  end
+
+  #
+  # Determine the correct action to take and get it started
+  #
+  def run!
+    raise 'Incomplete class, concrete implementation should implement #run!'
+  end
+
+  #
+  # Write out the contents to the file
+  #
+  def write_file(path, contents)
+    File.open(path, 'w') { |f| f.write(contents) }
+    add_log "Saving:\n #{contents}\nto: #{path}"
+  end
+
+  #
+  # Download a file to the given path
+  #
+  def download_file(file_url, file_path)
+    download = URI.parse(file_url).open
+    IO.copy_stream(download, file_path)
+    add_log "Downloaded file: #{file_url} to #{file_path}"
+  rescue StandardError => error
+    raise "Unable to download file from #{file_url} to #{file_path}, error: ##{error.message}"
+  end
+
+  #
+  # Copy a given file to a new location and record the log
+  #
+  def copy_file(from_path, to_path)
+    if File.exist? from_path
+      FileUtils.cp(from_path, to_path)
+      add_log "Copy file from: #{from_path} to: #{to_path}"
+    else
+      add_log "File not found: #{from_path}, copy not performed"
+    end
+  rescue StandardError => error
+    raise "Unable to copy file from #{from_path} to #{to_path}, error: ##{error.message}"
+  end
+
+  #
+  # Copy a given directory to a new location and record the log
+  #
+  def copy_dir(dir, to_path)
+    FileUtils.cp_r dir, to_path
+    add_log "Copy directory from: #{dir} to: #{to_path}"
+  end
+
+  #
+  # Remove the given file name
+  #
+  def remove_file(file_path)
+    return unless File.exist?(file_path)
+
+    FileUtils.remove_file file_path
+    add_log "Removing file: #{file_path}"
+  end
+
+  #
+  # Remove the given file name
+  #
+  def remove_dir(dir_path)
+    return unless File.exist?(dir_path)
+
+    FileUtils.remove_dir dir_path
+    add_log "Removing dir: #{dir_path}"
+  end
+
+  #
+  # Create a directory and record it
+  #
+  def mkdir(dir)
+    return if File.exist?(dir)
+
+    FileUtils.mkdir dir
+    add_log "Created directory: #{dir}"
+  end
+
+  alias make_dir mkdir
+
+  #
+  # Unzip a given file
+  #
+  def unzip_file(file_path, to_dir)
+    run_command "unzip #{file_path}", to_dir, error_texts: 'unzip:'
+  end
+
+  #
+  # Run the command capturing the command output and any standard error to the log.
+  #
+  def run_command(command, dir = '/tmp', options = {})
+    command = command.join(' ') if command.is_a?(Array)
+    output = Tempfile.open('run-command-', '/tmp') do |f|
+      Dir.chdir(dir) { `#{command} > #{f.path} 2>&1` }
+      mask_keywords(f.open.read, options[:mask_texts])
+    end
+    output = 'Success' if output.blank?
+    command = mask_keywords(command, options[:mask_texts])
+    if block_given?
+      yield output
+    else
+      logs.create!(dir: dir, command: command, message: output)
+    end
+    check_for_text(output, options[:error_texts], true)
+    check_for_text(output, options[:required_texts], false)
+    output
+  end
+
+  #
+  # Mask keywords if given in the command
+  #
+  def mask_keywords(output, keywords = [])
+    return output if keywords.blank?
+
+    keywords = [keywords] if keywords.is_a?(String)
+    keywords.each do |keyword|
+      output = output.gsub(keyword, '***********')
+    end
+    output
+  end
+
+  #
+  # Check if any occurrences were found (or not found)
+  #
+  def check_for_text(output, texts = [], inclusive = true)
+    return if texts.blank?
+
+    texts = [texts] if texts.is_a?(String)
+    texts.each do |text|
+      if inclusive
+        raise "Error: found text (#{text}) - #{output}" if output.match?(/#{text}/)
+      else
+        raise "Error: missing text (#{text}) - #{output}" unless output.match?(/#{text}/)
+      end
+    end
+  end
+
+  #
+  # Add a job log message
+  #
+  def add_log(message)
+    logs.create!(message: message)
+  end
+
+  #
+  # Which to sort by
+  #
+  def sort_fields
+    %i[created_at]
+  end
+
+end

data/lib/app/models/command_job_log.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+#
+# Capture command log messages for a given job
+#
+class CommandJobLog
+  include StandardModel
+  #
+  # Fields
+  #
+  field :message, type: String
+  field :command, type: String
+  field :dir, type: String
+  #
+  # Relationships
+  #
+  belongs_to :job, inverse_of: :logs, class_name: 'CommandJob'
+  #
+  # Validations
+  #
+  validates :message, presence: true
+
+  #
+  # Display message
+  #
+  def display_message
+    if dir.present?
+      "Dir: #{dir}\nCommand: #{command}\nOutput: #{message}"
+    else
+      message
+    end
+  end
+end

data/lib/web47core.rb

@@ -12,6 +12,8 @@ require 'app/models/concerns/core_system_configuration'
 require 'app/models/concerns/core_account'
 require 'app/models/concerns/secure_fields'
 require 'app/models/concerns/encrypted_password'
+require 'app/models/command_job'
+require 'app/models/command_job_log'
 require 'app/models/delayed_job'
 require 'app/models/redis_configuration'
 require 'app/models/notification'
@@ -38,6 +40,7 @@ require 'app/jobs/cron/switchboard_sync_models'
 require 'app/jobs/cron/trim_audit_logs'
 require 'app/jobs/cron/trim_cron_servers'
 require 'app/jobs/cron/trim_failed_delayed_jobs'
+require 'app/jobs/cron/trim_command_jobs'
 require 'app/jobs/cron/trim_notifications'
 #
 # Audit Logs

data/lib/web47core/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Web47core
-  VERSION = '2.0.0'
+  VERSION = '2.0.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web47core
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Chris Schroeder
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-03 00:00:00.000000000 Z
+date: 2021-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -714,10 +714,13 @@ files:
 - lib/app/jobs/cron/tab.rb
 - lib/app/jobs/cron/trim_audit_logs.rb
 - lib/app/jobs/cron/trim_collection.rb
+- lib/app/jobs/cron/trim_command_jobs.rb
 - lib/app/jobs/cron/trim_cron_servers.rb
 - lib/app/jobs/cron/trim_failed_delayed_jobs.rb
 - lib/app/jobs/cron/trim_notifications.rb
 - lib/app/models/audit_log.rb
+- lib/app/models/command_job.rb
+- lib/app/models/command_job_log.rb
 - lib/app/models/concerns/app47_logger.rb
 - lib/app/models/concerns/cdn_url.rb
 - lib/app/models/concerns/cipher_able.rb