web47core 2.0.1 → 2.2.15

data/lib/app/jobs/cron/restart_orphaned_delayed_jobs.rb

@@ -0,0 +1,44 @@
+#
+# Run in cron
+#
+module Cron
+  #
+  # Cycle through all members and tell them to sync with with switchboard
+  #
+  class RestartOrphanedDelayedJobs < Job
+    cron_tab_entry :hourly
+
+    #
+    # Only run when we have background jobs and we have the time keeper plugin installed
+    #
+    def self.valid_environment?
+      Delayed::Worker.plugins.include?(Delayed::Plugins::TimeKeeper) &&
+        SystemConfiguration.delayed_job_restart_orphaned? &&
+        Delayed::Backend::Mongoid::Job.count.positive?
+    rescue StandardError
+      false
+    end
+
+    #
+    # Cycle through delayed jobs, looking for running jobs
+    # skip if we don't have any records or low sample set
+    # skip if the allowed time is less than allowed by the job
+    # look to see if have a worker associated with the delayed job
+    # If we dont have a worker or the worker is dead, restart the job
+    #
+    def execute
+      Delayed::Backend::Mongoid::Job.each do |delayed_job|
+        next unless delayed_job.running?
+
+        metric = Delayed::Jobs::Metric.where(name: delayed_job.display_name).first
+        next if metric.blank? || metric.count < 30 # not enough data to make a call
+
+        run_time = Time.now.utc - delayed_job.locked_at
+        next if run_time < metric.max_allowed_seconds # still within parameters
+
+        worker = delayed_job.worker
+        delayed_job.resubmit if worker.blank? || worker.dead?
+      end
+    end
+  end
+end

data/lib/app/jobs/cron/server.rb

@@ -6,6 +6,7 @@ module Cron
   #
   class Server
     include StandardModel
+    include ServerProcessAble
     #
     # Constants
     #
@@ -15,20 +16,19 @@ module Cron
     #
     # Fields
     #
-    field :host_name, type: String
-    field :pid, type: Integer
     field :desired_server_count, type: Integer, default: 0
     field :current_server_count, type: Integer, default: 0
-    field :last_check_in_at, type: Time, default: Time.now.utc
     field :state, type: String, default: STATE_SECONDARY
     #
     # Validations
     #
-    validates :host_name, presence: true
-    validates :pid, presence: true
-    validates :last_check_in_at, presence: true
     validates :state, inclusion: { in: ALL_STATES }
     validate :high_lander
+    #
+    # Callback
+    #
+    before_validation :ensure_last_check_in
+
     #
     # Go through the logic once a minute
     #
@@ -89,7 +89,7 @@ module Cron
     # Warm up a server on the next evaluation
     #
     def self.warm_up_server
-      return unless SystemConfiguration.aws_auto_scaling_configured?
+      return unless auto_scaling_configured?
 
       primary_server.auto_scale([primary_server.desired_server_count + 1, 10].min)
     end
@@ -148,18 +148,11 @@ module Cron
       !alive?
     end
 
-    #
-    # Perform a check in for the server
-    #
-    def check_in
-      set({ last_check_in_at: Time.now.utc })
-    end
-
     #
     # Auto scale environment
     #
     def check_auto_scale
-      return unless SystemConfiguration.aws_auto_scaling_configured?
+      return unless auto_scaling_configured?
 
       if delayed_jobs_count.eql?(0)
         handle_zero_job_count
@@ -181,6 +174,24 @@ module Cron
       @sys_config ||= SystemConfiguration.configuration
     end
 
+    #
+    # Test if autoscaling is configured, return false if there is an error
+    #
+    def auto_scaling_configured?
+      @auto_scaling_configured ||= sys_config.aws_auto_scaling_configured?
+    rescue StandardError
+      false
+    end
+
+    #
+    # Test if autoscaling is configured, return false if there is an error
+    #
+    def self.auto_scaling_configured?
+      SystemConfiguration.aws_auto_scaling_configured?
+    rescue StandardError
+      false
+    end
+
     #
     # Returns the AutoScalingGroup associated with the account
     #
@@ -258,6 +269,8 @@ module Cron
       client.update_auto_scaling_group(auto_scaling_group_name: sys_config.aws_auto_scaling_group_name,
                                        min_size: desired_count,
                                        desired_capacity: desired_count)
+    rescue StandardError => error
+      App47Logger.log_error "Unable to set auto scaler to #{desired_count}", error
     end
 
     #
@@ -283,5 +296,9 @@ module Cron
     def inactive_count
       desired_server_count
     end
+
+    def ensure_last_check_in
+      self.last_check_in_at ||= Time.now.utc
+    end
   end
 end

data/lib/app/jobs/cron/switchboard_sync_configuration.rb

@@ -13,6 +13,8 @@ module Cron
     #
     def self.valid_environment?
       SystemConfiguration.switchboard_configured?
+    rescue StandardError
+      false
     end
 
     #

data/lib/app/jobs/cron/switchboard_sync_models.rb

@@ -13,6 +13,8 @@ module Cron
     #
     def self.valid_environment?
       SystemConfiguration.switchboard_configured? && Web47core::Config.switchboard_able_models.present?
+    rescue StandardError
+      false
     end
 
     #

data/lib/app/jobs/cron/trim_collection.rb

@@ -57,7 +57,7 @@ module Cron
     # Allowed time the amount of time allowed to exists before deleting
     #
     def allowed_time
-      30.days.ago
+      30.days.ago.utc
     end
   end
 end

data/lib/app/jobs/cron/trim_delayed_job_metrics.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Cron
+  #
+  # Clean up Audit Logs that have not been updated in 90 days
+  #
+  class TrimDelayedJobMetrics < TrimCollection
+    #
+    # Fetch each Audit Log and delete it if hasn't been updated in 90 days
+    #
+    def collection
+      Delayed::Jobs::Metric.all
+    end
+
+    #
+    # Return which field to use for comparison when trimming objects
+    #
+    def comparison_field
+      :last_run_at
+    end
+
+    #
+    # Allowed time the amount of time allowed to exists before deleting
+    #
+    def allowed_time
+      12.months.ago.utc
+    end
+  end
+end

data/lib/app/jobs/cron/trim_delayed_job_workers.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Cron
+  #
+  # Clean up Audit Logs that have not been updated in 90 days
+  #
+  class TrimDelayedJobWorkers < TrimCollection
+    #
+    # Fetch each Audit Log and delete it if hasn't been updated in 90 days
+    #
+    def collection
+      Delayed::Jobs::Worker.all
+    end
+
+    #
+    # Check if the cron job server hasn't reported in a while
+    #
+    def archive?(worker)
+      super && worker.runs.blank?
+    rescue StandardError => error
+      App47Logger.log_warn "Unable to archive item #{worker.inspect}", error
+      false
+    end
+
+    #
+    # Return which field to use for comparison when trimming objects
+    #
+    def comparison_field
+      :last_check_in_at
+    end
+
+    #
+    # Allowed time the amount of time allowed to exists before deleting
+    #
+    def allowed_time
+      2.days.ago.utc
+    end
+  end
+end

data/lib/app/jobs/cron/trim_failed_delayed_jobs.rb

@@ -35,6 +35,8 @@ module Cron
     #
     def self.valid_environment?
       SystemConfiguration.slack_api_url.present?
+    rescue StandardError
+      false
     end
   end
 end

data/lib/app/models/api_token.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+#
+# A specific API token to be used as part of a collection of tokens
+#
+class ApiToken
+  include StandardModel
+  include ApiTokenable
+end

data/lib/app/models/command_job.rb

@@ -110,7 +110,7 @@ class CommandJob
   end
 
   def failure_or_cancelled?
-    job_state?([STATE_FAIL, STATE_CANCELLED], true)
+    job_state?([STATE_FAIL, STATE_CANCELLED], default_state: true)
   end
 
   #
@@ -118,11 +118,11 @@ class CommandJob
   # state. If there is a match, then return true, otherwise return false.
   # If there is an error, return the default.
   #
-  def job_state?(states, default = false)
+  def job_state?(states, default_state: false)
     states.is_a?(Array) ? states.include?(state) : states.eql?(state)
   rescue StandardError => error
-    App47Logger.log_warn "Unable to check job failed or cancelled #{self.inspect}", error
-    default
+    App47Logger.log_warn "Unable to check job failed or cancelled #{inspect}", error
+    default_state
   end
 
   #
@@ -323,8 +323,9 @@ class CommandJob
     else
       logs.create!(dir: dir, command: command, message: output)
     end
-    check_for_text(output, options[:error_texts], true)
-    check_for_text(output, options[:required_texts], false)
+    options[:output_limit] ||= -1
+    check_for_text(output, options[:error_texts], output_limit: options[:output_limit])
+    check_for_text(output, options[:required_texts], inclusive_check: false, output_limit: options[:output_limit])
     output
   end
 
@@ -343,16 +344,17 @@ class CommandJob
 
   #
   # Check if any occurrences were found (or not found)
+  # For most command jobs, we want to see the full output. -1 accomplishes this
   #
-  def check_for_text(output, texts = [], inclusive = true)
+  def check_for_text(output, texts = [], inclusive_check: true, output_limit: -1)
     return if texts.blank?
 
     texts = [texts] if texts.is_a?(String)
     texts.each do |text|
-      if inclusive
-        raise "Error: found text (#{text}) - #{output}" if output.match?(/#{text}/)
+      if inclusive_check
+        raise "Error: found text (#{text}) - #{output[0...output_limit]}" if output.match?(/#{text}/)
       else
-        raise "Error: missing text (#{text}) - #{output}" unless output.match?(/#{text}/)
+        raise "Error: missing text (#{text}) - #{output[0...output_limit]}" unless output.match?(/#{text}/)
       end
     end
   end
@@ -370,5 +372,4 @@ class CommandJob
   def sort_fields
     %i[created_at]
   end
-
 end

data/lib/app/models/concerns/api_tokenable.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+#
+# API tokenable, support for the api token in user, but also may be applied elsewhere
+#
+module ApiTokenable
+  extend ActiveSupport::Concern
+
+  def self.included(base)
+    base.class_eval do
+      # store api token
+      field :api_token, type: String
+      # if the api token should be reset
+      field :reset_api_token, type: Boolean, default: true
+      field :last_authenticated_at, type: Time
+      field :last_authenticated_ip, type: String
+      # call back to reset the api token.
+      before_save :assign_api_token, if: :reset_api_token
+      # set the index on api token
+      index({ api_token: 1 }, background: true)
+
+      def cycle_api_token
+        self.set api_token: SecureRandom.urlsafe_base64
+        api_token
+      end
+
+      private
+
+      #
+      # set the api token
+      #
+      def assign_api_token
+        self.reset_api_token = false
+        self.api_token = SecureRandom.urlsafe_base64
+      end
+    end
+  end
+end

data/lib/app/models/concerns/aws_configuration.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+#
+# AWS Configuration
+#
+module AwsConfiguration
+  extend ActiveSupport::Concern
+
+  def self.included(base)
+    base.class_eval do
+      #
+      # Fields
+      #
+      field :aws_region, type: String
+      field :aws_access_key_id, type: String
+      field :aws_secret_access_key, type: String
+      field :aws_auto_scaling_group_name, type: String
+    end
+  end
+
+  #
+  # Make sure the password doesn't get blanked out on an update
+  #
+  def secure_fields
+    super + %i[aws_secret_access_key]
+  end
+
+  #
+  # Determine if AWS is configured
+  #
+  def aws_configured?
+    [aws_region.present?, aws_access_key_id.present?, aws_secret_access_key.present?].all?
+  end
+
+  #
+  # Determine if auto scaling group is configured
+  #
+  def aws_auto_scaling_configured?
+    aws_configured? && aws_auto_scaling_group_name.present?
+  end
+
+  #
+  # AWS client.
+  #
+  def aws_ec2_client
+    return nil unless aws_configured?
+
+    @aws_ec2_client ||= Aws::EC2::Client.new(region: aws_region,
+                                             credentials: Aws::Credentials.new(aws_access_key_id,
+                                                                               aws_secret_access_key))
+  end
+
+  #
+  # S3 Client
+  #
+  def aws_s3_client
+    return nil unless aws_configured?
+
+    # We want this to remake itself each time because it is possible that the
+    # => user would change the access keys in between actions. Huh?
+    @aws_s3_client ||= Aws::S3::Client.new(region: aws_vault_bucket_region,
+                                           access_key_id: aws_access_key_id,
+                                           secret_access_key: aws_secret_access_key)
+  end
+end

data/lib/app/models/concerns/cdn_url.rb

@@ -17,6 +17,13 @@
 #
 module CdnUrl
   extend ActiveSupport::Concern
+  #
+  # Constants
+  #
+  STYLE_S3_FILE_PATH = ':class/:attachment/:id/:style.:extension' unless defined? STYLE_S3_FILE_PATH
+  STYLE_FILE_PATH = 'public/system/:class/:attachment/:id/:style.:extension' unless defined? STYLE_FILE_PATH
+  STYLE_S3_FILE_URL = ':s3_domain_url' unless defined? STYLE_S3_FILE_URL
+  STYLE_FILE_URL = ':rails_root/public/system/:class/:attachment/:id/:style.:extension' unless defined? STYLE_FILE_URL
 
   def method_missing(method, *args)
     if method.to_s.start_with? 'cdn_'

data/lib/app/models/concerns/core_smtp_configuration.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+#
+# SMTP Configuration
+#
+module CoreSmtpConfiguration
+  extend ActiveSupport::Concern
+
+  def self.included(base)
+    base.class_eval do
+      #
+      # Fields
+      #
+      field :default_email, type: String, default: 'support@app47.com'
+      field :support_email, type: String, default: 'support@app47.com'
+      field :smtp_name, type: String
+      field :smtp_address, type: String
+      field :smtp_domain, type: String
+      field :smtp_port, type: Integer, default: 587
+      field :smtp_user_name, type: String
+      field :smtp_password, type: String
+      field :smtp_enable_starttls_auto, type: Boolean, default: false
+      field :mailgun_api_key, type: String
+      field :email_notification_ttl, type: Integer, default: 180
+    end
+    base.extend SmtpClassMethods
+  end
+
+  #
+  # Class methods for smtp configuration
+  #
+  module SmtpClassMethods
+    def smtp_configuration
+      output = {}
+      config = configuration
+      fields = %w[name address domain port user_name password enable_starttls_auto]
+      fields.each do |field|
+        field_name = "smtp_#{field}".to_sym
+        output[field.to_sym] = config.send(field_name)
+      end
+      output
+    end
+  end
+
+  #
+  # Make sure the password doesn't get blanked out on an update
+  #
+  def secure_fields
+    super + %i[smtp_password mailgun_api_key]
+  end
+
+  #
+  # Determine if SMTP is configured
+  #
+  def smtp_configured?
+    smtp_name.present? && smtp_address.present? && smtp_domain.present?
+  end
+
+  #
+  # Determine if mailgun is configured
+  #
+  def mail_gun_configured?
+    smtp_configured? && mailgun_api_key.present?
+  end
+end