web47core 0.0.10 → 0.1.0

data/lib/app/jobs/cron/tab.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+#
+# Value object for a cron tab entry
+#
+module Cron
+  class Tab
+    include StandardModel
+    include SearchAble
+    #
+    # Constants
+    #
+    WILDCARD = '*' unless defined? WILDCARD
+    TIME_UNITS = %i[min hour wday mday month].freeze unless defined? TIME_UNITS
+    #
+    # Fields
+    #
+    field :name, type: String
+    field :enabled, type: Boolean, default: true
+    field :min, type: String, default: 0
+    field :hour, type: String, default: 0
+    field :wday, type: String, default: WILDCARD
+    field :mday, type: String, default: WILDCARD
+    field :month, type: String, default: WILDCARD
+    field :last_run_at, type: Time
+    #
+    # Validations
+    #
+    validates :name, presence: true, uniqueness: true
+    validate :valid_values
+
+    #
+    # The method might look a bit obtuse, but basically we want to compare the time values for
+    #  * Minute
+    #  * Hour
+    #  * Day of Week
+    #  * Day of Month
+    #  * Month
+    #
+    def time_to_run?(time)
+      enabled? && TIME_UNITS.collect { |unit| valid_time?(time, unit, send(unit)) }.all?
+    end
+
+    #
+    # For completion of class, but must be implemented by child class
+    #
+    def run
+      set last_run_at: Time.now.utc
+    end
+
+    private
+
+    #
+    # Check that all values are within the range
+    #
+    def valid_values
+      valid_range :min, 0..59
+      valid_range :hour, 0..23
+      valid_range :month, 0..11
+      valid_range :wday, 0..6
+      valid_range :mday, 0..30
+    end
+
+    def valid_range(field, range)
+      value = send(field)
+      valid = case value
+              when WILDCARD
+                true
+              when Integer
+                range.include?(value)
+              else
+                if value.include?('/')
+                  (numerator, divisor) = value.split('/')
+                  range.include?(divisor.to_i) && numerator.eql?(WILDCARD)
+                elsif value.include?(',')
+                  options = value.split(',')
+                  options.collect { |o| range.include?(o.to_i) }.all?
+                else
+                  range.include?(value.to_i)
+                end
+              end
+      errors.add(field, "Invalid value, allowed range: #{range}") unless valid
+    end
+
+    #
+    # Test if the target value matches the time unit or the wild card, or the comma separated list
+    # 0 - matches the zero value
+    # * - Wildcard, any value matches
+    # */15 - matches any value where dividing by 15 is even, or the modulus is zero
+    # 5,10,15 - matches on 5, 10 and 15 values
+    #
+    def valid_time?(time, unit, target)
+      case target
+      when WILDCARD
+        true
+      when Integer
+        time.send(unit).eql?(target)
+      else
+        if target.include?('/')
+          divisor = target.split('/').last.to_i
+          (time.send(unit) % divisor).zero?
+        elsif target.include?(',')
+          options = target.split(',')
+          options.collect { |o| time.send(unit.eql?(o.to_i)) }.any?
+        else
+          time.send(unit).eql?(target.to_i)
+        end
+      end
+    end
+  end
+end

data/lib/app/jobs/cron/trim_collection.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Cron
+  #
+  # Clean up the collection items that have not been updated in 30 days
+  #
+  class TrimCollection < Job
+    cron_tab_entry :daily
+    #
+    # Fetch each item and delete it if hasn't been updated in 30 days
+    #
+    def perform
+      # Rails.cache.reconnect
+      count = 0
+      total = collection.count
+      while count <= total
+        collection.limit(250).skip(count).each { |item| item.destroy if archive?(item) }
+        count += 250
+      end
+    end
+
+    #
+    # Test if this should be archived
+    #
+    def archive?(item)
+      item.updated_at < allowed_time
+    end
+
+    #
+    # Allowed time the amount of time allowed to exists before deleting
+    #
+    def allowed_time
+      30.days.ago
+    end
+  end
+end

data/lib/app/jobs/cron/trim_cron_servers.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Cron
+  #
+  # Clean up Audit Logs that have not been updated in 90 days
+  #
+  class TrimCronServers < TrimCollection
+    #
+    # Fetch each Audit Log and delete it if hasn't been updated in 90 days
+    #
+    def collection
+      Cron::Server.all
+    end
+
+    #
+    # Check if the cron job server hasn't reported in a while
+    #
+    def archive?(item)
+      item.dead?
+    end
+  end
+end

data/lib/app/jobs/cron/trim_failed_delayed_jobs.rb

@@ -0,0 +1,40 @@
+module Cron
+  #
+  # Automatically update Product Version Lifecycle State to EOL If it is past it's EOL date
+  # Runs Daily
+  #
+  class TrimFailedDelayedJobs < TrimCollection
+    #
+    # Return the failed collection
+    #
+    def collection
+      Delayed::Backend::Mongoid::Job.where(:failed_at.exists => true)
+    end
+
+    def archive?(job)
+      if 'Delayed::PerformableMethod'.eql?(job.name)
+
+        data = { object: job.failed_object.inspect,
+                 method: job.failed_method_name,
+                 args: job.failed_args,
+                 failed_at: job.failed_at,
+                 attempts: job.attempts,
+                 locked_by: job.locked_by }
+        SlackNotification.say(data, template: :failed_delayed_job)
+        true
+      else
+        false
+      end
+    rescue StandardError => error
+      log_error "Unable to determine if we should archive job: #{job}", error
+      false
+    end
+
+    #
+    # If the slack API is not present, don't delete jobs or we wont know about it.
+    #
+    def self.valid_environment?
+      SystemConfiguration.slack_api_url.present? && Delayed::Backend::Mongoid::Job.count.positive?
+    end
+  end
+end

data/lib/app/jobs/cron/trim_notifications.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Cron
+  #
+  # Trim notifications in system after 30 days.
+  #
+  # This was historically done by a database index, however with the introduction
+  # of adding an email attachment, and thus another collection, this needed to move
+  # to a daily job here so that:
+  # 1. the object hierarchy is deleted
+  # 2. The paperclip call backs are fired to remove the file from S3
+  #
+  # In case you are wondering why not make it an embedded class and allow the index
+  # method to still work, the answer is that it won't clean up the S3 files.
+  #
+  class TrimNotifications < TrimCollection
+    #
+    # Return the collection
+    #
+    def collection
+      Notification.all
+    end
+  end
+end

data/lib/app/jobs/cron/worker.rb

@@ -0,0 +1,70 @@
+require 'timeout'
+require 'active_support/dependencies'
+require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/class/attribute_accessors'
+require 'active_support/hash_with_indifferent_access'
+require 'active_support/core_ext/hash/indifferent_access'
+require 'logger'
+require 'benchmark'
+
+module Cron
+  class Worker # rubocop:disable ClassLength
+    DEFAULT_LOG_LEVEL = 'info'.freeze
+
+    cattr_accessor :logger, :default_log_level
+
+    def self.reset
+      self.default_log_level ||= DEFAULT_LOG_LEVEL
+    end
+
+    def self.reload_app?
+      defined?(ActionDispatch::Reloader) && Rails.application.config.cache_classes == false
+    end
+
+    def initialize(options = {})
+      super()
+      @exit = options[:exit_on_complete].presence
+    end
+
+    # Every worker has a unique name which by default is the pid of the process. There are some
+    # advantages to overriding this with something which survives worker restarts:  Workers can
+    # safely resume working on tasks which are locked by themselves. The worker will assume that
+    # it crashed before.
+    def name
+      @name ||= "CronServer:#{Socket.gethostname} pid:#{Process.pid}"
+    end
+
+    def start # rubocop:disable CyclomaticComplexity, PerceivedComplexity
+      trap('TERM') { stop }
+      trap('INT') { stop }
+
+      say 'Starting cron jobs server'
+
+      self.class.lifecycle.run_callbacks(:execute, self) do
+        loop do
+          sleep Cron::Server.find_or_create_server.execute
+          break if stop?
+        end
+      end
+
+      say 'Exiting cron jobs server...'
+    end
+
+    def stop
+      @exit = true
+    end
+
+    def stop?
+      !!@exit
+    end
+
+    def say(text, level = default_log_level)
+      text = "[CronServer(#{name})] #{text}"
+      return if logger.blank?
+
+      logger.send(level, "#{Time.now.strftime('%FT%T%z')}: #{text}")
+    end
+  end
+end
+
+Cron::Worker.reset

data/lib/app/models/concerns/switchboard_able.rb

@@ -0,0 +1,130 @@
+#
+# Objects that can be synced to switchboard. They must implement the following methods
+#  * switchboard_name - the name of the object in switchboard land, apps, members, etc..
+#  * switchboard_payload - the payload to send to switchboard when updating or deleting
+#
+module SwitchboardAble
+  extend ActiveSupport::Concern
+
+  def self.included(base)
+    base.class_eval do
+      field :switchboard_id, type: String
+      after_create :switchboard_upsert
+      # after_update :switchboard_upsert
+      before_destroy :switchboard_delete
+    end
+  end
+
+  #
+  # Return the name in switchboard, must be implemented by the class
+  #
+  def switchboard_name
+    raise 'Method (switchboard_name) must be implemented by the concrete class'
+  end
+
+  #
+  # Return the payload in switchboard, must be implemented by the class
+  #
+  def switchboard_payload
+    raise 'Method (switchboard_payload) must be implemented by the concrete class'
+  end
+
+  #
+  # Generic handler to determine if we should insert or update this object
+  #
+  def switchboard_upsert
+    return unless community_access?
+
+    switchboard_id.present? ? switchboard_update : switchboard_insert
+  end
+
+  handle_asynchronously :switchboard_upsert
+
+  private
+
+  #
+  # Return false if this object should not uploaded to switchboard, true otherwise
+  #
+  def community_access?
+    true
+  end
+
+  #
+  # Delete the object from switchboard
+  #
+  def switchboard_delete
+    return unless SystemConfiguration.switchboard_configured? && switchboard_id.present?
+
+    RestClient.delete(switchboard_url, ACCESS_TOKEN: sw_api_token) do |response, _request, _result, &block|
+      case response.code
+      when 200
+        App47Logger.log_debug "Switchboard deleted #{inspect}"
+      else
+        App47Logger.log_error "Unable to delete the switchboard object #{inspect}, #{response.inspect}"
+        response.return!(&block)
+      end
+    end
+  end
+
+  #
+  # Update the object in switchboard
+  #
+  def switchboard_update
+    return unless SystemConfiguration.switchboard_configured? && switchboard_id.present?
+
+    RestClient.put(switchboard_url,
+                   switchboard_payload.to_json,
+                   ACCESS_TOKEN: sw_api_token,
+                   content_type: 'application/json') do |response, _request, _result, &block|
+      case response.code
+      when 200
+        App47Logger.log_debug "Switchboard updated #{inspect}"
+      when 404, 302
+        unset :switchboard_id
+      else
+        App47Logger.log_error "Unable to update the switchboard object #{inspect}, #{response.inspect}"
+        response.return!(&block)
+      end
+    end
+  end
+
+  #
+  # Insert the object into switchboard
+  #
+  def switchboard_insert
+    return unless SystemConfiguration.switchboard_configured?
+
+    RestClient.post(switchboard_url,
+                    switchboard_payload.to_json,
+                    ACCESS_TOKEN: sw_api_token,
+                    content_type: 'application/json') do |response, _request, _result, &block|
+      case response.code
+      when 200
+        json = JSON.parse(response.body)
+        set switchboard_id: json['results'][switchboard_name.singularize]['id']
+      else
+        App47Logger.log_error "Unable to insert to switchboard object #{inspect}, #{response.inspect}"
+        response.return!(&block)
+      end
+    end
+  end
+
+  #
+  # The URL for this switchboard item
+  #
+  def switchboard_url
+    components = [SystemConfiguration.switchboard_base_url,
+                  'stacks',
+                  SystemConfiguration.switchboard_stack_id,
+                  switchboard_name]
+    components << switchboard_id if switchboard_id.present?
+    components.join('/') + '.json'
+  end
+
+  #
+  # Return the api token
+  #
+  def sw_api_token
+    @sw_api_token ||= SystemConfiguration.switchboard_stack_api_token
+  end
+end

data/lib/app/models/delayed_job.rb

@@ -0,0 +1,80 @@
+module Delayed
+  module Backend
+    module Mongoid
+      #
+      # Extend the Mongoid delayed job to add additional methods
+      #
+      class Job
+        #
+        # Helper to describe the status of the job
+        #
+        def status_description
+          return 'Failed' if failed?
+
+          locked_by.present? ? locked_by : "Scheduled (#{attempts})"
+        end
+
+        #
+        # Display name for the job
+        #
+        def display_name
+          payload_object.job_data['job_class']
+        rescue StandardError
+          name
+        end
+
+        #
+        # Safely get the payload object
+        #
+        def job_payload
+          payload_object.inspect
+        rescue StandardError => error
+          "Unable to retrieve payload: #{error.message}"
+        end
+
+        #
+        # Resubmit the job for processing
+        #
+        def resubmit
+          set locked_at: nil, locked_by: nil, failed_at: nil, last_error: nil, run_at: Time.now.utc
+        end
+
+        #
+        # Return the failed object out of the YAML
+        #
+        def failed_object
+          failed_yaml.object
+        rescue StandardError => error
+          "Unknown object: #{error.message}"
+        end
+
+        #
+        # Return the failed method out of the YAML
+        #
+        def failed_method_name
+          failed_yaml.method_name
+        rescue StandardError => error
+          "Unknown method_name: #{error.message}"
+        end
+
+        #
+        # Return the failed arguments out of the YAML
+        #
+        def failed_args
+          failed_yaml.args
+        rescue StandardError => error
+          "Unknown args: #{error.message}"
+        end
+
+        #
+        # Return the failed YAML
+        #
+        def failed_yaml
+          @failed_yaml = YAML.load(handler)
+        rescue StandardError
+          ''
+        end
+      end
+    end
+  end
+end

data/lib/templates/slack/failed_delayed_job.liquid

@@ -0,0 +1,7 @@
+*Delayed Jobs Failed* due to missing document, ~removing from queue!~
+*Object:* ```{{object}}```
+*Method:* `{{method_name}}`
+*Arguments:* `{{args}}`
+*Failed At:* `{{failed_at}}`
+*Attempts:* `{{attempts}}`
+*Last Run By:* `{{locked_by}}`

data/lib/web47core.rb

@@ -6,8 +6,10 @@ require 'app/models/concerns/time_zone_able'
 require 'app/models/concerns/standard_model'
 require 'app/models/concerns/core_system_configuration'
 require 'app/models/concerns/core_account'
+require 'app/models/delayed_job'
 require 'app/models/redis_configuration'
 require 'app/models/notification'
+require 'app/models/sms_notification'
 require 'app/models/template'
 require 'app/models/notification_template'
 require 'app/models/email_notification'
@@ -15,3 +17,33 @@ require 'app/models/email_template'
 require 'app/models/slack_notification'
 require 'app/models/smtp_configuration'
 require 'app/models/sms_notification'
+#
+# Cron
+#
+require 'app/jobs/application_job'
+require 'app/jobs/cron/job'
+require 'app/jobs/cron/tab'
+require 'app/jobs/cron/job_tab'
+require 'app/jobs/cron/command'
+require 'app/jobs/cron/worker'
+require 'app/jobs/cron/server'
+require 'app/jobs/cron/trim_collection'
+require 'app/jobs/cron/switchboard_sync_configuration'
+require 'app/jobs/cron/switchboard_sync_models'
+require 'app/jobs/cron/trim_notifications'
+require 'app/jobs/cron/trim_cron_servers'
+require 'app/jobs/cron/trim_failed_delayed_jobs'
+
+class Web47core
+  include Singleton
+  attr_accessor :email_able_models, :switchboard_able_models
+
+  def initialize
+    @email_able_models = []
+    @switchboard_able_models = []
+  end
+
+  def self.config
+    instance
+  end
+end

data/test/jobs/cron/switchboard_sync_configuration_test.rb

@@ -0,0 +1,64 @@
+require 'test_helper'
+
+module Cron
+  class SwitchboardSyncConfigurationTest < ActiveSupport::TestCase
+    setup do
+      @config = SystemConfiguration.configuration
+    end
+
+    context 'valid_environment?' do
+      should 'not run' do
+        refute Cron::SwitchboardSyncConfiguration.valid_environment?
+      end
+
+      should 'run' do
+        @config.switchboard_stack_api_token = 'abc123'
+        @config.switchboard_stack_id = 'abc123'
+        assert @config.save
+        assert Cron::SwitchboardSyncConfiguration.valid_environment?
+      end
+    end
+
+    context 'update system configuration' do
+      setup do
+        @config.switchboard_base_url = 'https://switchboard.test.com'
+        @config.switchboard_stack_api_token = 'abc124w'
+        @config.switchboard_stack_id = 'sid'
+        assert @config.save
+      end
+
+      should 'update system configuration' do
+        stub = stub_request(:get, "#{SystemConfiguration.switchboard_base_url}/stacks/sid/items.json").to_return(status: 200, body: { results: { cdn_url: 'https://cdn.nowhere.com' } }.to_json)
+        Cron::SwitchboardSyncConfiguration.new.perform
+        assert_not_nil @config.reload
+        assert_equal 'https://cdn.nowhere.com', @config.cdn_url
+        assert_requested stub
+      end
+
+      should 'update crontab' do
+        App47Logger.expects(:log_debug).once
+        Cron::JobTab.ensure_cron_tabs
+        tab = Cron::JobTab.find_by name: 'switchboard_sync_models'
+        assert_equal '0 0 * * *', tab.to_string
+        stub = stub_request(:get, "#{SystemConfiguration.switchboard_base_url}/stacks/sid/items.json").to_return(status: 200, body: { results: { switchboard_sync_models_crontab: '1 2 3 4 5' } }.to_json)
+        Cron::SwitchboardSyncConfiguration.new.perform
+        assert_not_nil tab.reload
+        assert_equal '1 2 3 4 5', tab.to_string
+        assert_requested stub
+      end
+      should 'deal with unknown' do
+        App47Logger.expects(:log_debug).never
+        App47Logger.expects(:log_warn).once
+        Cron::JobTab.ensure_cron_tabs
+        tab = Cron::JobTab.find_by name: 'switchboard_sync_models'
+        assert_equal '0 0 * * *', tab.to_string
+        stub = stub_request(:get, "#{SystemConfiguration.switchboard_base_url}/stacks/sid/items.json").to_return(status: 200, body: { results: { sync_knowledge_base_job_crontab: '1 2 3 4 5' } }.to_json)
+        Cron::SwitchboardSyncConfiguration.new.perform
+        assert_nil Cron::JobTab.where(name: 'sync_knowledge_base_job_crontab').first
+        assert_not_nil tab.reload
+        assert_equal '0 0 * * *', tab.to_string
+        assert_requested stub
+      end
+    end
+  end
+end

data/test/jobs/cron/trim_cron_servers_test.rb

@@ -0,0 +1,28 @@
+require 'test_helper'
+
+module Cron
+  class TrimCronServersTest < ActiveSupport::TestCase
+    context 'Execute job not deleting servers' do
+      should 'not throw exceptions if no servers ' do
+        assert_nothing_raised { Cron::TrimCronServers.perform_now }
+      end
+      should 'not delete any servers due to last updated' do
+        10.times.each { |n| Cron::Server.create!(host_name: "ip-#{n}", pid: n.to_s).set(last_check_in_at: Time.now.utc) }
+        assert_equal 10, Cron::Server.all.count
+        assert_no_difference 'Cron::Server.all.count' do
+          assert_nothing_raised { Cron::TrimCronServers.perform_now }
+        end
+      end
+    end
+    context 'Execute jobs deleting servers' do
+      should 'delete five server logs' do
+        5.times.each { |n| Cron::Server.create!(host_name: "ip-#{n}", pid: n.to_s).set(last_check_in_at: Time.now.utc) }
+        5.times.each { |n| Cron::Server.create!(host_name: "ip-#{n}", pid: (100+n).to_s).set(last_check_in_at: 1.hour.ago.utc) }
+        assert_equal 10, Cron::Server.all.count
+        assert_difference 'Cron::Server.all.count', -5 do
+          assert_nothing_raised { Cron::TrimCronServers.perform_now }
+        end
+      end
+    end
+  end
+end