web47core 0.0.10 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97cb8fc1a923a3c6a911a319404529ef67ad9de4d1f20d6c90b088f99d8e366a
-  data.tar.gz: 53ce4e9da8a4279b4bc4eb3ffb7a0f58b9ec1844c6f1f42adeed9d3e572f3f7e
+  metadata.gz: b3949ac1edae0d03528d687d4eb39024813fc8fa98325d0407a982800c68ff77
+  data.tar.gz: 250122ca58e430a2a366f3adebba47c7c83e17061a5946690fb8010ccbcc293c
 SHA512:
-  metadata.gz: e3598aefea3e3b1411e4e4c60c1096129ba90fc5d33ae2c0907762d223ce234f3a4c56e50d28b2dc218a82544a51e9003bc20e0bf4f68bff70d0e767667887ad
-  data.tar.gz: 773f67eac21317c9da0c286c1071383f864b42f4a1d36dcef624bf3968dedf40d96452f9a3eb252057fb805dd6ce66c4302746e72838a4fcd856d30c23e95563
+  metadata.gz: 74c4529df3ec5931a3b7e1d9f53123253dca0beddf26612db997e08d5ce1950897e3306cf5d10ff56d0be1ee581e077915d9d7a8b141765fcce018b5ee4b856b
+  data.tar.gz: 4b9b67895fba934193b72badc6a4da20d7dce717dd53169152a0445a581278564a027ea42e4b56c3a1f0b97417160e090632ba0f4aabe6a52159c0d26221ed06

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    web47core (0.0.10)
+    web47core (0.1.0)
       activesupport (~> 5.0)
       aws-sdk-ec2 (> 1.140, <= 1.160)
+      daemons
       delayed_job_mongoid (~> 2.3)
       email_format
       haml
@@ -92,6 +93,7 @@ GEM
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     crass (1.0.6)
+    daemons (1.3.1)
     database_cleaner (1.8.3)
     delayed_job (4.1.8)
       activesupport (>= 3.0, < 6.1)

data/bin/cron_server

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
+require 'cron/command'
+Cron::Worker.logger ||= Logger.new(File.join(Rails.root, 'log', 'cron_server.log'))
+Cron::Command.new(ARGV).daemonize

data/lib/app/jobs/application_job.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+#
+# Base application job that all jobs extend from
+#
+class ApplicationJob < ActiveJob::Base
+  include App47Logger
+
+  #
+  # If this job should run in this current environment, defaults to true
+  #
+  def self.valid_environments
+    []
+  end
+
+  #
+  # Is this job in a valid environment
+  #
+  def self.valid_environment?
+    my_environments = valid_environments
+    (my_environments.empty? || my_environments.include?(Rails.env))
+  end
+end

data/lib/app/jobs/cron/command.rb

@@ -0,0 +1,79 @@
+unless ENV['RAILS_ENV'] == 'test'
+  begin
+    require 'daemons'
+  rescue LoadError
+    raise "You need to add gem 'daemons' to your Gemfile if you wish to use it."
+  end
+end
+require 'fileutils'
+require 'optparse'
+require 'pathname'
+
+module Cron
+  class Command # rubocop:disable ClassLength
+
+    DIR_PWD = Pathname.new Dir.pwd
+
+    def initialize(args) # rubocop:disable MethodLength
+      @options = { pid_dir: "#{root}/tmp/pids", log_dir: "#{root}/log", monitor: false }
+
+      opts = OptionParser.new do |opt|
+        opt.banner = "Usage: #{File.basename($PROGRAM_NAME)} [options] start|stop|restart|run"
+
+        opt.on('-h', '--help', 'Show this message') do
+          puts opt
+          exit 1
+        end
+        opt.on('-e', '--environment=NAME', 'Specifies the environment to run this delayed jobs under (test/development/production).') do |_e|
+          STDERR.puts 'The -e/--environment option has been deprecated and has no effect. Use RAILS_ENV and see http://github.com/collectiveidea/delayed_job/issues/7'
+        end
+        opt.on('--pid-dir=DIR', 'Specifies an alternate directory in which to store the process ids.') do |dir|
+          @options[:pid_dir] = dir
+        end
+        opt.on('--log-dir=DIR', 'Specifies an alternate directory in which to store the delayed_job log.') do |dir|
+          @options[:log_dir] = dir
+        end
+        opt.on('-m', '--monitor', 'Start monitor process.') do
+          @options[:monitor] = true
+        end
+        opt.on('--exit-on-complete', 'Exit when no more jobs are available to run. This will exit if all jobs are scheduled to run in the future.') do
+          @options[:exit_on_complete] = true
+        end
+        opt.on('--daemon-options a, b, c', Array, 'options to be passed through to daemons gem') do |daemon_options|
+          @daemon_options = daemon_options
+        end
+      end
+      @args = opts.parse!(args) + (@daemon_options || [])
+    end
+
+    def daemonize # rubocop:disable PerceivedComplexity
+      dir = @options[:pid_dir]
+      FileUtils.mkdir_p(dir) unless File.exist?(dir)
+
+      Cron::JobTab.ensure_cron_tabs
+      Cron::Worker.logger ||= Logger.new(File.join(@options[:log_dir], 'cron.log'))
+      Daemons.run_proc('cron_server',
+                       dir: options[:pid_dir],
+                       dir_mode: :normal,
+                       monitor: @options[:monitor],
+                       ARGV: @args) do
+        run @options
+      end
+    end
+
+    def run(options = {})
+      Dir.chdir(root)
+
+      Cron::Worker.new(options).start
+    rescue StandardError => error
+      App47Logger.log_error 'Unable to start Cron Server', error
+      exit 1
+    end
+
+    private
+
+    def root
+      @root ||= defined?(::Rails.root) ? ::Rails.root : DIR_PWD
+    end
+  end
+end

data/lib/app/jobs/cron/job.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Cron
+  #
+  # Base class for WeeklyJobs.
+  #
+  # Support running only on certain days as defined by day_of_week method
+  #
+  class Job < ApplicationJob
+    cattr_accessor :cron_tab
+    #
+    # Method to set the cron_tab
+    #
+    def self.cron_tab_entry(entry)
+      self.cron_tab = entry
+    end
+
+    #
+    # Sends support an email if something goes awry
+    #
+    def send_support_email(error, event)
+      email = EmailNotification.new
+      email.to = 'support@app47.com'
+      params = { current_date: I18n.l(Time.zone.today, format: :medium),
+                 event: event,
+                 error: error }
+      email.from_template('support_ticket_notification', params)
+      email.send_notification
+    end
+  end
+end

data/lib/app/jobs/cron/job_tab.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+#
+# Value object for a cron tab entry
+#
+module Cron
+  class JobTab < Tab
+    unless defined? FRAMEWORK_CLASSES
+      FRAMEWORK_CLASSES = %w[job trim_collection command server tab job_tab worker].freeze
+    end
+    cattr_accessor :jobs
+    #
+    # Validations
+    #
+    validates :name, presence: true, uniqueness: true
+    validate :valid_name
+
+    class << self
+      #
+      # Pull form system configuration
+      #
+      def from_string(name, value)
+        tab = JobTab.find_or_initialize_by name: name
+        return unless tab.valid?
+
+        tab.update_from_string(value)
+        tab
+      end
+
+      def ensure_cron_tabs
+        self.jobs = []
+        Cron.constants.each do |job|
+          job_name = job.to_s.underscore
+          next if FRAMEWORK_CLASSES.include?(job_name) ||
+            job_name.end_with?('_test') ||
+            job_name.start_with?('base_')
+
+          jobs << job_name
+          klass = "cron/#{job_name}".camelize.constantize
+          tab = JobTab.find_or_initialize_by name: job_name
+          next if tab.persisted?
+
+          configure_cron_tab(tab, klass.cron_tab)
+        end
+        purge_cron_tabs
+      end
+
+
+      private
+
+      def configure_cron_tab(tab, type)
+        case type
+        when :always
+          tab.update! min: WILDCARD, hour: WILDCARD
+        when :hourly
+          tab.update! min: 0, hour: WILDCARD
+        when :daily
+          tab.update! min: 0, hour: 0
+        when :weekly
+          tab.update! min: 0, hour: 0, wday: 0
+        when :monthly
+          tab.update! min: 0, hour: 0, mday: 0
+        else
+          tab.update_from_string(type)
+          tab.save!
+        end
+      end
+
+      def purge_cron_tabs
+        JobTab.all.each do |tab|
+          tab.destroy unless jobs.include?(tab.name)
+        end
+      end
+    end
+
+    #
+    # Update our values based on the value
+    #
+    def update_from_string(value)
+      values = value.split(' ')
+      self.min = values[0]
+      self.hour = values[1]
+      self.mday = values[2]
+      self.month = values[3]
+      self.wday = values[4]
+    end
+
+    #
+    # Run this job cron tab
+    #
+    def run
+      return unless valid_environment?
+
+      cron_job_class.perform_later
+      super
+    end
+
+    #
+    # Is this enabled and a valid environment
+    #
+    def valid_environment?
+      enabled? && cron_job_class.valid_environment?
+    end
+
+    #
+    # Return the class associated with this job cron tab
+    #
+    def cron_job_class
+      @cron_job_class ||= "cron/#{name}".camelize.constantize
+    end
+
+    #
+    # Convert back to a standard string value
+    #
+    def to_string
+      [min, hour, mday, month, wday].join(' ')
+    end
+
+    #
+    # Test the name is the list of jobs discovered
+    #
+    def valid_name
+      errors.add(:name, 'Invalid Tab') unless jobs.include?(name)
+    end
+  end
+end

data/lib/app/jobs/cron/server.rb

@@ -0,0 +1,285 @@
+#
+# Handle the coordination of which server should be running the cron jobs
+#
+module Cron
+  class Server
+    include StandardModel
+    #
+    # Constants
+    #
+    STATE_PRIMARY = 'primary'.freeze unless defined? STATE_PRIMARY
+    STATE_SECONDARY = 'secondary'.freeze unless defined? STATE_SECONDARY
+    ALL_STATES = [STATE_PRIMARY, STATE_SECONDARY].freeze unless defined? ALL_STATES
+    #
+    # Fields
+    #
+    field :host_name, type: String
+    field :pid, type: Integer
+    field :desired_server_count, type: Integer, default: 0
+    field :current_server_count, type: Integer, default: 0
+    field :last_check_in_at, type: Time, default: Time.now.utc
+    field :state, type: String, default: STATE_SECONDARY
+    #
+    # Validations
+    #
+    validates :host_name, presence: true
+    validates :pid, presence: true
+    validates :last_check_in_at, presence: true
+    validates :state, inclusion: { in: ALL_STATES }
+    validate :high_lander
+    #
+    # Go through the logic once a minute
+    #
+    def execute
+      if primary?
+        run_cron_jobs
+      else
+        primary = Cron::Server.where(state: STATE_PRIMARY).first
+        if primary.blank? || primary.dead?
+          become_primary
+          run_cron_jobs
+        end
+      end
+      time_to_next_run
+    rescue StandardError => error
+      App47Logger.log_error 'Unable to run cron jobs', error
+      time_to_next_run
+    ensure
+      check_in
+    end
+
+    def run_cron_jobs
+      run_jobs
+      check_auto_scale
+    end
+
+    #
+    # Run all cron tab jobs
+    #
+    def run_jobs
+      now = Time.now.utc
+      CronTab.all.each { |tab| tab.run if tab.time_to_run?(now) }
+    end
+
+    #
+    # Determine the next minute to run,
+    #
+    def time_to_next_run
+      60 - Time.now.utc.to_i % 60
+    end
+
+    #
+    # Find a record for this server
+    #
+    def self.find_or_create_server
+      Cron::Server.find_or_create_by!(host_name: Socket.gethostname, pid: Process.pid)
+    end
+
+    #
+    # Find a the current master
+    #
+    def self.primary_server
+      Cron::Server.where(state: STATE_PRIMARY).first
+    end
+
+    #
+    # Warm up a server on the next evaluation
+    #
+    def self.warm_up_server
+      return unless SystemConfiguration.auto_scaling_configured?
+
+      primary_server.auto_scale([primary_server.desired_server_count + 1, 10].min)
+    end
+
+    #
+    # Become primary, making others secondary
+    #
+    def become_primary
+      Cron::Server.each(&:become_secondary)
+      # sleep a small amount of time to randomize a new primary
+      sleep rand(1..15)
+      # Check to see if another node already became primary
+      primary = Cron::Server.primary_server
+      return if primary.present? && primary.alive?
+
+      # no one else is in, so become primary
+      update_attributes! state: STATE_PRIMARY, last_check_in_at: Time.now.utc
+    end
+
+    #
+    # Become secondary node
+    #
+    def become_secondary(user = nil)
+      if user.present?
+        update_attributes_and_log! user, state: STATE_SECONDARY
+      else
+        update_attributes! state: STATE_SECONDARY
+      end
+    end
+
+    #
+    # Am I the primary server
+    #
+    def primary?
+      alive? && STATE_PRIMARY.eql?(state)
+    end
+
+    #
+    # Am I a secondary server
+    #
+    def secondary?
+      STATE_SECONDARY.eql?(state)
+    end
+
+    #
+    # Return true if I've reported in the last two minutes
+    #
+    def alive?
+      last_check_in_at >= 90.seconds.ago.utc
+    end
+
+    #
+    # Is the server dead, meaning is it not reporting within the last two minutes
+    #
+    def dead?
+      !alive?
+    end
+
+    #
+    # Perform a check in for the server
+    #
+    def check_in
+      set last_check_in_at: Time.now.utc
+    end
+
+    #
+    # Auto scale environment
+    #
+    def check_auto_scale
+      return unless SystemConfiguration.auto_scaling_configured?
+
+      if delayed_jobs_count.eql?(0)
+        handle_zero_job_count
+      else
+        handle_auto_scale_jobs
+      end
+    end
+
+    #
+    # Returns the AWS AutoScaling Client
+    #
+    def client
+      credentials = { access_key_id: sys_config.access_key_id,
+                      secret_access_key: sys_config.secret_access_key,
+                      region: sys_config.region }
+      @client ||= Aws::AutoScaling::Client.new(credentials)
+    end
+
+    def sys_config
+      @sys_config ||= SystemConfiguration.configuration
+    end
+
+    #
+    # Returns the AutoScalingGroup associated with the account
+    #
+    def auto_scaling_group
+      filter = { auto_scaling_group_names: [sys_config.auto_scaling_group_name] }
+      @auto_scaling_group ||= client.describe_auto_scaling_groups(filter).auto_scaling_groups.first
+    end
+
+    #
+    # Returns a count of the Delayed Jobs in queue that have not failed
+    #
+    def delayed_jobs_count
+      @delayed_jobs_count ||= Delayed::Backend::Mongoid::Job.where(failed_at: nil).read(mode: :primary).count
+    end
+
+    #
+    # Returns the current value of 'desired capacity' for the AutoScalingGroup
+    #
+    def current_desired_capacity
+      current = auto_scaling_group.desired_capacity
+      set current_server_count: current
+      current
+    rescue StandardError
+      0
+    end
+
+    #
+    # Calls the 'auto_scale' method with a 'desired_count' of 0 unless the capacity is already at 0
+    #
+    def handle_zero_job_count
+      return if current_desired_capacity.eql?(0)
+
+      auto_scale
+    end
+
+    #
+    # Calls the 'auto_scale' method with a variable 'desired_count' based on how many jobs are running
+    # We don't need any more workers if the job count is less than 1,000
+    #
+    def handle_auto_scale_jobs
+      return if delayed_jobs_count < 50
+
+      case delayed_jobs_count
+      when 50..250
+        auto_scale(1)
+      when 251..500
+        auto_scale(2)
+      when 501..1_000
+        auto_scale(3)
+      when 1_001..2_000
+        auto_scale(4)
+      when 2_001..3_999
+        auto_scale(4)
+      when 4_000..7_999
+        auto_scale(5)
+      when 8_000..10_999
+        auto_scale(5)
+      when 11_000..13_999
+        auto_scale(6)
+      when 14_000..17_999
+        auto_scale(6)
+      else
+        auto_scale(7)
+      end
+    end
+
+    #
+    # Sets the desired and minimum number of EC2 instances to run
+    #
+    def auto_scale(desired_count = 0)
+      set desired_server_count: desired_count
+      # Make sure we don't remove any workers with assigned jobs by accident
+      return if desired_count.positive? && desired_count <= current_desired_capacity
+
+      client.update_auto_scaling_group(auto_scaling_group_name: sys_config.auto_scaling_group_name,
+                                       min_size: desired_count,
+                                       desired_capacity: desired_count)
+    end
+
+    #
+    # Look to make sure there is only one primary
+    #
+    def high_lander
+      return if secondary? # Don't need to check if not primary
+
+      primary = Cron::Server.where(state: STATE_PRIMARY).first
+      errors.add(:state, 'there can only be one primary') unless primary.blank? || primary.eql?(self)
+    end
+
+    #
+    # Returns the count of active servers
+    #
+    def active_count
+      current_server_count
+    end
+
+    #
+    # Returns the count of inactive servers
+    #
+    def inactive_count
+      desired_server_count
+    end
+  end
+end

data/lib/app/jobs/cron/switchboard_sync_configuration.rb

@@ -0,0 +1,66 @@
+#
+# Run in cron
+#
+module Cron
+  #
+  # Sync configuration with switchboard
+  #
+  class SwitchboardSyncConfiguration < Job
+    cron_tab_entry :hourly
+
+    #
+    # Only run in environments where switchboard is configured
+    #
+    def self.valid_environment?
+      SystemConfiguration.switchboard_configured?
+    end
+
+    #
+    # Cycle through all configuration keys
+    #
+    def perform
+      Rails.cache.reconnect
+      RestClient.get(switchboard_url,
+                     ACCESS_TOKEN: SystemConfiguration.switchboard_stack_api_token,
+                     content_type: 'application/json') do |response, _request, _result, &block|
+        case response.code
+        when 200
+          json = JSON.parse(response.body)
+          config = SystemConfiguration.configuration
+          json['results'].each { |key, value| update_config(config, key, value) }
+          config.save!
+        else
+          App47Logger.log_error "Unable to fetch switchboard config, #{response.inspect}"
+          response.return!(&block)
+        end
+      end
+    end
+
+    #
+    # First see if it's updateable against system config, then see if it's in JobCronTab
+    #
+    def update_config(config, key, value)
+      config.send("#{key}=", value) if config.respond_to?("#{key}=")
+      return unless key.end_with?('_crontab')
+
+      name = key.chomp('_crontab')
+      tab = Cron::JobTab.from_string(name, value)
+      if tab.present? && tab.valid?
+        tab.save
+        App47Logger.log_debug "Crontab #{name} updated with #{value}"
+      else
+        App47Logger.log_warn "Unable to update crontab #{name} updated with #{value}"
+      end
+    end
+
+    #
+    # Generate the switchboard URL
+    #
+    def switchboard_url
+      [SystemConfiguration.switchboard_base_url,
+       'stacks',
+       SystemConfiguration.switchboard_stack_id,
+       'items.json'].join('/')
+    end
+  end
+end

data/lib/app/jobs/cron/switchboard_sync_models.rb

@@ -0,0 +1,25 @@
+#
+# Run in cron
+#
+module Cron
+  #
+  # Cycle through all members and tell them to sync with with switchboard
+  #
+  class SwitchboardSyncModels <Job
+    cron_tab_entry :daily
+
+    #
+    # Only run in environments where switchboard is configured
+    #
+    def self.valid_environment?
+      SystemConfiguration.switchboard_configured? && Web47core.config.switchboard_able_models.present?
+    end
+
+    #
+    # Cycle through the collection and perform an upsert on it
+    #
+    def perform
+      Web47core.config.switchboard_able_models.each { |model| model.each(&:switchboard_upsert) }
+    end
+  end
+end