web47core 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3949ac1edae0d03528d687d4eb39024813fc8fa98325d0407a982800c68ff77
-  data.tar.gz: 250122ca58e430a2a366f3adebba47c7c83e17061a5946690fb8010ccbcc293c
+  metadata.gz: b81b89188cf055248f7ddfdad9f4690db1b5933006de7af6050143ecd0cc5ce2
+  data.tar.gz: b02ca0f7865c5c8a6f0b39979dd7afd7ff5f750552c69bd7798c71e9bfcb6342
 SHA512:
-  metadata.gz: 74c4529df3ec5931a3b7e1d9f53123253dca0beddf26612db997e08d5ce1950897e3306cf5d10ff56d0be1ee581e077915d9d7a8b141765fcce018b5ee4b856b
-  data.tar.gz: 4b9b67895fba934193b72badc6a4da20d7dce717dd53169152a0445a581278564a027ea42e4b56c3a1f0b97417160e090632ba0f4aabe6a52159c0d26221ed06
+  metadata.gz: 8c5f6bf9c0d5e689e07fc9cb399abc878cd7ee997d90a02d62e7ca8f27aed12468e416a1e7d24f41b7a499e131a04bfd3654f753700d8e61edfb6c5de704995b
+  data.tar.gz: 33d4911d88b3b3dd2e37934d8ad9fcb367c02b2f866f0f0a0ab121a8a2fa99472bc4e7b882137de7345d008a4b1377f5ccc56aeea64e016b76ba350c5539629a

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    web47core (0.1.0)
+    web47core (0.1.1)
       activesupport (~> 5.0)
-      aws-sdk-ec2 (> 1.140, <= 1.160)
+      aws-sdk-autoscaling
       daemons
       delayed_job_mongoid (~> 2.3)
       email_format
@@ -67,15 +67,15 @@ GEM
     ansi (1.5.0)
     arel (9.0.0)
     aws-eventstream (1.0.3)
-    aws-partitions (1.289.0)
+    aws-partitions (1.290.0)
+    aws-sdk-autoscaling (1.33.0)
+      aws-sdk-core (~> 3, >= 3.71.0)
+      aws-sigv4 (~> 1.1)
     aws-sdk-core (3.92.0)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-ec2 (1.151.0)
-      aws-sdk-core (~> 3, >= 3.71.0)
-      aws-sigv4 (~> 1.1)
     aws-sigv4 (1.1.1)
       aws-eventstream (~> 1.0, >= 1.0.2)
     bson (4.8.2)

data/README.md

@@ -78,6 +78,14 @@ _Please do not ship to production code using the git repo, as the production ser
 9.  `SlackNotification` 
 10.  `SmsNotification`
 11.  `Template` 
+12.  `App47Logger` and `App47LoggerTest`
+13.  `Emailable` or `EmailAble` and `EamailbleTest`, be sure to change `Emailable` to `EmailAble`
+14.  `RedisConfiguration` and `RedisConfgurationTest`
+15.  `Searchable` and `SearchableTest`, be sure to change `Searchable` to `SearchAble`
+15.  `TimezoneAble` and `TimezoneAbleTest`
+15.  `CronJobServer` and `CronJobServerTest`
+15.  `JobCronTab` and `JobCronTabTest`
+15.  `CronTab` 
 
 #### Models
 ##### Concerns 
@@ -108,3 +116,27 @@ class Account
   has_many :users
 end
 ```
+##### Cron
+Cron infrastructure for running jobs every minute, hour, day, week, month or some combination
+1. Remove `config/initializers/job_cron_tabs.rb` that was previous used to ensure cron tabs, they will be loaded automatically now
+2. Remove the `Delayed::Backend::Mongoid::Job` class definition, not the settings from `config/initializers/delayed_job.rb`, these are included in this gem now.
+3. Cron jobs should extend from `Cron::Job` now and specify `cron_tab_entry` within the class. 
+```ruby
+module Cron
+  class MyJob < Job
+    cron_tab_entry :daily
+    def perform
+       # do your work
+    end
+  end
+end
+```
+Allowed values of cron_tab_entry are
+* :always - run every minute
+* :hourly - run every hour
+* :daily - run every day
+* :weekly - run every week on Sunday
+* :monthly - run every month on the first day of the month
+* '*/5 1,3 * 2 *' - Run according to unix crontab entry format. This would run every tuesday on the 1st and 3rd hour, for any minute divisable by 5, so 0, 5, 10, 15, etc..
+
+4. 

data/lib/app/jobs/cron/server.rb

@@ -42,7 +42,7 @@ module Cron
       end
       time_to_next_run
     rescue StandardError => error
-      App47Logger.log_error 'Unable to run cron jobs', error
+      App47Logger.log_error 'Unable to run cron server', error
       time_to_next_run
     ensure
       check_in
@@ -58,7 +58,7 @@ module Cron
     #
     def run_jobs
       now = Time.now.utc
-      CronTab.all.each { |tab| tab.run if tab.time_to_run?(now) }
+      Cron::Tab.all.each { |tab| tab.run if tab.time_to_run?(now) }
     end
 
     #
@@ -86,7 +86,7 @@ module Cron
     # Warm up a server on the next evaluation
     #
     def self.warm_up_server
-      return unless SystemConfiguration.auto_scaling_configured?
+      return unless SystemConfiguration.aws_auto_scaling_configured?
 
       primary_server.auto_scale([primary_server.desired_server_count + 1, 10].min)
     end
@@ -103,7 +103,7 @@ module Cron
       return if primary.present? && primary.alive?
 
       # no one else is in, so become primary
-      update_attributes! state: STATE_PRIMARY, last_check_in_at: Time.now.utc
+      update! state: STATE_PRIMARY, last_check_in_at: Time.now.utc
     end
 
     #
@@ -111,9 +111,10 @@ module Cron
     #
     def become_secondary(user = nil)
       if user.present?
-        update_attributes_and_log! user, state: STATE_SECONDARY
+        # update_attributes_and_log! user, state: STATE_SECONDARY
+        update! state: STATE_SECONDARY
       else
-        update_attributes! state: STATE_SECONDARY
+        update! state: STATE_SECONDARY
       end
     end
 
@@ -156,7 +157,7 @@ module Cron
     # Auto scale environment
     #
     def check_auto_scale
-      return unless SystemConfiguration.auto_scaling_configured?
+      return unless SystemConfiguration.aws_auto_scaling_configured?
 
       if delayed_jobs_count.eql?(0)
         handle_zero_job_count
@@ -169,10 +170,9 @@ module Cron
     # Returns the AWS AutoScaling Client
     #
     def client
-      credentials = { access_key_id: sys_config.access_key_id,
-                      secret_access_key: sys_config.secret_access_key,
-                      region: sys_config.region }
-      @client ||= Aws::AutoScaling::Client.new(credentials)
+      @client ||= Aws::AutoScaling::Client.new(access_key_id: sys_config.aws_access_key_id,
+                                               secret_access_key: sys_config.aws_secret_access_key,
+                                               region: sys_config.aws_region)
     end
 
     def sys_config
@@ -253,7 +253,7 @@ module Cron
       # Make sure we don't remove any workers with assigned jobs by accident
       return if desired_count.positive? && desired_count <= current_desired_capacity
 
-      client.update_auto_scaling_group(auto_scaling_group_name: sys_config.auto_scaling_group_name,
+      client.update_auto_scaling_group(auto_scaling_group_name: sys_config.aws_auto_scaling_group_name,
                                        min_size: desired_count,
                                        desired_capacity: desired_count)
     end

data/lib/app/models/concerns/core_system_configuration.rb

@@ -47,8 +47,8 @@ module CoreSystemConfiguration
       field :default_time_zone, type: String, default: 'US/Eastern'
       # AWS
       field :aws_region, type: String
-      field :aws_access_key, type: String
-      field :aws_access_secret, type: String
+      field :aws_access_key_id, type: String
+      field :aws_secret_access_key, type: String
       field :aws_auto_scaling_group_name, type: String
       # Zendesk
       field :zendesk_token, type: String
@@ -154,7 +154,7 @@ module CoreSystemConfiguration
   # Determine if AWS is configured
   #
   def aws_configured?
-    [aws_region.present?, aws_access_key.present?, aws_access_secret.present?].all?
+    [aws_region.present?, aws_access_key_id.present?, aws_secret_access_key.present?].all?
   end
 
   #
@@ -164,16 +164,6 @@ module CoreSystemConfiguration
     aws_configured? && aws_auto_scaling_group_name.present?
   end
 
-  #
-  # AWS client.
-  #
-  def aws_ec2_client
-    return nil unless aws_configured?
-
-    Aws::EC2::Client.new region: aws_region,
-                         credentials: Aws::Credentials.new(aws_access_key, aws_access_secret)
-  end
-
   #
   # Return the zendesk documentation URL
   #

data/lib/app/models/concerns/email_able.rb

@@ -1,3 +1,9 @@
+begin
+  require 'email_format'
+rescue LoadError
+  raise "You need to add gem 'email_format' to your Gemfile if you wish to use it."
+end
+
 #
 # Objects that emails can be sent too...
 #
@@ -15,7 +21,7 @@ module EmailAble
       #
       # Validations
       #
-      validates :email, presence: true, email_format: { strict: true }
+      validates :email, email_format: { strict: true }
       #
       # Callbacks
       #

data/lib/app/models/concerns/time_zone_able.rb

@@ -23,6 +23,10 @@ module TimeZoneAble
     end
   end
 
+  def time_zone_options
+    TZInfo::Timezone.all_identifiers
+  end
+
   #
   # Return the given time in the localized time for this object
   #

data/test/jobs/cron/server_test.rb

@@ -0,0 +1,160 @@
+require 'test_helper'
+
+module Cron
+  class ServerTest < ActiveSupport::TestCase
+    context 'Delayed job counting' do
+      setup do
+        @server = Cron::Server.find_or_create_server
+        Delayed::Worker.delay_jobs = true
+      end
+      teardown do
+        Delayed::Worker.delay_jobs = false
+      end
+      should 'return zero' do
+        assert_equal 0, @server.delayed_jobs_count
+      end
+      should 'return three' do
+        3.times do
+          MockDelayedJob.new.background
+        end
+        assert_equal 3, @server.delayed_jobs_count
+      end
+      should 'return zero with failed job' do
+        MockDelayedJob.new.background
+        job = Delayed::Backend::Mongoid::Job.first
+        job.set failed_at: Time.now.utc
+        assert_equal 0, @server.delayed_jobs_count
+      end
+    end
+    context 'AutoScalingJob' do
+      setup do
+        @auto_scaling_group_name = 'redmonocle-testing-group'
+        @config = SystemConfiguration.first
+        @config.set(aws_access_key_id: 'access_key_id',
+                    aws_secret_access_key: 'secret_access_key',
+                    aws_region: 'region',
+                    aws_auto_scaling_group_name: @auto_scaling_group_name)
+      end
+      should 'not create any more instances if there are 1 - 249 jobs' do
+        Cron::Server.any_instance.expects(:delayed_jobs_count).returns(49).at_least_once
+        Aws::AutoScaling::Client.any_instance.expects(:update_auto_scaling_group).never
+        Cron::Server.new.check_auto_scale
+      end
+      should 'create additional instances if there are 1,000 - 1,999 jobs and the number of instances currently running is less than the desired number of instances' do
+        Cron::Server.any_instance.expects(:current_desired_capacity).returns(0).at_least_once
+        Cron::Server.any_instance.expects(:delayed_jobs_count).returns(51).at_least_once
+        params = { auto_scaling_group_name: @auto_scaling_group_name, min_size: 1, desired_capacity: 1 }
+        Aws::AutoScaling::Client.any_instance.expects(:update_auto_scaling_group).with(params).once
+        Cron::Server.new.check_auto_scale
+      end
+      should 'not create any additional instances if there are 1,000 - 1,999 jobs and the number of desired instances is less than or equal to the number of instances currently running' do
+        Cron::Server.any_instance.expects(:current_desired_capacity).returns(1).at_least_once
+        Cron::Server.any_instance.expects(:delayed_jobs_count).returns(200).at_least_once
+        Aws::AutoScaling::Client.any_instance.expects(:update_auto_scaling_group).never
+        Cron::Server.new.check_auto_scale
+      end
+      should 'create additional instances if there are 2,000 - 2,999 jobs and the number of instances currently running is less than the desired number of instances' do
+        Cron::Server.any_instance.expects(:current_desired_capacity).returns(1).at_least_once
+        Cron::Server.any_instance.expects(:delayed_jobs_count).returns(300).at_least_once
+        params = { auto_scaling_group_name: @auto_scaling_group_name, min_size: 2, desired_capacity: 2 }
+        Aws::AutoScaling::Client.any_instance.expects(:update_auto_scaling_group).with(params).once
+        Cron::Server.new.check_auto_scale
+      end
+      should 'not create any additional instances if there are 2,000 - 2,999 jobs and the number of desired instances is less than or equal to the number of instances currently running' do
+        Cron::Server.any_instance.expects(:current_desired_capacity).returns(3).at_least_once
+        Cron::Server.any_instance.expects(:delayed_jobs_count).returns(750).at_least_once
+        Aws::AutoScaling::Client.any_instance.expects(:update_auto_scaling_group).never
+        Cron::Server.new.check_auto_scale
+      end
+      should 'create additional instances if there are 3,000 - 3,999 jobs and the number of instances currently running is less than the desired number of instances' do
+        Cron::Server.any_instance.expects(:current_desired_capacity).returns(1).at_least_once
+        Cron::Server.any_instance.expects(:delayed_jobs_count).returns(300).at_least_once
+        params = { auto_scaling_group_name: @auto_scaling_group_name, min_size: 2, desired_capacity: 2 }
+        Aws::AutoScaling::Client.any_instance.expects(:update_auto_scaling_group).with(params).once
+        Cron::Server.new.check_auto_scale
+      end
+      should 'not create any additional instances if there are 3,000 - 3,999 jobs and the number of desired instances is less than or equal to the number of instances currently running' do
+        Cron::Server.any_instance.expects(:current_desired_capacity).returns(5).at_least_once
+        Cron::Server.any_instance.expects(:delayed_jobs_count).returns(3_000).at_least_once
+        Aws::AutoScaling::Client.any_instance.expects(:update_auto_scaling_group).never
+        Cron::Server.new.check_auto_scale
+      end
+      should 'create additional instances if there are 4,000 jobs or more and the number of instances currently running is less than the desired number of instances' do
+        Cron::Server.any_instance.expects(:current_desired_capacity).returns(1).at_least_once
+        Cron::Server.any_instance.expects(:delayed_jobs_count).returns(700).at_least_once
+        params = { auto_scaling_group_name: @auto_scaling_group_name, min_size: 3, desired_capacity: 3 }
+        Aws::AutoScaling::Client.any_instance.expects(:update_auto_scaling_group).with(params).once
+        Cron::Server.new.check_auto_scale
+      end
+      should 'not create any additional instances if there are 4,000 jobs or more and the number of desired instances is less than or equal to the number of instances currently running' do
+        Cron::Server.any_instance.expects(:current_desired_capacity).returns(7).at_least_once
+        Cron::Server.any_instance.expects(:delayed_jobs_count).returns(4_000).at_least_once
+        Aws::AutoScaling::Client.any_instance.expects(:update_auto_scaling_group).never
+        Cron::Server.new.check_auto_scale
+      end
+      should 'remove all instances if there are no jobs left to run' do
+        Cron::Server.any_instance.expects(:current_desired_capacity).returns(7).at_least_once
+        Cron::Server.any_instance.expects(:delayed_jobs_count).returns(0).at_least_once
+        params = { auto_scaling_group_name: @auto_scaling_group_name, min_size: 0, desired_capacity: 0 }
+        Aws::AutoScaling::Client.any_instance.expects(:update_auto_scaling_group).with(params).once
+        Cron::Server.new.check_auto_scale
+      end
+      should 'create additional instances  by warming up a server' do
+        Cron::Server.any_instance.expects(:current_desired_capacity).returns(0).at_least_once
+        # Cron::Server.any_instance.expects(:delayed_jobs_count).returns(0).at_least_once
+        params = { auto_scaling_group_name: @auto_scaling_group_name, min_size: 1, desired_capacity: 1 }
+        Aws::AutoScaling::Client.any_instance.expects(:update_auto_scaling_group).with(params).once
+        Cron::Server.find_or_create_server.become_primary
+        Cron::Server.warm_up_server
+      end
+    end
+    context 'Run jobs' do
+      setup do
+        Cron::JobTab.ensure_cron_tabs
+        Cron::Tab.any_instance.expects(:time_to_run?).returns(true).at_least_once
+        Cron::TrimCronServers.expects(:perform_later).once
+        Cron::TrimNotifications.expects(:perform_later).once
+      end
+      should 'run jobs' do
+        assert_nothing_raised do
+          Cron::Server.find_or_create_server.execute
+        end
+      end
+    end
+    context 'Highlander' do
+      setup do
+        @s1 = Cron::Server.create! host_name: 's1', pid: '1123'
+        @s2 = Cron::Server.create! host_name: 's2', pid: '1124'
+        @s3 = Cron::Server.create! host_name: 's3', pid: '1125'
+        @s4 = Cron::Server.create! host_name: 's4', pid: '1126'
+      end
+      should 'allow one to become primary' do
+        assert_nothing_raised { @s1.become_primary }
+        assert_equal 1, Cron::Server.where(state: 'primary').count
+      end
+      should 'prevent second primary' do
+        assert_nothing_raised { @s1.become_primary }
+        refute @s2.update_attributes state: 'primary'
+        assert_equal 1, Cron::Server.where(state: 'primary').count
+      end
+      should 'determine to be alive ' do
+        server = Cron::Server.new host_name: 's1', pid: '1123', last_check_in_at: Time.now.utc
+        assert server.alive?
+        refute server.dead?
+      end
+      should 'determine to be dead' do
+        server = Cron::Server.new host_name: 's1', pid: '1123', last_check_in_at: 2.minutes.ago
+        refute server.alive?
+        assert server.dead?
+      end
+    end
+  end
+end
+
+class MockDelayedJob
+  def background
+
+  end
+
+  handle_asynchronously :background
+end

data/test/models/concerns/system_configuration_test.rb

@@ -33,8 +33,8 @@ class SystemConfigurationTest < ActiveSupport::TestCase
   should_have_field :zendesk_updates_path, type: String, klass: SystemConfiguration, default: 'hc'
   # AWS
   should_have_field :aws_region, type: String, klass: SystemConfiguration
-  should_have_field :aws_access_key, type: String, klass: SystemConfiguration
-  should_have_field :aws_access_secret, type: String, klass: SystemConfiguration
+  should_have_field :aws_secret_access_key, type: String, klass: SystemConfiguration
+  should_have_field :aws_secret_access_key, type: String, klass: SystemConfiguration
   should_have_field :aws_auto_scaling_group_name, type: String, klass: SystemConfiguration
   # Switchboard
   should_have_field :switchboard_base_url, type: String, klass: SystemConfiguration, default: 'https://switchboard.app47.com'
@@ -128,27 +128,16 @@ class SystemConfigurationTest < ActiveSupport::TestCase
       assert SystemConfiguration.configuration.update aws_region: 'us'
       refute SystemConfiguration.aws_configured?
       refute SystemConfiguration.aws_auto_scaling_configured?
-      assert SystemConfiguration.configuration.update aws_access_key: 'key'
+      assert SystemConfiguration.configuration.update aws_access_key_id: 'key'
       refute SystemConfiguration.aws_configured?
       refute SystemConfiguration.aws_auto_scaling_configured?
-      assert SystemConfiguration.configuration.update aws_access_secret: 'abc123'
+      assert SystemConfiguration.configuration.update aws_secret_access_key: 'abc123'
       assert SystemConfiguration.aws_configured?
       refute SystemConfiguration.aws_auto_scaling_configured?
       assert SystemConfiguration.configuration.update aws_auto_scaling_group_name: 'g1'
       assert SystemConfiguration.aws_configured?
       assert SystemConfiguration.aws_auto_scaling_configured?
     end
-    should 'not retrieve client' do
-      assert_nil SystemConfiguration.aws_ec2_client
-    end
-    should 'retrieve client' do
-      assert SystemConfiguration.configuration.update aws_region: 'us',
-                                                      aws_access_key: 'key',
-                                                      aws_access_secret: 'key'
-      client = SystemConfiguration.aws_ec2_client
-      assert_not_nil client
-      assert client.is_a?(Aws::EC2::Client)
-    end
   end
 
   context 'twilio configuration' do

data/test/test_helper.rb

@@ -15,7 +15,7 @@ if ENV['CODACY_PROJECT_TOKEN']
 end
 
 require 'rubygems'
-require 'aws-sdk-ec2'
+require 'aws-sdk-autoscaling'
 require 'mongoid'
 require 'delayed_job_mongoid'
 require 'email_format'

data/web47core.gemspec

@@ -8,7 +8,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 Gem::Specification.new do |spec|
   spec.required_ruby_version = '~> 2.4.1'
   spec.name          = 'web47core'
-  spec.version       = '0.1.0'
+  spec.version       = '0.1.1'
   spec.authors       = ['Chris Schroeder']
   spec.email         = ['chris@app47.com']
   spec.summary       = 'App47 Web Core Library.'
@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
 
   # spec.add_dependency 'json'
   spec.add_runtime_dependency 'activesupport', '~> 5.0'
-  spec.add_runtime_dependency 'aws-sdk-ec2', '> 1.140', '<= 1.160'
+  spec.add_runtime_dependency 'aws-sdk-autoscaling'
   spec.add_runtime_dependency 'delayed_job_mongoid', '~> 2.3'
   spec.add_runtime_dependency 'daemons'
   spec.add_runtime_dependency 'email_format'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: web47core
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Chris Schroeder
@@ -25,25 +25,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-ec2
+  name: aws-sdk-autoscaling
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '1.140'
-    - - "<="
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.160'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '1.140'
-    - - "<="
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.160'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: delayed_job_mongoid
   requirement: !ruby/object:Gem::Requirement
@@ -556,6 +550,7 @@ files:
 - test/fixtures/redis/options.yml
 - test/fixtures/redis/sentinel.yml
 - test/fixtures/redis/url.yml
+- test/jobs/cron/server_test.rb
 - test/jobs/cron/switchboard_sync_configuration_test.rb
 - test/jobs/cron/trim_cron_servers_test.rb
 - test/jobs/cron/trim_failed_delayed_jobs_test.rb
@@ -611,6 +606,7 @@ test_files:
 - test/fixtures/redis/options.yml
 - test/fixtures/redis/sentinel.yml
 - test/fixtures/redis/url.yml
+- test/jobs/cron/server_test.rb
 - test/jobs/cron/switchboard_sync_configuration_test.rb
 - test/jobs/cron/trim_cron_servers_test.rb
 - test/jobs/cron/trim_failed_delayed_jobs_test.rb