herdst_worker 0.2.20 → 0.2.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ce5fcef880ba98bfb395902e833adb341b994af3ad88075578e63e00ee44323
-  data.tar.gz: b7a9f4fc95c34e96fbcc07efb82ef57ad67744fd645dfe95866685c7ab0ec478
+  metadata.gz: 7df01a32a23f6963c2b6a6f080aa917ef9d33c3dc72928f28f8162d86f4f42db
+  data.tar.gz: d222ec3faf7a24c0b5d73b905c384356635d6061104a7724dc84186588f81609
 SHA512:
-  metadata.gz: 73f8e2525c067e07115e0e67b4d55937983a185d98fb682c70f9b3ff7738ee0821602c38c7c786d1ef9b03b3696bb30cd2c234fe5fb8fe0cc8f1ead6fec2b233
-  data.tar.gz: 3a084a98cd4d0da4cd8bd2e2116585512f358d0fdd8787528e3ebd38a632e8defa240475554d1169c56c8dd5b6800c6f2e578173b41e81d69d969b9bbef19bba
+  metadata.gz: 2b2a5d21ad826cab67bf3c78bbcdf4be85bca917ea0c79475bedbc073d47ad5b9d5d9b6ec4b1b713005711f552d0d571c643c36e88197c9e3e7fa0aef6b0c411
+  data.tar.gz: be88aa8b2fbd5a004832a90ba992dfdd8c3108059d1fdaf34017242fb3c09c1a8d39213e3e6f7d3249b5919876245ef30371a5aa9dfa73b438d9de4c6858f870

data/bin/herdst_worker

@@ -11,6 +11,16 @@ application_file = arg_pair["application"] || "application.rb"
 application_path = "#{Dir.pwd}/#{application_file}"
 application_env = arg_pair["env"] || "production"
 queue_enabled = arg_pair["queue_enabled"] == "false" ? false : true
+primary_process = true
+runtime = ((3600 * 4.75) + (rand * (3600 * 0.25))).floor # Between 4.75 hours and 5 hours
+
+if arg_pair["primary"].to_s == "false"
+    primary_process = false
+end
+
+if arg_pair["runtime"]
+    runtime = arg_pair["runtime"].to_i || runtime
+end
 
 raise "Please specify a name for the application E.g. --name=[name]" unless application_name
 raise "Please specify a family for the application E.g. --family=[family]" unless application_family
@@ -21,7 +31,7 @@ ENV["RAILS_ENV"] = application_env
 
 require_relative "../lib/herdst_worker"
 
-application_instance = HerdstWorker::Application.new(application_env, application_name, application_family, queue_enabled)
+application_instance = HerdstWorker::Application.new(application_env, application_name, application_family, queue_enabled, primary_process, runtime)
 
 require_relative application_path
 

data/bin/herdst_worker_console

@@ -22,7 +22,7 @@ require "irb"
 require "irb/completion"
 require_relative "../lib/herdst_worker"
 
-application_instance = HerdstWorker::Application.new(application_env, application_name, application_family, false)
+application_instance = HerdstWorker::Application.new(application_env, application_name, application_family, false, false, 0)
 
 require_relative application_path
 

data/bin/herdst_worker_health

@@ -1,8 +1,8 @@
 #!/usr/bin/env ruby
 
 tmp_path = Dir.pwd + "/tmp"
-process_id = File.new(tmp_path + "/process_id").read rescue nil
-process_status = File.new(tmp_path + "/process_status").read rescue nil
+process_id = File.new(tmp_path + "/process_id").read.strip rescue nil
+process_status = File.new(tmp_path + "/process_status").read.strip rescue nil
 unhealthy = process_id == nil || ["stopped"].include?(process_status)
 
 Kernel.exit(unhealthy ? 1 : 0)

data/lib/herdst_worker/application/facade.rb

@@ -4,7 +4,6 @@ require "aws-sdk-ecs"
 require_relative '../configuration/facade'
 require_relative '../log/facade'
 require_relative '../autoload/facade'
-require_relative '../signals/facade'
 require_relative '../queue/facade'
 require_relative '../adapters/facade'
 
@@ -15,13 +14,14 @@ module HerdstWorker
         
         attr_accessor :name, :family
         attr_accessor :logger, :config, :autoload
-        attr_accessor :poller_enabled, :queues, :poller_url, :queue
+        attr_accessor :poller_enabled, :queues, :poller_url, :queue, :primary_process, :runtime
         
         
-        def initialize(env, name, family, poller_enabled)
+        def initialize(env, name, family, poller_enabled, primary_process, runtime)
             self.name = name
             self.family = family
             self.poller_enabled = poller_enabled
+            self.runtime = runtime
             self.queues = ActiveSupport::HashWithIndifferentAccess.new
             
             HerdstWorker.set_application(self)
@@ -44,7 +44,7 @@ module HerdstWorker
         
         def get_queue(name)
             if self.queues[name]
-                HerdstWorker::Queue::Facade.new(self, false, self.queues[name], 20)
+                HerdstWorker::Queue::Facade.new(self, false, 0, self.queues[name], 20)
             else
                 nil
             end
@@ -61,9 +61,7 @@ module HerdstWorker
             if self.queue == nil
                 self.logger.info "Starting Application (#{$$})"
                 
-                HerdstWorker::Signals::Facade.listen(self.config.paths.temp)
-                
-                self.queue = HerdstWorker::Queue::Facade.new(self, self.poller_enabled, self.poller_url, 20)
+                self.queue = HerdstWorker::Queue::Facade.new(self, self.poller_enabled, self.runtime, self.poller_url, 20)
                 self.queue.start
             end
         end
@@ -91,24 +89,11 @@ module HerdstWorker
 
 
         def register_new_task
-            # info = self.config.metadata.get_service_config
-            # credentials = self.config.metadata.get_aws_credentials!
-            # ecs_client = Aws::ECS::Client.new(
-            #     :region => info[:region],
-            #     :credentials => credentials
-            # )
-
-            # resp = ecs_client.update_service({
-            #     :cluster => info[:cluster],
-            #     :service => info[:service_name],
-
-            #     # :task_definition => "amazon-ecs-sample", 
-
-            #     :desired_count => desired_count, 
-            # })
-
-            # puts "info"
-            # puts info.inspect
+            if self.primary_process
+                self.queue.stop
+            else
+                self.queue.stop
+            end
         end
 
 

data/lib/herdst_worker/queue/facade.rb

@@ -7,12 +7,13 @@ module HerdstWorker
         class Facade
             
             
-            attr_accessor :app, :enabled, :url, :queue_wait_time, :processor
+            attr_accessor :app, :enabled, :runtime, :url, :queue_wait_time, :processor
             
             
-            def initialize(app, enabled, url, queue_wait_time)
+            def initialize(app, enabled, runtime, url, queue_wait_time)
                 self.app = app
                 self.enabled = enabled
+                self.runtime = runtime
                 self.url = url
                 self.queue_wait_time = queue_wait_time
             end
@@ -28,11 +29,6 @@ module HerdstWorker
             end
             
             
-            def halt
-                self.get_processor.halt
-            end
-            
-            
             def stop
                 self.get_processor.stop
             end
@@ -40,7 +36,7 @@ module HerdstWorker
             
             def get_processor
                 unless self.processor
-                    self.processor = HerdstWorker::Queue::Processor.new(app, self.enabled, self.url, self.queue_wait_time)
+                    self.processor = HerdstWorker::Queue::Processor.new(app, self.enabled, self.runtime, self.url, self.queue_wait_time)
                 end
 
                 self.processor

data/lib/herdst_worker/queue/processor.rb

@@ -9,12 +9,12 @@ module HerdstWorker
             
             
             attr_accessor :app, :enabled, :queue_url, :queue_wait_time, :poller
-            attr_accessor :start_time, :stop_processing_at, :stop_at, :run_duration, :stop_suration
-            attr_accessor :processor_status, :job_count, :max_jobs
+            attr_accessor :start_time, :run_duration, :expire_at
+            attr_accessor :processor_status, :processor_expired, :job_count, :max_jobs
             attr_accessor :attempt_threshold, :visibility_timeout, :ignored_notifications
             
             
-            def initialize(app, enabled, queue_url, queue_wait_time)
+            def initialize(app, enabled, runtime, queue_url, queue_wait_time)
                 self.app = app
                 self.enabled = enabled
                 self.queue_url = queue_url
@@ -27,12 +27,12 @@ module HerdstWorker
                 self.ignored_notifications = [
                     "AmazonSnsSubscriptionSucceeded"
                 ]
-                
-                # Set the start time 
-                self.run_duration = (3600 * 4) + (rand * 900).floor # Four hours + bit more (< 15 minutes)
-                self.stop_suration = 300 # 5 minutes
+
+                self.processor_status = "starting"
+                self.processor_expired = false
+                self.run_duration = runtime
                 self.reset_time
-                
+
                 # Start the processor as working
                 self.set_status "starting"
 
@@ -49,7 +49,7 @@ module HerdstWorker
                     self.poller.poll(:wait_time_seconds => self.queue_wait_time, :skip_delete => false) do |msg|
                         process_message(msg)
                     end
-
+                    
                     self.app.deregister_task
                 else
                     raise "Cannot start a queue which is not enabled"
@@ -72,14 +72,6 @@ module HerdstWorker
             end
             
             
-            # Sets the processor status to finishing. The sqs before action will 
-            # take care of setting the idle state once all jobs have finished.
-            def halt
-                return if self.processor_status == "finishing"
-                set_status "finishing"
-            end
-            
-            
             # Sets the processor status to stopping. The sqs before action will 
             # take care of stopping the application once all jobs have finished.
             def stop
@@ -118,23 +110,11 @@ module HerdstWorker
                 # After hours of running terminate application. 
                 # The app will automatically restart in production
                 current_time = Time.now.utc.to_i
-
-                if (self.processor_status == "stopping") && (current_time >= self.stop_at)
-                    self.app.deregister_task
-
-                elsif (self.processor_status == "working") && (current_time >= self.stop_processing_at)
+                
+                if (self.processor_status == "working") && (current_time >= self.expire_at) && (self.processor_expired == false)
+                    self.processor_expired = true
                     self.app.force_stop(900)
                     self.app.register_new_task
-
-                    set_status "stopping"
-
-                # On finishing wait for jobs to complete and then set status to idle
-                elsif self.processor_status == "finishing"
-                    if self.job_count == 0
-                        self.app.logger.queue.warn "Setting processor status to idle"
-                        set_status "idle"
-                    end
-                    
                 end
 
                 # On stopping wait for jobs to complete and then set status
@@ -181,50 +161,51 @@ module HerdstWorker
                         self.job_count -= 1
                         
                     }.rescue { |ex|
-                        if will_fail_permanently
-                            self.app.logger.queue.error "Message failed #{attempt_number} times, Reporting and failing permanently. \n#{ex.to_s} \n#{ex.backtrace.join("\n")}"
-                            Sentry.capture_exception(ex, {
-                                :level => "fatal",
-                                :extra => {
-                                    "queue_attempts" => attempt_number,
-                                    "queue_message_body" => msg.body
-                                }
-                            })
+                        begin
+                            if will_fail_permanently
+                                self.app.logger.queue.error "Message failed #{attempt_number} times, Reporting and failing permanently. \n#{ex.to_s} \n#{ex.backtrace.join("\n")}"
+                                Sentry.capture_exception(ex, {
+                                    :level => "fatal",
+                                    :extra => {
+                                        "queue_attempts" => attempt_number,
+                                        "queue_message_body" => msg.body
+                                    }
+                                })
 
-                        else
-                            self.app.logger.queue.error "Message failed #{attempt_number} times, Adding back to queue."
+                            else
+                                self.app.logger.queue.error "Message failed #{attempt_number} times, Adding back to queue."
+                                
+                                self.poller.client.send_message({
+                                    :queue_url => self.poller.queue_url,
+                                    :message_body => msg.body,
+                                    :delay_seconds => self.visibility_timeout,
+                                    :message_attributes => msg_attrs.merge({
+                                        "attempts" => {
+                                            :string_value => attempt_number.to_s,
+                                            :data_type => "Number"
+                                        }
+                                    })
+                                })
+                            end
                             
                             if self.app.config.is_dev?
-                                puts ex.inspect
-                                puts ex.backtrace
+                                self.app.logger.queue.error "Processor Error:"
+                                self.app.logger.queue.error ex.message
+                                self.app.logger.queue.error ex.backtrace
                             end
-                            
-                            replaced_message = {
-                                :queue_url => self.poller.queue_url,
-                                :message_body => msg.body,
-                                :delay_seconds => self.visibility_timeout,
-                                :message_attributes => msg_attrs.merge({
-                                    "attempts" => {
-                                        :string_value => attempt_number.to_s,
-                                        :data_type => "Number"
-                                    }
-                                })
-                            }
-                            
-                            self.poller.client.send_message replaced_message
-                        end
-                        
-                        if self.app.config.is_dev?
-                            self.app.logger.queue.error "Processor Error:"
-                            self.app.logger.queue.error ex.message
-                            self.app.logger.queue.error ex.backtrace
+
+                        rescue Exception => inner_ex
+                            self.app.logger.queue.error inner_ex.message
+
+                        ensure
+                            self.job_count -= 1
+
                         end
-                        
-                        self.job_count -= 1
                     }.execute
                 else
                     self.poller.change_message_visibility_timeout(msg, 5)
                     throw :skip_delete
+
                 end
             end
             
@@ -232,8 +213,7 @@ module HerdstWorker
             private
                 def reset_time
                     self.start_time = Time.now.utc.to_i
-                    self.stop_processing_at = self.start_time + self.run_duration
-                    self.stop_at = self.stop_processing_at + self.stop_suration
+                    self.expire_at = self.start_time + self.run_duration
                 end
             
         end

data/lib/herdst_worker/version.rb

@@ -1,3 +1,3 @@
 module HerdstWorker
-  VERSION = '0.2.20'
+  VERSION = '0.2.22'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: herdst_worker
 version: !ruby/object:Gem::Version
-  version: 0.2.20
+  version: 0.2.22
 platform: ruby
 authors:
 - Herd.St
@@ -262,7 +262,6 @@ files:
 - lib/herdst_worker/queue/facade.rb
 - lib/herdst_worker/queue/processor.rb
 - lib/herdst_worker/queue/runner.rb
-- lib/herdst_worker/signals/facade.rb
 - lib/herdst_worker/version.rb
 homepage: https://herd.st
 licenses:

data/lib/herdst_worker/signals/facade.rb

@@ -1,35 +0,0 @@
-module HerdstWorker
-    module Signals
-        class Facade
-            
-            def self.listen(process_path)
-                write_process_file(process_path)
-                
-                # Start
-                Signal.trap "USR1" do |x|
-                    HerdstWorker::Queue::Facade.start
-                end
-                
-                # Halt
-                Signal.trap "USR2" do |x|
-                    HerdstWorker::Queue::Facade.halt
-                end
-                
-                # Stop (abort)
-                Signal.trap "ABRT" do |x|
-                    HerdstWorker::Queue::Facade.stop
-                end
-            end
-            
-            private
-                def self.write_process_file(process_path)
-                    # Write process id to file so we can signal the current process
-                    process_id = "#{$$}"
-                    process_file = process_path + "/process_id"
-                    
-                    File.open(process_file, "w") { |file| file.write(process_id) }
-                end
-            
-        end
-    end
-end