herdst_worker 0.2.21 → 0.2.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9fa8ccaf7c0b1ea579be583057020ce2499ad6d7fd1ea806f526c81d0a3adfe3
-  data.tar.gz: 1c4af6793b81b741e308cad5db603fce45bd81dfb408ece724a4091c1c910471
+  metadata.gz: 1a7cc2b6da6b5a813708b3a1bea2d19e42bbdc2e8827e51cb10ff3f2d65c81f0
+  data.tar.gz: e97c06d7f2f48041b15331a8fe90d152e96483e04204ce86f6f278014add1792
 SHA512:
-  metadata.gz: 7bf2ce6ce4e78f2f8b409c63d390774028e0296afd81dde6664ba962f60089a8ea275d850adfbffd5970145eb499225d4a377df229c250761db40aa592521e24
-  data.tar.gz: 42c9f0b24900bef7ede09b10d0200cafa878f24dbd7d8506c4ea69f11555496b1c70bf6b67da800c1c2cc88c5409d5b8a1818e965cb05891fb4bf50b451acecb
+  metadata.gz: 05b7f71887d5d35e58099262c12308347c5817232bd5d0ea2d5a9d8dace2cbf775f8dc63e6e9a786d3eaa598fa129bdbe5f9f6fd844b6fc09fa680ac01002348
+  data.tar.gz: f715accff030919d5ffff2107595ec47982949b9a3e2791c8a5cd0c71811cf8c131f9e995f788a3977d6681f4a37e447eaaba342381928804b2fc1756cc3f462

data/bin/herdst_worker

@@ -11,6 +11,16 @@ application_file = arg_pair["application"] || "application.rb"
 application_path = "#{Dir.pwd}/#{application_file}"
 application_env = arg_pair["env"] || "production"
 queue_enabled = arg_pair["queue_enabled"] == "false" ? false : true
+primary_process = true
+runtime = ((3600 * 4.75) + (rand * (3600 * 0.25))).floor # Between 4.75 hours and 5 hours
+
+if arg_pair["primary"].to_s == "false"
+    primary_process = false
+end
+
+if arg_pair["runtime"]
+    runtime = arg_pair["runtime"].to_i || runtime
+end
 
 raise "Please specify a name for the application E.g. --name=[name]" unless application_name
 raise "Please specify a family for the application E.g. --family=[family]" unless application_family
@@ -21,7 +31,7 @@ ENV["RAILS_ENV"] = application_env
 
 require_relative "../lib/herdst_worker"
 
-application_instance = HerdstWorker::Application.new(application_env, application_name, application_family, queue_enabled)
+application_instance = HerdstWorker::Application.new(application_env, application_name, application_family, queue_enabled, primary_process, runtime)
 
 require_relative application_path
 

data/bin/herdst_worker_console

@@ -22,7 +22,7 @@ require "irb"
 require "irb/completion"
 require_relative "../lib/herdst_worker"
 
-application_instance = HerdstWorker::Application.new(application_env, application_name, application_family, false)
+application_instance = HerdstWorker::Application.new(application_env, application_name, application_family, false, false, 0)
 
 require_relative application_path
 

data/lib/herdst_worker/application/facade.rb

@@ -4,7 +4,6 @@ require "aws-sdk-ecs"
 require_relative '../configuration/facade'
 require_relative '../log/facade'
 require_relative '../autoload/facade'
-require_relative '../signals/facade'
 require_relative '../queue/facade'
 require_relative '../adapters/facade'
 
@@ -15,22 +14,24 @@ module HerdstWorker
         
         attr_accessor :name, :family
         attr_accessor :logger, :config, :autoload
-        attr_accessor :poller_enabled, :queues, :poller_url, :queue
+        attr_accessor :poller_enabled, :queues, :poller_url, :queue, :primary_process, :runtime
         
         
-        def initialize(env, name, family, poller_enabled)
+        def initialize(env, name, family, poller_enabled, primary_process, runtime)
             self.name = name
             self.family = family
             self.poller_enabled = poller_enabled
+            self.runtime = runtime
             self.queues = ActiveSupport::HashWithIndifferentAccess.new
-            
+
             HerdstWorker.set_application(self)
             
             self.logger = HerdstWorker::Log::Facade.new(get_logger_level(env))
             self.config = HerdstWorker::Configuration::Facade.new(env, name)
             self.autoload = HerdstWorker::Autoload::Facade.new(self)
             self.set_inflections
-            
+            self.set_process_id_file
+
             HerdstWorker::Adapters::Facade.bootstrap(self)
         end
         
@@ -44,7 +45,7 @@ module HerdstWorker
         
         def get_queue(name)
             if self.queues[name]
-                HerdstWorker::Queue::Facade.new(self, false, self.queues[name], 20)
+                HerdstWorker::Queue::Facade.new(self, false, 0, self.queues[name], 20)
             else
                 nil
             end
@@ -61,9 +62,7 @@ module HerdstWorker
             if self.queue == nil
                 self.logger.info "Starting Application (#{$$})"
                 
-                HerdstWorker::Signals::Facade.listen(self.config.paths.temp)
-                
-                self.queue = HerdstWorker::Queue::Facade.new(self, self.poller_enabled, self.poller_url, 20)
+                self.queue = HerdstWorker::Queue::Facade.new(self, self.poller_enabled, self.runtime, self.poller_url, 20)
                 self.queue.start
             end
         end
@@ -91,7 +90,11 @@ module HerdstWorker
 
 
         def register_new_task
-            
+            if self.primary_process
+                self.queue.stop
+            else
+                self.queue.stop
+            end
         end
 
 
@@ -134,6 +137,14 @@ module HerdstWorker
                 return "FATAL" if env == "test"
                 return "DEBUG"
             end
+
+            def set_process_id_file
+                # Write process id to file so we can signal the current process
+                process_id = "#{$$}"
+                process_file = self.config.paths.temp + "/process_id"
+                
+                File.open(process_file, "w") { |file| file.write(process_id) }
+            end
             
             
     end

data/lib/herdst_worker/queue/facade.rb

@@ -7,12 +7,13 @@ module HerdstWorker
         class Facade
             
             
-            attr_accessor :app, :enabled, :url, :queue_wait_time, :processor
+            attr_accessor :app, :enabled, :runtime, :url, :queue_wait_time, :processor
             
             
-            def initialize(app, enabled, url, queue_wait_time)
+            def initialize(app, enabled, runtime, url, queue_wait_time)
                 self.app = app
                 self.enabled = enabled
+                self.runtime = runtime
                 self.url = url
                 self.queue_wait_time = queue_wait_time
             end
@@ -28,11 +29,6 @@ module HerdstWorker
             end
             
             
-            def halt
-                self.get_processor.halt
-            end
-            
-            
             def stop
                 self.get_processor.stop
             end
@@ -40,7 +36,7 @@ module HerdstWorker
             
             def get_processor
                 unless self.processor
-                    self.processor = HerdstWorker::Queue::Processor.new(app, self.enabled, self.url, self.queue_wait_time)
+                    self.processor = HerdstWorker::Queue::Processor.new(app, self.enabled, self.runtime, self.url, self.queue_wait_time)
                 end
 
                 self.processor

data/lib/herdst_worker/queue/processor.rb

@@ -9,12 +9,12 @@ module HerdstWorker
             
             
             attr_accessor :app, :enabled, :queue_url, :queue_wait_time, :poller
-            attr_accessor :start_time, :stop_processing_at, :stop_at, :run_duration, :stop_suration
-            attr_accessor :processor_status, :job_count, :max_jobs
+            attr_accessor :start_time, :run_duration, :expire_at
+            attr_accessor :processor_status, :processor_expired, :job_count, :max_jobs
             attr_accessor :attempt_threshold, :visibility_timeout, :ignored_notifications
             
             
-            def initialize(app, enabled, queue_url, queue_wait_time)
+            def initialize(app, enabled, runtime, queue_url, queue_wait_time)
                 self.app = app
                 self.enabled = enabled
                 self.queue_url = queue_url
@@ -27,12 +27,12 @@ module HerdstWorker
                 self.ignored_notifications = [
                     "AmazonSnsSubscriptionSucceeded"
                 ]
-                
-                # Set the start time 
-                self.run_duration = (3600 * 4) + (rand * 900).floor # Four hours + bit more (< 15 minutes)
-                self.stop_suration = 300 # 5 minutes
+
+                self.processor_status = "starting"
+                self.processor_expired = false
+                self.run_duration = runtime
                 self.reset_time
-                
+
                 # Start the processor as working
                 self.set_status "starting"
 
@@ -49,7 +49,7 @@ module HerdstWorker
                     self.poller.poll(:wait_time_seconds => self.queue_wait_time, :skip_delete => false) do |msg|
                         process_message(msg)
                     end
-
+                    
                     self.app.deregister_task
                 else
                     raise "Cannot start a queue which is not enabled"
@@ -72,14 +72,6 @@ module HerdstWorker
             end
             
             
-            # Sets the processor status to finishing. The sqs before action will 
-            # take care of setting the idle state once all jobs have finished.
-            def halt
-                return if self.processor_status == "finishing"
-                set_status "finishing"
-            end
-            
-            
             # Sets the processor status to stopping. The sqs before action will 
             # take care of stopping the application once all jobs have finished.
             def stop
@@ -118,23 +110,11 @@ module HerdstWorker
                 # After hours of running terminate application. 
                 # The app will automatically restart in production
                 current_time = Time.now.utc.to_i
-
-                if (self.processor_status == "stopping") && (current_time >= self.stop_at)
-                    self.app.deregister_task
-
-                elsif (self.processor_status == "working") && (current_time >= self.stop_processing_at)
+                
+                if (self.processor_status == "working") && (current_time >= self.expire_at) && (self.processor_expired == false)
+                    self.processor_expired = true
                     self.app.force_stop(900)
                     self.app.register_new_task
-
-                    set_status "stopping"
-
-                # On finishing wait for jobs to complete and then set status to idle
-                elsif self.processor_status == "finishing"
-                    if self.job_count == 0
-                        self.app.logger.queue.warn "Setting processor status to idle"
-                        set_status "idle"
-                    end
-                    
                 end
 
                 # On stopping wait for jobs to complete and then set status
@@ -233,8 +213,7 @@ module HerdstWorker
             private
                 def reset_time
                     self.start_time = Time.now.utc.to_i
-                    self.stop_processing_at = self.start_time + self.run_duration
-                    self.stop_at = self.stop_processing_at + self.stop_suration
+                    self.expire_at = self.start_time + self.run_duration
                 end
             
         end

data/lib/herdst_worker/version.rb

@@ -1,3 +1,3 @@
 module HerdstWorker
-  VERSION = '0.2.21'
+  VERSION = '0.2.23'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: herdst_worker
 version: !ruby/object:Gem::Version
-  version: 0.2.21
+  version: 0.2.23
 platform: ruby
 authors:
 - Herd.St
@@ -262,7 +262,6 @@ files:
 - lib/herdst_worker/queue/facade.rb
 - lib/herdst_worker/queue/processor.rb
 - lib/herdst_worker/queue/runner.rb
-- lib/herdst_worker/signals/facade.rb
 - lib/herdst_worker/version.rb
 homepage: https://herd.st
 licenses:

data/lib/herdst_worker/signals/facade.rb

@@ -1,35 +0,0 @@
-module HerdstWorker
-    module Signals
-        class Facade
-            
-            def self.listen(process_path)
-                write_process_file(process_path)
-                
-                # Start
-                Signal.trap "USR1" do |x|
-                    HerdstWorker::Queue::Facade.start
-                end
-                
-                # Halt
-                Signal.trap "USR2" do |x|
-                    HerdstWorker::Queue::Facade.halt
-                end
-                
-                # Stop (abort)
-                Signal.trap "ABRT" do |x|
-                    HerdstWorker::Queue::Facade.stop
-                end
-            end
-            
-            private
-                def self.write_process_file(process_path)
-                    # Write process id to file so we can signal the current process
-                    process_id = "#{$$}"
-                    process_file = process_path + "/process_id"
-                    
-                    File.open(process_file, "w") { |file| file.write(process_id) }
-                end
-            
-        end
-    end
-end