skyrunner 0.0.13 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: acab7006b6eec4ef27bb95523a7731881a2c137a
-  data.tar.gz: b6d2dafafa9aa2c779c4ae5023720e32756924c0
+  metadata.gz: b6d11252dc57758004a10bba1e6d9321f07a6629
+  data.tar.gz: fe23c5593a322c8a39a90db60b8d0f6632768a19
 SHA512:
-  metadata.gz: 33db221875b2e2fb7f195ce792b96a6acb0405a2caa61f138c46cb29b66e67756eb54f26c3d59f207f1ce02518712aae4ec6266ced4f506b06fbabb4ce70596e
-  data.tar.gz: 2759e97bb2315a1f7307706524fa3ff87dd265acf498138d3085c22492c84eff062ea440112bd2f6fccd689dc1272434fd84a5170598e28e1d6715a893016061
+  metadata.gz: 6b2dc81fcdd2416d16973511e929731556661c6b8e4b4b9c0ef578ed3c500b61ce744e42925caa08709b503915b5c37224cb82841e1bb94468ec5e52ee7f960e
+  data.tar.gz: b61a040c8e9c8d3a829a3e1a56b546d1fcc61fa699d2c27dc5b1d2f1eaa6c9415f834ff8b88b627731e956ae2926c6dc6cc57e0f4ee6ef767d0ebb3b5757ad67

data/README.md

@@ -31,7 +31,7 @@ To start a consumer
 bundle exec rake skyrunner:consume
 ``
 
-To gracefully shut down a consumer, send it SIGINT or SIGTERM. It will finish up processing the messages it has de-queued before terminating.
+To gracefully shut down a consumer, send it SIGINT. It will finish up processing the messages it has de-queued before terminating.
 
 See `jobs/example_job.rb` for an example job. To run a job, just call `execute!` on the job, passing any named job arguments you want. The job class should implement the method `run`. This method will get passed the job arguments. For each task you want consumers to run, `run` should yield an array of two elements, the first being the name of the method on the job class to run for the task, and the second a Hash of method arguments.
 

data/bin/skyrunner

@@ -12,11 +12,7 @@ logger.level = Log4r::DEBUG
 SkyRunner.logger = logger
 
 trap("INT") do
-  SkyRunner.stop_consuming!
-end
-
-trap("TERM") do
-  SkyRunner.stop_consuming!
+  SkyRunner.stop_consuming!(true)
 end
 
 opts = Trollop::options do
@@ -29,6 +25,7 @@ opts = Trollop::options do
       Valid commands:
 
       consume - Starts consuming tasks.
+      test - Runs a test job.
       init - Creates DynamoDB table and SQS queue for SkyRunner.
       purge - Purges and re-creates DynamoDB table and SQS queue for SkyRunner. (Destructive!)
   END
@@ -37,14 +34,16 @@ opts = Trollop::options do
   opt :sqs_queue_name, "SQS queue use for tasks.", default: "skyrunner_tasks", type: :string
   opt :namespace, "Namespace of jobs to consume.", default: "default", type: :string
   opt :batch_size, "Number of tasks to consume per batch.", default: 10
+  opt :num_threads, "Number of consumer threads.", default: 10
 end
 
 SkyRunner.dynamo_db_table_name = opts[:dynamo_db_table_name]
 SkyRunner.sqs_queue_name = opts[:sqs_queue_name]
 SkyRunner.job_namespace = opts[:namespace]
 SkyRunner.consumer_batch_size = opts[:batch_size].to_i
+SkyRunner.num_threads = opts[:num_threads].to_i
 
-COMMANDS = ["init", "purge", "consume"]
+COMMANDS = ["init", "purge", "consume", "test"]
 
 Trollop::die "Must specify command" unless COMMANDS.include?(ARGV[0])
 
@@ -57,4 +56,9 @@ when "purge"
   SkyRunner.init!(purge: true)
 when "consume"
   SkyRunner.consume!
+when "test"
+  $: << "."
+  require "#{File.dirname(__FILE__)}/../jobs/example_job"
+  ExampleJobModule::ExampleJob.new.execute!(number_of_tasks: 500)
+  SkyRunner.consume!
 end

data/lib/generators/sky_runner/install/templates/skyrunner.rake

@@ -7,11 +7,7 @@ namespace :skyrunner do
   desc "Starts consuming SkyRunner tasks."
   task consume: :environment do
     trap("INT") do
-      SkyRunner.stop_consuming!
-    end
-
-    trap("TERM") do
-      SkyRunner.stop_consuming!
+      SkyRunner.stop_consuming!(true)
     end
 
     SkyRunner.consume! do |exception|

data/lib/generators/sky_runner/install/templates/skyrunner.rb

@@ -16,4 +16,9 @@ SkyRunner.setup do |config|
   # this should provide sufficient time for a consumer to process 10 tasks, for example. (default 90)
   #
   # config.visibility_timeout = 90
+
+  # Set the number of concurrent threads for the consumer process.
+  # (If greater than one, you obviously need to make sure your tasks are thread-safe.)
+  #
+  # config.num_threads = 10
 end

data/lib/skyrunner/job.rb

@@ -67,6 +67,7 @@ module SkyRunner::Job
         1.upto(5) do
           flush.()
           sleep 5 if pending_args.size > 0
+          break if pending_args.size == 0
         end
       end
     end
@@ -74,6 +75,7 @@ module SkyRunner::Job
     1.upto(5) do
       flush.() if pending_args.size > 0
       sleep 5 if pending_args.size > 0
+      break if pending_args.size == 0
     end
 
     handle_task_completed!

data/lib/skyrunner/version.rb

@@ -1,3 +1,3 @@
 module Skyrunner
-  VERSION = "0.0.13"
+  VERSION = "0.1.0"
 end

data/lib/skyrunner.rb

@@ -63,14 +63,55 @@ module SkyRunner
   def self.consume!(&block)
     queue = sqs_queue
     table = dynamo_db_table
-
     raise "Queue #{SkyRunner::sqs_queue_name} not found. Try running 'skyrunner init'" unless queue
     raise "DynamoDB table #{SkyRunner::dynamo_db_table_name} not found. Try running 'skyrunner init'" unless table && table.exists?
 
+    local_queue = Queue.new
+    error_queue = Queue.new
+
+    threads = []
+
+    1.upto(SkyRunner::num_threads) do
+      threads << Thread.new do
+        sqs = AWS::SQS.new
+
+        loop do
+          break if SkyRunner::stop_consuming?
+          sleep 1 unless local_queue.size > 0
+
+          klass, job_id, task_args, message = local_queue.pop
+
+          SkyRunner::log :info, "Run Task: #{task_args} Job: #{job_id} Message: #{message.id}"
+
+          job = klass.new
+          job.skyrunner_job_id = job_id
+
+          begin
+            job.consume!(task_args)
+            message.delete
+          rescue Exception => e
+            error_queue.push(e)
+            SkyRunner::log :error, "Task Failed: #{task_args} Job: #{job_id} #{e.message} #{e.backtrace.join("\n")}"
+          end
+        end
+      end
+    end
+
     log :info, "Consumer started."
 
     loop do
-      return true if stop_consuming
+      if error_queue.size > 0
+        SkyRunner::stop_consuming!
+
+        while error_queue.size > 0
+          error = error_queue.pop
+          yield error if block_given?
+        end
+      end
+
+      return true if stop_consuming?
+
+      sleep 1 while local_queue.size >= SkyRunner.num_threads
 
       received_messages = []
 
@@ -80,37 +121,19 @@ module SkyRunner
 
       next unless received_messages.size > 0
 
-      failed = false
-
       table.batch_get(:all, received_messages.map { |m| m[1]["job_id"] }.uniq, consistent_read: true) do |record|
         received_messages.select { |m| m[1]["job_id"] == record["job_id"] }.each_with_index do |received_message|
           message = received_message[1]
           job_id = message["job_id"]
 
-          if record["namespace"] == SkyRunner.job_namespace && record["failed"] == 0 && !failed
+          if record["namespace"] == SkyRunner.job_namespace && record["failed"] == 0 && error_queue.size == 0
             start_time = Time.now
 
             begin
               klass = Kernel.const_get(record["class"])
-
               task_args = message["task_args"]
-              log :info, "Run Task: #{task_args} Job: #{job_id}"
-
-              job = klass.new
-              job.skyrunner_job_id = job_id
-
-              begin
-                job.consume!(task_args)
-                received_message[0].delete
-
-                yield false if block_given?
-              rescue Exception => e
-                failed = true
-                log :error, "Task Failed: #{task_args} Job: #{job_id} #{e.message} #{e.backtrace.join("\n")}"
-                yield e if block_given?
-              end
+              local_queue.push([klass, job_id, task_args, received_message[0]])
             rescue NameError => e
-              failed = true
               log :error, "Task Failed: No such class #{record["class"]} #{e.message}"
               yield e if block_given?
             end
@@ -118,6 +141,10 @@ module SkyRunner
         end
       end
     end
+
+    threads.each { |t| t.join }
+
+    true
   end
 
   def self.dynamo_db_table
@@ -165,10 +192,27 @@ module SkyRunner
   mattr_accessor :logger
   @@logger = Log4r::Logger.new("skyrunner")
 
-  mattr_accessor :stop_consuming
+  mattr_accessor :num_threads
+  @@num_threads = 10
+
+  mattr_accessor :stop_consuming_flag
+
+  @@stop_consuming_mutex = Mutex.new
+
+  def self.stop_consuming?
+    @@stop_consuming_mutex.synchronize do
+      SkyRunner::stop_consuming_flag
+    end
+  end
 
-  def self.stop_consuming!
-    SkyRunner::stop_consuming = true
+  def self.stop_consuming!(its_a_trap=false)
+    if its_a_trap
+      SkyRunner::stop_consuming_flag = true
+    else
+      @@stop_consuming_mutex.synchronize do
+        SkyRunner::stop_consuming_flag = true
+      end
+    end
   end
 
   private

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: skyrunner
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.1.0
 platform: ruby
 authors:
 - Greg Fodor