skyrunner 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b828d1a9dbc216ff8aca55ce2e8007f07ff1c47d
-  data.tar.gz: 21ed7eb84af89b74578726cb69faadc981c6fdf6
+  metadata.gz: 3b4147a678d048c85f857479136563287a39576e
+  data.tar.gz: b1ccac1353150bc7948332135aefd8c90bbefb5b
 SHA512:
-  metadata.gz: f78e396093a0dcbe578b1976911b29e4eab5fdf01ecaffff0405ebbc7394b642923c5b3b47da2da6c4419168cbb8b5932646eddc3391a79c319e43b5b6e44e2a
-  data.tar.gz: 43e38c229e0f32c9cddc66ffa375657f96f1e2b8de18e26cf030c14e96a16d065cd813f895281b8eb7ab4cd95fc0807a778117b8f75a8dc8d1766fa0c3c9e25a
+  metadata.gz: c4c80e760302f36ccdafc30b7c76b9f86838728ee479f578fd71f8e401285845fcb07a7c4003787f2d0d843959f6a9edf7dcddf7d6decec56aaff462aeee2ed8
+  data.tar.gz: a56cfeae042ce3387ffaa1cb757f06d26639f65b2b9ed4789aefe04bdc02442b65367943ca2625161e4826bac8a5bfac010e38c062bf981aa2bdbb13aa68562d

data/Gemfile

@@ -7,3 +7,4 @@ gem "aws-sdk"
 gem "activesupport"
 gem "log4r"
 gem "trollop"
+gem "retries"

data/bin/skyrunner

@@ -38,8 +38,7 @@ end
 
 SkyRunner.dynamo_db_table_name = opts[:dynamo_db_table_name]
 SkyRunner.sqs_queue_name = opts[:sqs_queue_name]
-SkyRunner.consumer_batch_size = opts[:batch_size].to_i
-SkyRunner.num_threads = opts[:num_threads].to_i
+SkyRunner.consumer_threads = opts[:num_threads].to_i
 
 COMMANDS = ["init", "purge", "consume", "test"]
 
@@ -57,6 +56,6 @@ when "consume"
 when "test"
   $: << "."
   require "#{File.dirname(__FILE__)}/../jobs/example_job"
-  ExampleJobModule::ExampleJob.new.execute!(number_of_tasks: 100)
+  ExampleJobModule::ExampleJob.new.execute!(number_of_tasks: 1000)
   SkyRunner.consume!
 end

data/lib/generators/sky_runner/install/templates/skyrunner.rb

@@ -5,17 +5,13 @@ SkyRunner.setup do |config|
   config.dynamo_db_table_name = "skyrunner_jobs_#{Rails.env}"
   config.sqs_queue_name = "skyrunner_tasks_#{Rails.env}"
 
-  # Set the number of tasks for a consumer to pull and run from SQS at a time. (Max 10, default 10)
-  #
-  # config.consumer_batch_size = 10
-
   # Set the visibility timeout of queue items. If the consumer batch size (above) is set to 10,
   # this should provide sufficient time for a consumer to process 10 tasks, for example. (default 90)
   #
   # config.visibility_timeout = 90
 
-  # Set the number of concurrent threads for the consumer process.
+  # Set the number of concurrent consumer threads when running the consumer.
   # (If greater than one, you obviously need to make sure your tasks are thread-safe.)
   #
-  # config.num_threads = 10
+  # config.consumer_threads = 10
 end

data/lib/skyrunner/job.rb

@@ -34,13 +34,17 @@ module SkyRunner::Job
     table = SkyRunner.dynamo_db_table
     queue = SkyRunner.sqs_queue
 
-    record = table.items.put(job_id: job_id, class: self.class.name, args: args.to_json, total_tasks: 1, completed_tasks: 0, done: 0, failed: 0)
+    record = nil
+
+    SkyRunner::retry_dynamo_db do
+      record = table.items.put(id: job_id, task_id: job_id, class: self.class.name, args: args.to_json, total_tasks: 1, completed_tasks: 0, done: 0, failed: 0)
+    end
 
     pending_args = []
 
     flush = lambda do
       messages = pending_args.map do |task_args|
-        { job_id: job_id, task_id: SecureRandom.hex, task_args: task_args }.to_json
+        { job_id: job_id, task_id: SecureRandom.hex, task_args: task_args, job_class: self.class.name }.to_json
       end
 
       dropped_message_count = 0
@@ -57,7 +61,9 @@ module SkyRunner::Job
         end
       end
 
-      record.attributes.add({ total_tasks: messages.size - dropped_message_count })
+      SkyRunner::retry_dynamo_db do
+        record.attributes.add({ total_tasks: messages.size - dropped_message_count })
+      end
     end
 
     self.run(args) do |*task_args|
@@ -94,7 +100,7 @@ module SkyRunner::Job
   private 
 
   def dynamo_db_record
-    SkyRunner.dynamo_db_table.items[self.skyrunner_job_id]
+    SkyRunner.dynamo_db_table.items[self.skyrunner_job_id, self.skyrunner_job_id]
   end
 
   def handle_task_failed!
@@ -102,7 +108,10 @@ module SkyRunner::Job
 
     begin
       record = dynamo_db_record
-      record.attributes.add({ failed: 1 })
+
+      SkyRunner::retry_dynamo_db do
+        record.attributes.add({ failed: 1 })
+      end
 
       (self.class.job_event_methods[:failed] || []).each do |method|
         if self.method(method).arity == 0 && self.method(method).parameters.size == 0
@@ -111,6 +120,8 @@ module SkyRunner::Job
           self.send(method, JSON.parse(record.attributes["args"]).symbolize_keys)
         end
       end
+
+      delete_task_records! rescue nil
     rescue Exception => e
     end
   end
@@ -119,15 +130,20 @@ module SkyRunner::Job
     return false unless self.skyrunner_job_id
 
     record = dynamo_db_record
+    new_attributes = nil
 
-    new_attributes = record.attributes.add({ completed_tasks: 1 }, return: :all_new)
+    SkyRunner::retry_dynamo_db do
+      new_attributes = record.attributes.add({ completed_tasks: 1 }, return: :all_new)
+    end
 
     if new_attributes["total_tasks"] == new_attributes["completed_tasks"]
       begin
         if_condition = { completed_tasks: new_attributes["total_tasks"], done: 0 }
 
-        record.attributes.update(if: if_condition) do |u|
-          u.add(done: 1)
+        SkyRunner::retry_dynamo_db do
+          record.attributes.update(if: if_condition) do |u|
+            u.add(done: 1)
+          end
         end
 
         (self.class.job_event_methods[:completed] || []).each do |method|
@@ -137,11 +153,69 @@ module SkyRunner::Job
             self.send(method, JSON.parse(record.attributes["args"]).symbolize_keys)
           end
         end
+
+        delete_task_records! rescue nil
       rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException => e
-        # This is OK, we had a double finisher.
+        # This is OK, we had a double finisher so lets block them.
       end
     end
 
     true
   end
+
+  def delete_task_records!
+    delete_batch_queue = Queue.new
+    mutex = Mutex.new
+    delete_items_queued = false
+    threads = []
+
+    1.upto([1, (SkyRunner.consumer_threads / 4.0).floor].max) do 
+      threads << Thread.new do
+
+        db_table = SkyRunner.dynamo_db_table
+
+        loop do
+          should_break = false
+
+          mutex.synchronize do
+            should_break = (SkyRunner::stop_consuming? || delete_items_queued) && delete_batch_queue.empty?
+          end
+
+          break if should_break
+
+          if delete_batch_queue.size > 0
+            batch = delete_batch_queue.pop
+
+            if batch
+              SkyRunner::retry_dynamo_db do
+                db_table.batch_delete(batch)
+              end
+            end
+          else
+            sleep 1
+          end
+        end
+      end
+    end
+
+    items_to_delete = []
+    table = SkyRunner.dynamo_db_table
+
+    table.items.query(hash_value: "#{self.skyrunner_job_id}-tasks", select: [:id, :task_id]) do |task_item|
+      items_to_delete << [task_item.attributes["id"], task_item.attributes["task_id"]]
+
+      if items_to_delete.size >= 25
+        delete_batch_queue << items_to_delete
+        items_to_delete = []
+      end
+    end
+
+    delete_batch_queue << items_to_delete unless items_to_delete.empty?
+    
+    mutex.synchronize do
+      delete_items_queued = true
+    end
+
+    threads.each(&:join)
+  end
 end

data/lib/skyrunner/version.rb

@@ -1,3 +1,3 @@
 module Skyrunner
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/lib/skyrunner.rb

@@ -5,6 +5,7 @@ require "active_support/core_ext"
 require "log4r"
 require "json"
 require "set"
+require "retries"
 
 module SkyRunner
   require "skyrunner/engine" if defined?(Rails)
@@ -33,7 +34,8 @@ module SkyRunner
       table = dynamo_db.tables.create(table_name, 
                                      SkyRunner.dynamo_db_read_capacity, 
                                      SkyRunner.dynamo_db_write_capacity,
-                                     hash_key: { job_id: :string })
+                                     hash_key: { id: :string },
+                                     range_key: { task_id: :string })
 
       sleep 1 while table.status == :creating
     end
@@ -61,92 +63,124 @@ module SkyRunner
   end
 
   def self.consume!(&block)
-    queue = sqs_queue
-    table = dynamo_db_table
-    raise "Queue #{SkyRunner::sqs_queue_name} not found. Try running 'skyrunner init'" unless queue
-    raise "DynamoDB table #{SkyRunner::dynamo_db_table_name} not found. Try running 'skyrunner init'" unless table && table.exists?
+    raise "Queue #{SkyRunner::sqs_queue_name} not found. Try running 'skyrunner init'" unless sqs_queue
+    raise "DynamoDB table #{SkyRunner::dynamo_db_table_name} not found. Try running 'skyrunner init'" unless dynamo_db_table && dynamo_db_table.exists?
 
     local_queue = Queue.new
-    error_queue = Queue.new
 
     threads = []
 
-    1.upto(SkyRunner::num_threads) do
+    1.upto(SkyRunner::consumer_threads) do
       threads << Thread.new do
+        table = SkyRunner::dynamo_db_table
+
         loop do
-          break if SkyRunner::stop_consuming? && local_queue.empty?
+          begin
+            if local_queue.empty?
+              break if SkyRunner::stop_consuming?
+
+              sleep 1
+              next
+            end
 
-          sleep 1 unless local_queue.size > 0
+            klass, job_id, task_id, task_args, message = local_queue.pop
 
-          klass, job_id, task_args, message = local_queue.pop
+            if klass
+              begin
+                # Avoid running the same task twice, enter record and raise error if exists already.
 
-          if klass
-            SkyRunner::log :info, "Run Task: #{task_args} Job: #{job_id} Message: #{message.id}"
+                SkyRunner::retry_dynamo_db do
+                  table.items.put({ id: "#{job_id}-tasks", task_id: task_id }, unless_exists: ["id", "task_id"])
+                end
 
-            job = klass.new
-            job.skyrunner_job_id = job_id
+                SkyRunner::log :info, "Run Task: #{task_args} Job: #{job_id} Message: #{message.id}"
 
-            begin
-              job.consume!(task_args)
-              message.delete
-            rescue Exception => e
-              message.delete rescue nil
-              error_queue.push(e)
-              SkyRunner::log :error, "Task Failed: #{task_args} Job: #{job_id} #{e.message} #{e.backtrace.join("\n")}"
+                job = klass.new
+                job.skyrunner_job_id = job_id
+
+                begin
+                  job.consume!(task_args)
+                  message.delete
+                rescue Exception => e
+                  message.delete rescue nil
+                  block.call(e) if block_given?
+                  SkyRunner::log :error, "Task Failed: #{task_args} Job: #{job_id} #{e.message} #{e.backtrace.join("\n")}"
+                end
+              rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException => e
+                message.delete rescue nil
+              end
             end
+          rescue Exception => e
+            puts e.message
+            puts e.backtrace.join("\n")
+            raise e
           end
         end
       end
     end
 
-    log :info, "Consumer started."
+    1.upto((SkyRunner::consumer_threads.to_f / SQS_MAX_BATCH_SIZE).ceil + 1) do
+      threads << Thread.new do
+        begin
+          loop do
+            table = SkyRunner::dynamo_db_table
+            queue = sqs_queue
 
-    loop do
-      if error_queue.size > 0
-        SkyRunner::stop_consuming!
+            break if SkyRunner::stop_consuming?
 
-        while error_queue.size > 0
-          error = error_queue.pop
-          yield error if block_given?
-        end
-      end
+            sleep 1 while local_queue.size >= SkyRunner.consumer_threads
 
-      return true if stop_consuming?
+            received_messages = []
 
-      sleep 1 while local_queue.size >= SkyRunner.num_threads
+            queue.receive_messages(limit: SQS_MAX_BATCH_SIZE, wait_time_seconds: 5) do |message|
+              received_messages << [message, JSON.parse(message.body)]
+            end
 
-      received_messages = []
+            next unless received_messages.size > 0
 
-      batch_size = [1, [SkyRunner.consumer_batch_size, SQS_MAX_BATCH_SIZE].min].max
+            job_ids = received_messages.map { |m| [m[1]["job_id"], m[1]["job_id"]] }.uniq
 
-      queue.receive_messages(limit: batch_size, wait_time_seconds: 5) do |message|
-        received_messages << [message, JSON.parse(message.body)]
-      end
+            job_records = {}
 
-      next unless received_messages.size > 0
-
-      table.batch_get(:all, received_messages.map { |m| m[1]["job_id"] }.uniq, consistent_read: true) do |record|
-        received_messages.select { |m| m[1]["job_id"] == record["job_id"] }.each do |received_message|
-          message, message_data = received_message
-          job_id = message_data["job_id"]
-
-          if record["failed"] == 0 && error_queue.empty?
-            begin
-              klass = Kernel.const_get(record["class"])
-              task_args = message_data["task_args"]
-              local_queue.push([klass, job_id, task_args, message])
-            rescue NameError => e
-              message.delete rescue nil
-              log :error, "Task Failed: No such class #{record["class"]} #{e.message}"
-              yield e if block_given?
+            SkyRunner::retry_dynamo_db do
+              # Read DynamoDB records into job and task lookup tables.
+              table.batch_get(["id", "task_id", "failed"], job_ids.uniq, consistent_read: true) do |record|
+                job_records[record["id"]] = record
+              end
+            end
+
+            received_messages.each do |received_message|
+              message, message_data = received_message
+              job_id = message_data["job_id"]
+              task_id = message_data["task_id"]
+
+              job_record = job_records[job_id]
+
+              if job_record && job_record["failed"] == 0
+                begin
+                  klass = Kernel.const_get(message_data["job_class"])
+                  task_args = message_data["task_args"]
+                  local_queue.push([klass, job_id, task_id, task_args, message])
+                rescue NameError => e
+                  block.call(e) if block_given?
+                  message.delete rescue nil
+                  log :error, "Task Failed: No such class #{message_data["job_class"]} #{e.message}"
+                end
+              else
+                message.delete rescue nil
+              end
             end
-          else
-            message.delete
           end
+        rescue Exception => e
+          puts e.message
+          puts e.backtrace.join("\n")
+          raise e
         end
       end
     end
 
+    log :info, "Consumer started."
+
     threads.each(&:join)
 
     true
@@ -188,14 +222,11 @@ module SkyRunner
   mattr_accessor :sqs_message_retention_period
   @@sqs_message_retention_period = 345600
 
-  mattr_accessor :consumer_batch_size
-  @@consumer_batch_size = 10
-
   mattr_accessor :logger
   @@logger = Log4r::Logger.new("skyrunner")
 
-  mattr_accessor :num_threads
-  @@num_threads = 10
+  mattr_accessor :consumer_threads
+  @@consumer_threads = 10
 
   mattr_accessor :stop_consuming_flag
 
@@ -217,6 +248,18 @@ module SkyRunner
     end
   end
 
+  def self.retry_dynamo_db(&block)
+    handler = Proc.new do |exception, num, delay|
+      if exception
+        SkyRunner.log :warn, "Having to retry DynamoDB requests. #{exception.message}"
+      end
+    end
+
+    with_retries(handler: handler, max_tries: 100, rescue: AWS::DynamoDB::Errors::ProvisionedThroughputExceededException, base_sleep_seconds: 2, max_sleep_seconds: 60) do
+      block.call
+    end
+  end
+
   private
 
   def self.dynamo_db

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: skyrunner
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Greg Fodor