skyrunner 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6980e3350176aaeafea768c35991b7dd4c7c8662
-  data.tar.gz: c9a0e818cd049ff11a4ae6f641de8815468c11c7
+  metadata.gz: 37d6d89a9e1f734ed6a46610d822b6ef19f80323
+  data.tar.gz: ea79aca278152621d7baa6cad890820daa6a5be6
 SHA512:
-  metadata.gz: 832397d587fa772a108a981e7aaf21e4001a29ff3caf0839362bbff446b3c2f43dcd23387c3584603c48d0fb809abe190289d345459e23a3bc105813916809e2
-  data.tar.gz: c8e4c17772bcbddd1da726278ad9ec5fbd2a63637cc8dfb683e610ef555d992fb0b931abf02bec5208063b4395ec3378d353a7b276040bcb5f946e3f7fb63414
+  metadata.gz: b851b35f64f494b2b340d7a476c3c9c6bd874c408e1c2c89bbf8fb8ebfb8a92146380619a63482ef4b4560f2d14a0bb7678342a701be9e2baa6c3980f00ec9a4
+  data.tar.gz: 2610b2bf6082dc62513e648d41a5133ec98db858940ff675273622a8a30abd4a353e05baf40e88f4b57d780fa4e5bd52da2bcef5f1ad3b6850e173f345c37f71

data/bin/skyrunner

@@ -56,6 +56,14 @@ when "consume"
 when "test"
   $: << "."
   require "#{File.dirname(__FILE__)}/../jobs/example_job"
-  ExampleJobModule::ExampleJob.new.execute!(number_of_tasks: 1000)
+
+  # Solo job, does not use DynamoDB for coordination.
+  1.upto(10) do |n|
+    ExampleJobModule::ExampleJob.new.execute!(number_of_tasks: 1)
+  end
+
+  # Batch job, uses DynamoDB to coordinate
+  ExampleJobModule::ExampleJob.new.execute!(number_of_tasks: 100)
+
   SkyRunner.consume!
 end

data/lib/skyrunner/job.rb

@@ -1,5 +1,6 @@
 module SkyRunner::Job
   attr_accessor :skyrunner_job_id
+  attr_accessor :skyrunner_job_is_solo
 
   def self.included(base)
     base.extend(ClassMethods)
@@ -27,26 +28,44 @@ module SkyRunner::Job
     end
   end
 
-  def execute!(args = {})
+  def is_solo?
+    self.skyrunner_job_is_solo
+  end
+
+  def execute!(job_args = {})
     job_id = SecureRandom.hex
     self.skyrunner_job_id = job_id
 
-    table = SkyRunner.dynamo_db_table
-    queue = SkyRunner.sqs_queue
-
+    table = nil
     record = nil
 
-    SkyRunner::retry_dynamo_db do
-      record = table.items.put(id: job_id, task_id: job_id, class: self.class.name, args: args.to_json, total_tasks: 1, completed_tasks: 0, done: 0, failed: 0)
-    end
-
+    queue = SkyRunner.sqs_queue
     pending_args = []
+    fired_solo = false
 
     flush = lambda do
       messages = pending_args.map do |task_args|
-        { job_id: job_id, task_id: SecureRandom.hex, task_args: task_args, job_class: self.class.name }.to_json
+        { job_id: job_id, task_id: SecureRandom.hex, job_args: job_args, task_args: task_args, job_class: self.class.name }
       end
 
+      if record.nil?
+        # Run an un-coordinated solo job if only one message
+        if messages.size > 1
+          SkyRunner::retry_dynamo_db do
+            table = SkyRunner.dynamo_db_table
+            record = table.items.put(id: job_id, task_id: job_id, class: self.class.name, args: job_args.to_json, total_tasks: 1, completed_tasks: 0, done: 0, failed: 0)
+          end
+        else
+          fired_solo = true
+
+          messages.each do |m|
+            m[:is_solo] = true
+          end
+        end
+      end
+
+      messages = messages.map(&:to_json)
+
       dropped_message_count = 0
       pending_args.clear
 
@@ -61,12 +80,14 @@ module SkyRunner::Job
         end
       end
 
-      SkyRunner::retry_dynamo_db do
-        record.attributes.add({ total_tasks: messages.size - dropped_message_count })
+      if record
+        SkyRunner::retry_dynamo_db do
+          record.attributes.add({ total_tasks: messages.size - dropped_message_count })
+        end
       end
     end
 
-    self.run(args) do |*task_args|
+    self.run(job_args) do |*task_args|
       pending_args << task_args
 
       if pending_args.size >= SkyRunner::SQS_MAX_BATCH_SIZE
@@ -84,15 +105,17 @@ module SkyRunner::Job
       break if pending_args.size == 0
     end
 
-    handle_task_completed!
+    unless fired_solo
+      handle_task_completed!(job_args)
+    end
   end
 
-  def consume!(task_args)
+  def consume!(job_args, task_args)
     begin
       self.send(task_args[0].to_sym, task_args[1].symbolize_keys)
-      handle_task_completed!
+      handle_task_completed!(job_args)
     rescue Exception => e
-      handle_task_failed! rescue nil
+      handle_task_failed!(job_args) rescue nil
       raise
     end
   end
@@ -103,46 +126,54 @@ module SkyRunner::Job
     SkyRunner.dynamo_db_table.items[self.skyrunner_job_id, self.skyrunner_job_id]
   end
 
-  def handle_task_failed!
+  def handle_task_failed!(job_args)
     return false unless self.skyrunner_job_id
 
     begin
-      record = dynamo_db_record
+      unless is_solo?
+        record = dynamo_db_record
 
-      SkyRunner::retry_dynamo_db do
-        record.attributes.add({ failed: 1 })
+        SkyRunner::retry_dynamo_db do
+          record.attributes.add({ failed: 1 })
+        end
       end
 
       (self.class.job_event_methods[:failed] || []).each do |method|
         if self.method(method).arity == 0 && self.method(method).parameters.size == 0
           self.send(method)
         else
-          self.send(method, JSON.parse(record.attributes["args"]).symbolize_keys)
+          self.send(method, job_args.symbolize_keys)
         end
       end
 
-      delete_task_records! rescue nil
+      unless is_solo?
+        delete_task_records! rescue nil
+      end
     rescue Exception => e
     end
   end
 
-  def handle_task_completed!
+  def handle_task_completed!(job_args)
     return false unless self.skyrunner_job_id
 
-    record = dynamo_db_record
-    new_attributes = nil
+    unless is_solo?
+      record = dynamo_db_record
+      new_attributes = nil
 
-    SkyRunner::retry_dynamo_db do
-      new_attributes = record.attributes.add({ completed_tasks: 1 }, return: :all_new)
+      SkyRunner::retry_dynamo_db do
+        new_attributes = record.attributes.add({ completed_tasks: 1 }, return: :all_new)
+      end
     end
 
-    if new_attributes["total_tasks"] == new_attributes["completed_tasks"]
+    if is_solo? || new_attributes["total_tasks"] == new_attributes["completed_tasks"]
       begin
-        if_condition = { completed_tasks: new_attributes["total_tasks"], done: 0 }
+        unless is_solo?
+          if_condition = { completed_tasks: new_attributes["total_tasks"], done: 0 }
 
-        SkyRunner::retry_dynamo_db do
-          record.attributes.update(if: if_condition) do |u|
-            u.add(done: 1)
+          SkyRunner::retry_dynamo_db do
+            record.attributes.update(if: if_condition) do |u|
+              u.add(done: 1)
+            end
           end
         end
 
@@ -150,11 +181,13 @@ module SkyRunner::Job
           if self.method(method).arity == 0 && self.method(method).parameters.size == 0
             self.send(method)
           else
-            self.send(method, JSON.parse(record.attributes["args"]).symbolize_keys)
+            self.send(method, job_args.symbolize_keys)
           end
         end
 
-        delete_task_records! rescue nil
+        unless is_solo?
+          delete_task_records! rescue nil
+        end
       rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException => e
         # This is OK, we had a double finisher so lets block them.
       end
@@ -164,6 +197,8 @@ module SkyRunner::Job
   end
 
   def delete_task_records!
+    return if is_solo?
+
     delete_batch_queue = Queue.new
     mutex = Mutex.new
     delete_items_queued = false

data/lib/skyrunner/version.rb

@@ -1,3 +1,3 @@
 module Skyrunner
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end

data/lib/skyrunner.rb

@@ -83,23 +83,26 @@ module SkyRunner
               next
             end
 
-            klass, job_id, task_id, task_args, message = local_queue.pop
+            klass, job_id, task_id, job_args, task_args, is_solo, message = local_queue.pop
 
             if klass
               begin
-                # Avoid running the same task twice, enter record and raise error if exists already.
+                unless is_solo
+                  # Avoid running the same task twice, enter record and raise error if exists already.
 
-                SkyRunner::retry_dynamo_db do
-                  table.items.put({ id: "#{job_id}-tasks", task_id: task_id }, unless_exists: ["id", "task_id"])
+                  SkyRunner::retry_dynamo_db do
+                    table.items.put({ id: "#{job_id}-tasks", task_id: task_id }, unless_exists: ["id", "task_id"])
+                  end
                 end
 
                 SkyRunner::log :info, "Run Task: #{task_args} Job: #{job_id} Message: #{message.id}"
 
                 job = klass.new
                 job.skyrunner_job_id = job_id
+                job.skyrunner_job_is_solo = is_solo
 
                 begin
-                  job.consume!(task_args)
+                  job.consume!(job_args, task_args)
                   message.delete
                 rescue Exception => e
                   message.delete rescue nil
@@ -138,14 +141,16 @@ module SkyRunner
 
             next unless received_messages.size > 0
 
-            job_ids = received_messages.map { |m| [m[1]["job_id"], m[1]["job_id"]] }.uniq
+            job_ids = received_messages.select { |m| !m[1]["is_solo"] }.map { |m| [m[1]["job_id"], m[1]["job_id"]] }.uniq
 
             job_records = {}
 
-            SkyRunner::retry_dynamo_db do
-              # Read DynamoDB records into job and task lookup tables.
-              table.batch_get(["id", "task_id", "failed"], job_ids.uniq, consistent_read: true) do |record|
-                job_records[record["id"]] = record
+            if job_ids.uniq.size > 0
+              SkyRunner::retry_dynamo_db do
+                # Read DynamoDB records into job and task lookup tables.
+                table.batch_get(["id", "task_id", "failed"], job_ids.uniq, consistent_read: true) do |record|
+                  job_records[record["id"]] = record
+                end
               end
             end
 
@@ -153,14 +158,16 @@ module SkyRunner
               message, message_data = received_message
               job_id = message_data["job_id"]
               task_id = message_data["task_id"]
+              is_solo = message_data["is_solo"]
+              job_args = message_data["job_args"]
+              task_args = message_data["task_args"]
 
               job_record = job_records[job_id]
 
-              if job_record && job_record["failed"] == 0
+              if is_solo || (job_record && job_record["failed"] == 0)
                 begin
                   klass = Kernel.const_get(message_data["job_class"])
-                  task_args = message_data["task_args"]
-                  local_queue.push([klass, job_id, task_id, task_args, message])
+                  local_queue.push([klass, job_id, task_id, job_args, task_args, is_solo, message])
                 rescue NameError => e
                   block.call(e) if block_given?
                   message.delete rescue nil
@@ -187,8 +194,12 @@ module SkyRunner
   end
 
   def self.dynamo_db_table
+    table = Thread.current.thread_variable_get(:skyrunner_dyn_table)
+    return table if table 
+
     dynamo_db.tables[SkyRunner.dynamo_db_table_name].tap do |table|
       table.load_schema if table && table.exists?
+      Thread.current.thread_variable_set(:skyrunner_dyn_table, table)
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: skyrunner
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Greg Fodor