skyrunner 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f470ededf6f80989cfd681b1948cbb4133096af7
-  data.tar.gz: 68e4bd35332959a1fbcaace10c327618906c8062
+  metadata.gz: 51e28867dae3ad56df63869dbf4b894350f0a848
+  data.tar.gz: 4a3483978b3bdec3504ec4ff5fdaf75a98e3da48
 SHA512:
-  metadata.gz: ddd6de5dc9e50e1fc3b59eb85316194ea177b97d27ee658d244199ecdc29e3f3023ca48ef60ab96445ccf469e99459b57c31663176de51cd1a811292e5adfcfd
-  data.tar.gz: 8a42bcaada30b93c13b3eacba76de78b206e19714a4a6ddbfd618829fb1b699547c54c2054245c2fd92663f50a2abeb742bb373c8482e1986214384bca055abd
+  metadata.gz: 3b97035c04bed836c34d2f6d102f74906e3eaa3b6b3a7e89667f28df9c8ba76a7433db4b3d7e57bcb75dce8e1c25519f4708586b904c6bb63d02e3c054242a4a
+  data.tar.gz: 5394bd636e6bcb4ea5b720b3265098aedcbe01a29e301085a7b1262ea8a29298c14c1d736c5663e18061e1198915684dbc07cf5d49712557293cc063737b09a0

data/README.md

@@ -39,7 +39,7 @@ To gracefully shut down a consumer, send it SIGINT (or hit Ctrl-C if you have a
 
 See `jobs/example_job.rb` for an example job. To run a job, just call `execute!` on the job, passing any named job arguments you want. The job class should implement the method `run`. This method will get passed the job arguments. For each task you want consumers to run, `run` should yield an array of two elements, the first being the name of the method on the job class to run for the task, and the second a Hash of method arguments. Note that the consumer is (by default) multi-threaded, so please be sure your task methods are thread-safe.
 
-You can specify `on_complete` and `on_failure` method(s) to call when the tasks are all completed, or if any of them fail, respectively. These methods will also be passed the original job arguments. Importantly, the completion method is guaranteed to be called once and only once, when the final task has been completed. If your job runs multiple tasks, SkyRunner will coordinate the job via DynamoDB and the last consumer will run the completion method. If your job runs a single task, DynamoDB is not needed for coordination and the consumer which consumes your task will call the completion or failure method.
+You can specify `on_completed` and `on_failed` method(s) to call when the tasks are all completed, or if any of them fail, respectively. These methods will also be passed the original job arguments. Importantly, the completion method is guaranteed to be called once and only once, when the final task has been completed. If your job runs multiple tasks, SkyRunner will coordinate the job via DynamoDB and the last consumer will run the completion method. If your job runs a single task, DynamoDB is not needed for coordination and the consumer which consumes your task will call the completion or failure method.
 
 If any of your tasks fail in a job, consumers will stop consuming tasks for that job and deplete any queued tasks on SQS. SkyRunner does not currently retry failed tasks, for now you'll need to implement your own retry logic in your task method instead.
 

data/bin/skyrunner

@@ -36,11 +36,11 @@ opts = Trollop::options do
   opt :num_threads, "Number of consumer threads.", default: 10
 end
 
-SkyRunner.dynamo_db_table_name = opts[:dynamo_db_table_name]
-SkyRunner.sqs_queue_name = opts[:sqs_queue_name]
-SkyRunner.consumer_threads = opts[:num_threads].to_i
+SkyRunner::dynamo_db_table_name = opts[:dynamo_db_table_name]
+SkyRunner::sqs_queue_name = opts[:sqs_queue_name]
+SkyRunner::consumer_threads = opts[:num_threads].to_i
 
-COMMANDS = ["init", "purge", "consume", "test"]
+COMMANDS = ["init", "purge", "consume", "test", "test_local"]
 
 Trollop::die "Must specify command" unless COMMANDS.include?(ARGV[0])
 
@@ -53,6 +53,12 @@ when "purge"
   SkyRunner.init!(purge: true)
 when "consume"
   SkyRunner.consume!
+when "test_local"
+  $: << "."
+  require "#{File.dirname(__FILE__)}/../jobs/example_job"
+
+  SkyRunner.run_locally = true
+  ExampleJobModule::ExampleJob.new.execute!(number_of_tasks: 100)
 when "test"
   $: << "."
   require "#{File.dirname(__FILE__)}/../jobs/example_job"

data/lib/skyrunner/job.rb

@@ -33,6 +33,8 @@ module SkyRunner::Job
   end
 
   def execute!(job_args = {})
+    return execute_local!(job_args) if SkyRunner::run_locally
+
     job_id = SecureRandom.hex
     self.skyrunner_job_id = job_id
 
@@ -120,8 +122,46 @@ module SkyRunner::Job
     end
   end
 
+  def fire_post_event_method(event_type, job_args)
+    (self.class.job_event_methods[event_type] || []).each do |method|
+      if self.method(method).arity == 0 && self.method(method).parameters.size == 0
+        self.send(method)
+      else
+        self.send(method, job_args.symbolize_keys)
+      end
+    end
+  end
+
   private 
 
+  def execute_local!(job_args = {})
+    job_id = SecureRandom.hex
+    self.skyrunner_job_id = job_id
+    task_arg_list = []
+
+    self.run(job_args) do |*task_args|
+      task_arg_list << task_args
+    end
+
+    task_arg_list.shuffle!
+
+    task_arg_list.each_with_index do |task_args, task_index|
+      task = self.class.new
+      task.skyrunner_job_id = job_id
+
+      begin
+        task.send(task_args[0].to_sym, task_args[1].symbolize_keys)
+      rescue Exception => e
+        task.fire_post_event_method(:failed, job_args)
+        break
+      end
+
+      if task_index == task_arg_list.size - 1
+        task.fire_post_event_method(:completed, job_args)
+      end
+    end
+  end
+
   def dynamo_db_record
     SkyRunner.dynamo_db_table.items[self.skyrunner_job_id, self.skyrunner_job_id]
   end
@@ -138,13 +178,7 @@ module SkyRunner::Job
         end
       end
 
-      (self.class.job_event_methods[:failed] || []).each do |method|
-        if self.method(method).arity == 0 && self.method(method).parameters.size == 0
-          self.send(method)
-        else
-          self.send(method, job_args.symbolize_keys)
-        end
-      end
+      self.fire_post_event_method(:failed, job_args)
 
       unless is_solo?
         delete_task_records! rescue nil
@@ -178,13 +212,7 @@ module SkyRunner::Job
           end
         end
 
-        (self.class.job_event_methods[:completed] || []).each do |method|
-          if self.method(method).arity == 0 && self.method(method).parameters.size == 0
-            self.send(method)
-          else
-            self.send(method, job_args.symbolize_keys)
-          end
-        end
+        self.fire_post_event_method(:completed, job_args)
 
         unless is_solo?
           delete_task_records! rescue nil
@@ -205,7 +233,7 @@ module SkyRunner::Job
     delete_items_queued = false
     threads = []
 
-    1.upto([1, (SkyRunner.consumer_threads / 4.0).floor].max) do 
+    1.upto([1, (SkyRunner::consumer_threads / 4.0).floor].max) do 
       threads << Thread.new do
 
         db_table = SkyRunner.dynamo_db_table

data/lib/skyrunner/version.rb

@@ -1,3 +1,3 @@
 module Skyrunner
-  VERSION = "0.1.8"
+  VERSION = "0.1.9"
 end

data/lib/skyrunner.rb

@@ -20,7 +20,7 @@ module SkyRunner
     table = self.dynamo_db_table
 
     if !table.exists? || params[:purge]
-      table_name = SkyRunner.dynamo_db_table_name
+      table_name = SkyRunner::dynamo_db_table_name
 
       if table.exists? && params[:purge]
         SkyRunner.log :warn, "Purging DynamoDB table #{table_name}."
@@ -32,8 +32,8 @@ module SkyRunner
       SkyRunner.log :info, "Creating DynamoDB table #{table_name}."
 
       table = dynamo_db.tables.create(table_name, 
-                                     SkyRunner.dynamo_db_read_capacity, 
-                                     SkyRunner.dynamo_db_write_capacity,
+                                     SkyRunner::dynamo_db_read_capacity, 
+                                     SkyRunner::dynamo_db_write_capacity,
                                      hash_key: { id: :string },
                                      range_key: { task_id: :string })
 
@@ -43,7 +43,7 @@ module SkyRunner
     queue = self.sqs_queue
 
     if !queue || params[:purge]
-      queue_name = SkyRunner.sqs_queue_name
+      queue_name = SkyRunner::sqs_queue_name
 
       if queue && params[:purge]
         SkyRunner.log :warn, "Purging SQS queue #{queue_name}. Waiting 65 seconds to re-create."
@@ -55,8 +55,8 @@ module SkyRunner
       SkyRunner.log :info, "Creating SQS queue #{queue_name}."
 
       queue = sqs.queues.create(queue_name, 
-                                visibility_timeout: SkyRunner.sqs_visibility_timeout,
-                                message_retention_period: SkyRunner.sqs_message_retention_period)
+                                visibility_timeout: SkyRunner::sqs_visibility_timeout,
+                                message_retention_period: SkyRunner::sqs_message_retention_period)
     end
 
     true
@@ -131,7 +131,7 @@ module SkyRunner
 
             break if SkyRunner::stop_consuming?
 
-            sleep 1 while local_queue.size >= SkyRunner.consumer_threads
+            sleep 1 while local_queue.size >= SkyRunner::consumer_threads
 
             received_messages = []
 
@@ -197,7 +197,7 @@ module SkyRunner
     table = Thread.current.thread_variable_get(:skyrunner_dyn_table)
     return table if table 
 
-    dynamo_db.tables[SkyRunner.dynamo_db_table_name].tap do |table|
+    dynamo_db.tables[SkyRunner::dynamo_db_table_name].tap do |table|
       table.load_schema if table && table.exists?
       Thread.current.thread_variable_set(:skyrunner_dyn_table, table)
     end
@@ -205,14 +205,14 @@ module SkyRunner
 
   def self.sqs_queue
     begin
-      sqs.queues.named(SkyRunner.sqs_queue_name)
+      sqs.queues.named(SkyRunner::sqs_queue_name)
     rescue AWS::SQS::Errors::NonExistentQueue => e
       return nil
     end
   end
 
   def self.log(type, message)
-    SkyRunner.logger.send(type, "[SkyRunner] #{message}")
+    SkyRunner::logger.send(type, "[SkyRunner] #{message}")
   end
 
   mattr_accessor :dynamo_db_table_name
@@ -239,6 +239,9 @@ module SkyRunner
   mattr_accessor :consumer_threads
   @@consumer_threads = 10
 
+  mattr_accessor :run_locally
+  @@run_locally = false
+
   mattr_accessor :stop_consuming_flag
 
   @@stop_consuming_mutex = Mutex.new

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skyrunner
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - Greg Fodor
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-21 00:00:00.000000000 Z
+date: 2013-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler