tq 0.1.6 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f1189c610b90b4d6e1af73a3e075981588797bff
-  data.tar.gz: dd0f3598d6261f985e8830d0bb73d848f29fce0c
+  metadata.gz: 877e9730d05610f3b211c7a0d69817f60149bde4
+  data.tar.gz: eb2d73a1f7e5a63659fe53041171bde055e59523
 SHA512:
-  metadata.gz: 0684ba5f822f4ee215c131d7c56d0f9f939309b8faa85c8cebcf140da75b9e0212586dac3e093acffb3b224af28c2d855aa23c3dbf4c3171e9678bb6b8c01bab
-  data.tar.gz: a178c5c079388733ad77f4e075bd1ecc14346a48aad514275a48d5a08161dbbe30ecf77dffef32c0d7ca8a5cc3f8ddf57d26752e4f268362cca0e691f35c5d6d
+  metadata.gz: 9c2539a74728b068c962eab6e1637559664e1a5a74514623893b756166c1246641c5b33dc5892e671ada06a7e2760bba2453137b0ba98f74813c729bbdbaed3b
+  data.tar.gz: 2981fa8c831cf086873d2d9dfd2fef886533dc01baaa5521b0d4ea2f6f6a701ed37a165299f558a92ba3397d98ddab54415f1658bad433192335a261063b9b45

data/lib/tq/app.rb

@@ -155,7 +155,11 @@ module TQ
     def _run(qin, qout, qerr)
       tasks = qin.lease!
       Parallel.each(tasks, :in_threads => @options['concurrency']) do |task| 
-        @worker.new(qin, qout, qerr, inherited_env).call(task)
+        if task.try?
+          @worker.new(qin, qout, qerr, inherited_env).call(task)
+        else
+          qin.finish!(task)
+        end
       end
     end
 

data/lib/tq/queue.rb

@@ -5,7 +5,11 @@ module TQ
 
   class Queue
    
-    DEFAULT_OPTIONS = { 'lease_secs' => 60, 'num_tasks' => 1 }
+    DEFAULT_OPTIONS = { 
+      'lease_secs' => 60, 
+      'num_tasks' => 1, 
+      'max_tries' => -1 
+    }
 
     attr_reader :client, :api
     def initialize(client, api, options={})
@@ -25,6 +29,10 @@ module TQ
       options({'name' => _})
     end
 
+    def option(key)
+      @options[key]
+    end
+
     def lease!(opts={})
       opts = @options.merge(opts)
       results = client.execute!(  
@@ -86,7 +94,15 @@ module TQ
     private
     
     def new_task(t)
-      Task.new(self, t['id'], timestamp_time(t['leaseTimestamp']), decode(t.payloadBase64), t) 
+      Task.new(
+        self, 
+        t['id'], 
+        timestamp_time(t['leaseTimestamp']),
+        t['retry_count'],
+        t['tag'],
+        decode(t.payloadBase64), 
+        t
+      ) 
     end
 
     def timestamp_time(t)
@@ -103,7 +119,7 @@ module TQ
     
   end
 
-  class Task < Struct.new(:queue, :id, :expires, :payload, :raw)
+  class Task < Struct.new(:queue, :id, :expires, :tries, :tag, :payload, :raw)
     
     def initialize(*args)
       super
@@ -134,6 +150,11 @@ module TQ
       self.expires < @clock.now
     end
 
+    def try?
+      max = self.queue.option('max_tries')
+      return (max == -1 or self.tries < max)
+    end
+
   end
   
 end

data/lib/version.rb

@@ -1,4 +1,4 @@
 
 module TQ
-  VERSION = '0.1.6'
+  VERSION = '0.2.1'
 end

data/test/helper.rb

@@ -26,18 +26,33 @@ module TestUtils
     attr_reader :project, :queue
     def initialize(project,queue)
       @project, @queue = project, queue
+      @service_auth = false
     end
 
     def auth_files(secrets,creds)
       @secrets_file, @creds_file = secrets, creds
+      @service_auth = false
+      return self
+    end
+    
+    def service_auth_files(issuer,p12)
+      @secrets_issuer, @secrets_p12 = issuer, p12
+      @service_auth = true
       return self
     end
 
     def authorized_client
-      # @authorized_client ||= TQ::App.new('test_app',nil).service_auth!(File.read(SERVICE_ISSUER_FILE).chomp, SERVICE_P12_FILE)
-      @authorized_client ||= TQ::App.new('test_app',nil).auth!(@secrets_file, @creds_file)
+      @authorized_client ||= (
+        if @service_auth
+          TQ::App.new('test_app',nil).service_auth!(
+            File.read(@secrets_issuer).chomp, @secrets_p12
+          )
+        else
+          TQ::App.new('test_app',nil).auth!(@secrets_file, @creds_file)
+        end
+      )
     end
-    
+
     # Note: inaccurate, don't use
     def peek()
       client, api = authorized_client

data/test/test_run.rb

@@ -39,7 +39,7 @@ class AppRunTests < Minitest::Spec
 
 
   def queue_helper(project,queue)
-    TestUtils::QueueHelper.new(project,queue).auth_files(CLIENT_SECRETS_FILE, CREDENTIALS_FILE)
+    TestUtils::QueueHelper.new(project,queue).service_auth_files(SERVICE_ISSUER_FILE, SERVICE_P12_FILE)
   end
 
   def tasks_on_queue(project,queue)
@@ -54,6 +54,10 @@ class AppRunTests < Minitest::Spec
     q = queue_helper(project,queue)
     tasks.each do |task| q.push!(task) end
   end
+ 
+  def service_run!(app)
+    app.service_run! File.read(SERVICE_ISSUER_FILE).chomp, SERVICE_P12_FILE
+  end
 
 
   describe "run!" do
@@ -113,7 +117,8 @@ class AppRunTests < Minitest::Spec
       app = TQ::App.new('test_app/0.0.0', mock_handler_class)
                .project(TASKQUEUE_PROJECT_ID)
                .stdin({ 'name' => 'test', 'num_tasks' => expected_calls, 'lease_secs' => TASKQUEUE_LEASE_SECS })
-      app.run! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+      # app.run! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+      service_run! app
 
       # assertions
       assert_equal expected_calls, actual_calls,
@@ -141,8 +146,9 @@ class AppRunTests < Minitest::Spec
       app = TQ::App.new('test_app/0.0.0', DoNothingWorker)
                .project(TASKQUEUE_PROJECT_ID)
                .stdin({ 'name' => 'test', 'num_tasks' => 1, 'lease_secs' => TASKQUEUE_LEASE_SECS })
-      app.run! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
-    
+      # app.run! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+      service_run! app
+
       sleep TASKQUEUE_LEASE_SECS + 1
       actual_tasks = clear_queue!(TASKQUEUE_PROJECT_ID,'test')
 
@@ -182,7 +188,8 @@ class AppRunTests < Minitest::Spec
                .project(TASKQUEUE_PROJECT_ID)
                .stdin({ 'name' => 'test', 'num_tasks' => 1, 'lease_secs' => TASKQUEUE_LEASE_SECS })
                .stdout({ 'name' => 'log' })
-      app.run! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+      # app.run! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+      service_run! app
     
       sleep TASKQUEUE_LEASE_SECS + 1
       actual_output_tasks = clear_queue!(TASKQUEUE_PROJECT_ID,'log')
@@ -228,10 +235,119 @@ class AppRunTests < Minitest::Spec
       app = TQ::App.new('test_app/0.0.0', ExtendWorker)
                .project(TASKQUEUE_PROJECT_ID)
                .stdin({ 'name' => 'test', 'num_tasks' => 1, 'lease_secs' => TASKQUEUE_LEASE_SECS + 2 })
-      app.run! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+      # app.run! CLIENT_SECRETS_FILE, CREDENTIALS_FILE
+      service_run! app
     
     end
 
+    it 'should retry tasks indefinitely by default' do
+
+      # setup
+      expected_tasks = [
+        { 'What is your name?' => 'Sir Lancelot', 'What is your quest?' => 'To seek the holy grail', 'What is your favorite color?' => 'blue' }
+      ]
+      push_tasks!(TASKQUEUE_PROJECT_ID,'test', expected_tasks)
+
+      # expectations
+      mock_handler_class = MiniTest::Mock.new
+      mock_handler = MiniTest::Mock.new
+
+      ## expect constructor receives task queue as first param 
+      expected_calls = 3
+      (0...expected_calls).each do
+        mock_handler_class.expect(:new, mock_handler) do |*args|
+          args.first.respond_to?('finish!')  
+        end
+      end
+
+      ## expect :call for each queued task up to expected_calls
+      actual_calls = 0 
+      (0...expected_calls).each do 
+        mock_handler.expect(:call, true) do |actual_task|
+          actual_calls += 1
+        end
+      end
+
+      # execution
+      app = TQ::App.new('test_app/0.0.0', mock_handler_class)
+               .project(TASKQUEUE_PROJECT_ID)
+               .stdin({ 'name' => 'test', 
+                        'num_tasks' => 1, 
+                        'lease_secs' => TASKQUEUE_LEASE_SECS
+                      })
+               .stdout({ 'name' => 'log' })
+      
+      service_run! app
+      sleep TASKQUEUE_LEASE_SECS + 1
+      service_run! app
+      sleep TASKQUEUE_LEASE_SECS + 1
+      service_run! app
+
+      # assertion
+
+      assert_equal expected_calls, actual_calls,
+        "Expected #{expected_calls} worker calls, was #{actual_calls}"
+
+      mock_handler_class.verify
+      mock_handler.verify
+
+    end
+
+    it 'should only run if current task retry count is less than queue max_tries' do
+
+      # setup
+      expected_tasks = [
+        { 'What is your name?' => 'Sir Lancelot', 'What is your quest?' => 'To seek the holy grail', 'What is your favorite color?' => 'blue' }
+      ]
+      push_tasks!(TASKQUEUE_PROJECT_ID,'test', expected_tasks)
+
+      # expectations
+      mock_handler_class = MiniTest::Mock.new
+      mock_handler = MiniTest::Mock.new
+
+      ## expect constructor receives task queue as first param 
+      max_tries = 2
+      expected_calls = max_tries
+      (0...expected_calls).each do
+        mock_handler_class.expect(:new, mock_handler) do |*args|
+          args.first.respond_to?('finish!')  
+        end
+      end
+
+      ## expect :call for each queued task up to expected_calls
+      actual_calls = 0 
+      (0...max_tries).each do 
+        mock_handler.expect(:call, true) do |actual_task|
+          actual_calls += 1
+        end
+      end
+
+      # execution
+      app = TQ::App.new('test_app/0.0.0', mock_handler_class)
+               .project(TASKQUEUE_PROJECT_ID)
+               .stdin({ 'name' => 'test', 
+                        'num_tasks' => 1, 
+                        'lease_secs' => TASKQUEUE_LEASE_SECS,
+                        'max_tries' => max_tries
+                      })
+               .stdout({ 'name' => 'log' })
+      
+      service_run! app
+      sleep TASKQUEUE_LEASE_SECS + 1
+      service_run! app
+      sleep TASKQUEUE_LEASE_SECS + 1
+      service_run! app
+
+      # assertion
+      assert_equal expected_calls, actual_calls,
+        "Expected #{expected_calls} worker calls, was #{actual_calls}"
+
+      mock_handler_class.verify
+      mock_handler.verify
+
+    end
+
+
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tq
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.1
 platform: ruby
 authors:
 - Eric Gjertsen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-03 00:00:00.000000000 Z
+date: 2016-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -91,7 +91,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Ruby client for Google App Engine TaskQueue (REST API)