cloudtasker 0.12.rc9 → 0.12.rc10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50cd6021ad7511d7b5b385a6086eb315d9381a4f43909a6e1e9a0defbc679ea4
-  data.tar.gz: dc16cc1330b14b46a5d6ade9f9c171981fd3aef5b3db67dd474e29548acc177f
+  metadata.gz: 5763bef46a0c554549150326375c3fb49c9eb77193b6862334dc1da72a5fe34f
+  data.tar.gz: 211272f9129642cb8c7260f639a6a76714161ac37c87dc5f073633a072781b44
 SHA512:
-  metadata.gz: 30feac9331b9e8113e71af6c44a20d2f47fcbd41652a0d345ab8808d272bd14271ebb4a98711e42b238f6202fa8023a37c543e0f4ec7d9f9f7a46cc2732d96aa
-  data.tar.gz: 9b70a3c0733b2c510fafe48232c65bb79f5ad4da6b90fbcff86c0736fee590fab5f89846df9c41855dc7084460425660ee42799975975775b2239b9b8997171a
+  metadata.gz: ac4012191c2878256abdc446eabc4cd4cf4ff822e426fc7a89e25c77ab2724c5b1d2e508fb88092ddde86680512797c50711d4188798a281cf275b3e03304a0e
+  data.tar.gz: 7a1558eed571862501a3e5a264cfa006014e9454c0f31cb915c9d5229aed6a9c47f539287cb447a226db4302b60d761496256791d9844cc8291164910ec8d6f5

data/CHANGELOG.md

@@ -1,8 +1,8 @@
 # Changelog
 
-## Latest RC [v0.12.rc8](https://github.com/keypup-io/cloudtasker/tree/v0.12.rc8) (2021-04-06)
+## Latest RC [v0.12.rc10](https://github.com/keypup-io/cloudtasker/tree/v0.12.rc10) (2021-05-31)
 
-[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.11.0...v0.12.rc8)
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.11.0...v0.12.rc10)
 
 **Improvements:**
 - ActiveJob: do not double log errors (ActiveJob has its own error logging)
@@ -10,11 +10,14 @@
 - Batch state: use native Redis hashes to store batch state instead of a serialized hash in a string key
 - Batch progress: restrict calculation to direct children by default. Allow depth to be specified. Calculating progress using all tree jobs created significant delays on large batches.
 - Batch redis usage: cleanup batches as they get completed or become dead to avoid excessive redis usage with large batches.
+- Batch expansion: Inject `parent_batch` in jobs. Can be used to expand the parent batch the job is in.
 - Configuration: allow configuration of Cloud Tasks `dispatch deadline` at global and worker level
 - Cron jobs: Use Redis Sets instead of key pattern matching for resource listing
 - Error logging: Use worker logger so as to include context (job args etc.)
 - Error logging: Do not log exception and stack trace separately, combine them instead.
 - Local server: Use Redis Sets instead of key pattern matching for resource listing
+- Local server: Guard against nil tasks to prevent job daemon failures
+- Performance: remove use of redis locks and rely on atomic transactions instead for Batch and Unique Job.
 - Worker: raise DeadWorkerError instead of MissingWorkerArgumentsError when arguments are missing. This is more consistent with what middlewares expect.
 - Worker redis usage: delete redis payload storage once the job is successful or dead instead of expiring the key.
 

data/docs/BATCH_JOBS.md

@@ -18,7 +18,7 @@ Cloudtasker.configure do |config|
 end
 ```
 
-## Example
+## Example: Creating a new batch
 
 The following example defines a worker that adds itself to the batch with different arguments then monitors the success of the batch.
 
@@ -47,6 +47,38 @@ class BatchWorker
 end
 ```
 
+## Example: Expanding the parent batch
+**Note**: `parent_batch` is available since `0.12.rc10`
+
+```ruby
+# All the jobs will be attached to the top parent batch.
+class BatchWorker
+  include Cloudtasker::Worker
+
+  def perform(level, instance)
+    # Use existing parent_batch or create a new one
+    current_batch = parent_batch || batch
+
+    3.times { |n| current_batch.add(self.class, level + 1, n) } if level < 2
+  end
+
+  # Invoked when any descendant (e.g. sub-sub job) is complete
+  def on_batch_node_complete(child)
+    logger.info("Direct or Indirect child complete: #{child.job_id}")
+  end
+
+  # Invoked when a direct descendant is complete
+  def on_child_complete(child)
+    logger.info("Direct child complete: #{child.job_id}")
+  end
+
+  # Invoked when all chidren have finished
+  def on_batch_complete
+    Rails.logger.info("Batch complete")
+  end
+end
+```
+
 ## Available callbacks
 
 The following callbacks are available on your workers to track the progress of the batch:

data/lib/cloudtasker/backend/redis_task.rb

@@ -39,7 +39,7 @@ module Cloudtasker
       def self.all
         if redis.exists?(key)
           # Use Schedule Set if available
-          redis.smembers(key).map { |id| find(id) }
+          redis.smembers(key).map { |id| find(id) }.compact
         else
           # Fallback to redis key matching and migrate tasks
           # to use Task Set instead.

data/lib/cloudtasker/batch/extension/worker.rb

@@ -6,7 +6,7 @@ module Cloudtasker
       # Include batch related methods onto Cloudtasker::Worker
       # See: Cloudtasker::Batch::Middleware#configure
       module Worker
-        attr_accessor :batch
+        attr_accessor :batch, :parent_batch
       end
     end
   end

data/lib/cloudtasker/batch/job.rb

@@ -73,8 +73,12 @@ module Cloudtasker
         # Load extension if not loaded already on the worker class
         worker.class.include(Extension::Worker) unless worker.class <= Extension::Worker
 
-        # Add batch capability
+        # Add batch and parent batch to worker
         worker.batch = new(worker)
+        worker.parent_batch = worker.batch.parent_batch
+
+        # Return the batch
+        worker.batch
       end
 
       #
@@ -259,9 +263,7 @@ module Cloudtasker
         migrate_batch_state_to_redis_hash
 
         # Update the batch state batch_id entry with the new status
-        redis.with_lock("#{batch_state_gid}/#{batch_id}", max_wait: BATCH_MAX_LOCK_WAIT) do
-          redis.hset(batch_state_gid, batch_id, status) if redis.hexists(batch_state_gid, batch_id)
-        end
+        redis.hset(batch_state_gid, batch_id, status) if redis.hexists(batch_state_gid, batch_id)
       end
 
       #
@@ -273,10 +275,7 @@ module Cloudtasker
         migrate_batch_state_to_redis_hash
 
         # Check that all child jobs have completed
-        redis.with_lock(batch_state_gid, max_wait: BATCH_MAX_LOCK_WAIT) do
-          # Check that all children are complete
-          redis.hvals(batch_state_gid).all? { |e| COMPLETION_STATUSES.include?(e) }
-        end
+        redis.hvals(batch_state_gid).all? { |e| COMPLETION_STATUSES.include?(e) }
       end
 
       #
@@ -427,8 +426,11 @@ module Cloudtasker
         # Perform job
         yield
 
-        # Save batch (if child workers have been enqueued)
-        setup
+        # Save batch if child jobs added
+        setup if jobs.any?
+
+        # Save parent batch if batch expanded
+        parent_batch&.setup if parent_batch&.jobs&.any?
 
         # Complete batch
         complete(:completed)

data/lib/cloudtasker/local_server.rb

@@ -84,7 +84,7 @@ module Cloudtasker
 
       # Deliver task
       begin
-        Thread.current['task'].deliver
+        Thread.current['task']&.deliver
       rescue Errno::EBADF, Errno::ECONNREFUSED => e
         raise(e) unless Thread.current['attempts'] < 3
 

data/lib/cloudtasker/unique_job/job.rb

@@ -149,25 +149,18 @@ module Cloudtasker
       # if taken by another job.
       #
       def lock!
-        redis.with_lock(unique_gid) do
-          locked_id = redis.get(unique_gid)
+        lock_acquired = redis.set(unique_gid, id, nx: true, ex: lock_ttl)
+        lock_already_acquired = !lock_acquired && redis.get(unique_gid) == id
 
-          # Abort job lock process if lock is already taken by another job
-          raise(LockError, locked_id) if locked_id && locked_id != id
-
-          # Take job lock if the lock is currently free
-          redis.set(unique_gid, id, ex: lock_ttl) unless locked_id
-        end
+        raise(LockError) unless lock_acquired || lock_already_acquired
       end
 
       #
       # Delete the job lock.
       #
       def unlock!
-        redis.with_lock(unique_gid) do
-          locked_id = redis.get(unique_gid)
-          redis.del(unique_gid) if locked_id == id
-        end
+        locked_id = redis.get(unique_gid)
+        redis.del(unique_gid) if locked_id == id
       end
     end
   end

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.12.rc9'
+  VERSION = '0.12.rc10'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.12.rc9
+  version: 0.12.rc10
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-04-19 00:00:00.000000000 Z
+date: 2021-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport