standard_procedure_operations 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9a47b950a0831c0bf0a1be69de7b7f75c80237ec73a75673397a0b8112aa846
-  data.tar.gz: 62e9741fcd4053d0f4139aa956f5d51025d644188b2da460d6b7550873257030
+  metadata.gz: c254fd322931c448af32ef7aef5bfba2f09293b5d2ab2b85d069783be09cb8e3
+  data.tar.gz: e329a253576b4c4756793a068eedbb275c9ddab4ce0af4059285c0d13ad5dffe
 SHA512:
-  metadata.gz: 611b50e5912231a593187ea4dc59c5e270674f65413b1c3bbd34d3e1f40849f817954282750d729bdadf23a1bacb35b202c9fb71b2edaccfa59dddd209746234
-  data.tar.gz: 5b4fbf04648dfe9347330385f1dfdfc32eeadfe4adbb89a015a562c79b0a7b7f6e6b0e91b8d65f3d934e7f3af19bbaf8856ea39555cdd1b649f2ae9f2937ab81
+  metadata.gz: 920f49215031fce81eabe5dd4aec50df392d1dfd93cc497eb47aafbb4544ab9defd378faf0bb99d88b0c5c5fbf326e2dd2e77df6f2d8c737589944e174c0e740
+  data.tar.gz: 2c5c2cc235ed70113c7881fb75f46a9962548bd333d589cc73cbcbc5fbeb50f030b5212e2e2bbd16b7b4b3509284aa1b8e04d151b84cfde07780c4da5f9a9e5c

data/README.md

@@ -295,7 +295,7 @@ For example, you create your task as:
 @alice = User.find 123
 @task = DoSomethingImportant.call user: @alice 
 ```
-There will not be a `TaskParticipant` record with a `context` of "data", `role` of "user" and `participant` of `@alice`.  
+There will be a `TaskParticipant` record with a `context` of "data", `role` of "user" and `participant` of `@alice`.  
 
 Likewise, you can see all the tasks that Alice was involved with using: 
 ```ruby
@@ -470,6 +470,16 @@ class WaitForSomething < Operations::Task
 end
 ```
 
+#### Zombie tasks
+
+There's a chance that the `Operations::TaskRunnerJob` might get lost - maybe there's a crash in some process and the job does not restart correctly.  As the process for handling background tasks relies on the task "waking up", performing the next action, then queuing up the next task-runner, if the background job does not queue as expected, the task will sit there, waiting forever.  
+
+To monitor for this, every task can be checked to see if it is a `zombie?`.  This means that the current time is more than 3 times the expected delay, compared to the `updated_at` field.  So if the `delay` is set to 1 minute and the task last woke up more than 3 minutes ago, it is classed as a zombie.  
+
+There are two ways to handle zombies.  
+- Manually; add a user interface listing your tasks with a "Restart" button.  The "Restart" button calls `restart` on the task (which internally schedules a new task runner job).
+- Automatically; set up a cron job which calls the `operations:restart_zombie_tasks` rake task.  This rake task searches for zombie jobs and calls `restart` on them.  Note that cron jobs have a minimum resolution of 1 minute so this will cause pauses in tasks with a delay measured in seconds.   Also be aware that a cron job that calls a rake task will load the entire Rails stack as a new process, so be sure that your server has sufficient memory to cope.  If you're using [SolidQueue](https://github.com/rails/solid_queue/), the job runner already sets up a separate "supervisor" process and allows you to define [recurring jobs](https://github.com/rails/solid_queue/#recurring-tasks) with a resolution of 1 second.  This may be a suitable solution, but I've not tried it yet.  
+
 ## Testing
 Because operations are intended to model long, complex, flowcharts of decisions and actions, it can be a pain coming up with the combinations of inputs to test every path through the sequence.  
 

data/app/models/operations/task/background.rb

@@ -1,6 +1,11 @@
 module Operations::Task::Background
   extend ActiveSupport::Concern
 
+  included do
+    scope :zombies, -> { zombies_at(Time.now) }
+    scope :zombies_at, ->(time) { where(becomes_zombie_at: ..time) }
+  end
+
   class_methods do
     def delay(value) = @background_delay = value
 
@@ -15,9 +20,15 @@ module Operations::Task::Background
     def timeout_handler = @on_timeout
 
     def with_timeout(data) = data.merge(_execution_timeout: execution_timeout.from_now.utc)
+
+    def restart_zombie_tasks = zombies.find_each { |t| t.restart! }
   end
 
+  def zombie? = Time.now > (updated_at + zombie_delay)
+
   private def background_delay = self.class.background_delay
+  private def zombie_delay = background_delay * 3
+  private def zombie_time = becomes_zombie_at || Time.now
   private def execution_timeout = self.class.execution_timeout
   private def timeout_handler = self.class.timeout_handler
   private def timeout!

data/app/models/operations/task.rb

@@ -35,9 +35,10 @@ module Operations
     end
 
     def perform_later
-      waiting!
+      update! status: "waiting", becomes_zombie_at: Time.now + zombie_delay
       TaskRunnerJob.set(wait_until: background_delay.from_now).perform_later self
     end
+    alias_method :restart!, :perform_later
 
     def self.call(**)
       build(background: false, **).tap do |task|

data/db/migrate/20250403075414_add_becomes_zombie_at_field.rb

@@ -0,0 +1,6 @@
+class AddBecomesZombieAtField < ActiveRecord::Migration[8.0]
+  def change
+    add_column :operations_tasks, :becomes_zombie_at, :datetime, null: true
+    add_index :operations_tasks, :becomes_zombie_at
+  end
+end

data/lib/operations/version.rb

@@ -1,3 +1,3 @@
 module Operations
-  VERSION = "0.5.1"
+  VERSION = "0.5.2"
 end

data/lib/tasks/operations_tasks.rake

@@ -1,4 +1,4 @@
-# desc "Explaining what the task does"
-# task :operations do
-#   # Task goes here
-# end
+desc "Restart any zombie tasks"
+task :restart_zombie_tasks do
+  Operations::Task.restart_zombie_tasks
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: standard_procedure_operations
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Rahoul Baruah
 bindir: bin
 cert_chain: []
-date: 2025-03-11 00:00:00.000000000 Z
+date: 2025-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -65,7 +65,8 @@ files:
 - app/models/operations/task_participant.rb
 - config/routes.rb
 - db/migrate/20250127160616_create_operations_tasks.rb
-- db/migrate/20250309_create_operations_task_participants.rb
+- db/migrate/20250309160616_create_operations_task_participants.rb
+- db/migrate/20250403075414_add_becomes_zombie_at_field.rb
 - lib/operations.rb
 - lib/operations/cannot_wait_in_foreground.rb
 - lib/operations/engine.rb