shikibu 0.1.0

data/lib/shikibu/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Shikibu
+  VERSION = '0.1.0'
+end

data/lib/shikibu/worker.rb

@@ -0,0 +1,389 @@
+# frozen_string_literal: true
+
+module Shikibu
+  # Background worker for workflow processing
+  class Worker
+    WORKFLOW_POLL_INTERVAL = 1
+    TIMER_CHECK_INTERVAL = 10
+    STALE_LOCK_INTERVAL = 60
+    MESSAGE_CHECK_INTERVAL = 5
+    MESSAGE_CLEANUP_INTERVAL = 3600 # 1 hour
+    DEFAULT_MESSAGE_RETENTION_DAYS = 7
+
+    attr_reader :app, :storage, :worker_id, :resume_wake_event, :message_wake_event, :outbox_wake_event
+
+    def initialize(app)
+      @app = app
+      @storage = app.storage
+      @worker_id = app.worker_id
+      @running = false
+      @threads = []
+      @is_leader = false
+      @leader_tasks = []
+
+      # Wake events for NOTIFY integration
+      @resume_wake_event = Notify::WakeEvent.new
+      @message_wake_event = Notify::WakeEvent.new
+      @outbox_wake_event = Notify::WakeEvent.new
+    end
+
+    def start
+      return if @running
+
+      @running = true
+
+      # All workers run these
+      @threads << Thread.new { run_workflow_resumption }
+      @threads << Thread.new { run_message_delivery }
+
+      # Leader election and leader-only tasks
+      @threads << Thread.new { run_leader_election }
+    end
+
+    def stop
+      @running = false
+      stop_leader_tasks
+      @threads.each { |t| t.join(5) }
+      @threads.clear
+    end
+
+    def running?
+      @running
+    end
+
+    def leader?
+      @is_leader
+    end
+
+    private
+
+    # Resume workflows that are in 'running' status but not locked
+    def run_workflow_resumption
+      consecutive_empty = 0
+
+      while @running
+        begin
+          workflows = storage.find_resumable_workflows(limit: 10)
+
+          if workflows.empty?
+            consecutive_empty += 1
+            backoff = calculate_backoff(consecutive_empty)
+
+            # Wait with wake event support (reset backoff on NOTIFY wake)
+            consecutive_empty = 0 if @resume_wake_event.wait(backoff)
+          else
+            consecutive_empty = 0
+            workflows.each do |wf|
+              break unless @running
+
+              resume_workflow_safe(wf[:instance_id])
+            end
+          end
+        rescue StandardError => e
+          log_error('workflow_resumption', e)
+          interruptible_sleep(5)
+        end
+      end
+    end
+
+    # Deliver messages to waiting workflows
+    def run_message_delivery
+      while @running
+        begin
+          # Find subscriptions with pending messages
+          messages_delivered = process_pending_messages
+
+          # Wait with wake event support
+          @message_wake_event.wait(MESSAGE_CHECK_INTERVAL) if messages_delivered.zero?
+        rescue StandardError => e
+          log_error('message_delivery', e)
+          interruptible_sleep(5)
+        end
+      end
+    end
+
+    # Leader election using system locks
+    def run_leader_election
+      while @running
+        begin
+          acquired = storage.try_acquire_system_lock('shikibu_leader', worker_id, timeout: 30)
+
+          if acquired && !@is_leader
+            @is_leader = true
+            start_leader_tasks
+          elsif !acquired && @is_leader
+            @is_leader = false
+            stop_leader_tasks
+          elsif acquired
+            # Refresh lock
+            storage.refresh_system_lock('shikibu_leader', worker_id, timeout: 30)
+          end
+
+          interruptible_sleep(10 + (rand * 3))
+        rescue StandardError => e
+          log_error('leader_election', e)
+          interruptible_sleep(5)
+        end
+      end
+
+      # Release lock on shutdown
+      storage.release_system_lock('shikibu_leader', worker_id) if @is_leader
+    end
+
+    def start_leader_tasks
+      @leader_tasks << Thread.new { run_timer_check }
+      @leader_tasks << Thread.new { run_stale_lock_cleanup }
+      @leader_tasks << Thread.new { run_timeout_check }
+      @leader_tasks << Thread.new { run_message_cleanup }
+      @leader_tasks << Thread.new { run_outbox_relay } if app.outbox_enabled?
+    end
+
+    def stop_leader_tasks
+      @is_leader = false # Signal threads to stop (they check @running && @is_leader)
+      @leader_tasks.each { |t| t.join(2) } # Wait up to 2 seconds for graceful shutdown
+      @leader_tasks.clear
+    end
+
+    # Check for expired timers (leader only)
+    def run_timer_check
+      while @running && @is_leader
+        begin
+          expired = storage.find_expired_timers(limit: 100)
+
+          expired.each do |timer|
+            break unless @running
+
+            handle_timer_expired(timer)
+          end
+
+          interruptible_sleep(TIMER_CHECK_INTERVAL)
+        rescue StandardError => e
+          log_error('timer_check', e)
+          interruptible_sleep(5)
+        end
+      end
+    end
+
+    # Cleanup stale locks and resume workflows (leader only)
+    def run_stale_lock_cleanup
+      while @running && @is_leader
+        begin
+          stale = storage.find_stale_locked_workflows(stale_threshold_seconds: 300)
+
+          stale.each do |wf|
+            break unless @running
+
+            instance_id = wf[:instance_id]
+
+            # Force release lock
+            storage.db[:workflow_instances]
+                   .where(instance_id: instance_id)
+                   .update(locked_by: nil, locked_at: nil, lock_expires_at: nil)
+
+            # Resume the workflow (like Edda's auto_resume_stale_workflows_periodically)
+            resume_workflow_safe(instance_id)
+          end
+
+          interruptible_sleep(STALE_LOCK_INTERVAL)
+        rescue StandardError => e
+          log_error('stale_lock_cleanup', e)
+          interruptible_sleep(10)
+        end
+      end
+    end
+
+    # Check for timed out subscriptions (leader only)
+    def run_timeout_check
+      while @running && @is_leader
+        begin
+          timed_out = storage.find_timed_out_subscriptions(limit: 100)
+
+          timed_out.each do |sub|
+            break unless @running
+
+            handle_subscription_timeout(sub)
+          end
+
+          interruptible_sleep(TIMER_CHECK_INTERVAL)
+        rescue StandardError => e
+          log_error('timeout_check', e)
+          interruptible_sleep(5)
+        end
+      end
+    end
+
+    # Cleanup old channel messages (leader only)
+    # Messages older than retention_days are deleted to prevent database growth
+    def run_message_cleanup
+      retention_days = app.message_retention_days || DEFAULT_MESSAGE_RETENTION_DAYS
+
+      while @running && @is_leader
+        begin
+          deleted_count = storage.cleanup_old_channel_messages(retention_days: retention_days)
+
+          warn "[Shikibu::Worker] Cleaned up #{deleted_count} old channel messages" if deleted_count.positive?
+
+          interruptible_sleep(MESSAGE_CLEANUP_INTERVAL)
+        rescue StandardError => e
+          log_error('message_cleanup', e)
+          interruptible_sleep(60)
+        end
+      end
+    end
+
+    # Relay outbox events to external message broker (leader only)
+    def run_outbox_relay
+      relayer = Outbox::Relayer.new(
+        storage: storage,
+        broker_url: app.broker_url,
+        wake_event: @outbox_wake_event,
+        poll_interval: app.outbox_poll_interval,
+        max_retries: app.outbox_max_retries,
+        max_age_hours: app.outbox_max_age_hours
+      )
+
+      relayer.start
+
+      # Wait while still leader
+      interruptible_sleep(1) while @running && @is_leader
+
+      relayer.stop
+    rescue StandardError => e
+      log_error('outbox_relay', e)
+    end
+
+    def resume_workflow_safe(instance_id)
+      app.resume_workflow(instance_id)
+    rescue LockNotAcquiredError
+      # Another worker got it, that's fine
+    rescue StandardError => e
+      log_error("resume_workflow(#{instance_id})", e)
+    end
+
+    def handle_timer_expired(timer)
+      instance_id = timer[:instance_id]
+      timer_id = timer[:timer_id]
+      activity_id = timer[:activity_id]
+
+      # Record timer expiration in history
+      storage.append_history(
+        instance_id: instance_id,
+        activity_id: activity_id || timer_id,
+        event_type: EventType::TIMER_EXPIRED,
+        event_data: { timer_id: timer_id, expired_at: Time.now.iso8601 }
+      )
+
+      # Remove timer subscription
+      storage.remove_timer(instance_id: instance_id, timer_id: timer_id)
+
+      # Update status to running for resumption
+      storage.update_instance_status(instance_id, Status::RUNNING)
+    rescue StandardError => e
+      log_error("handle_timer_expired(#{instance_id})", e)
+    end
+
+    def handle_subscription_timeout(sub)
+      instance_id = sub[:instance_id]
+      channel = sub[:channel]
+      activity_id = sub[:activity_id]
+
+      # Record timeout in history
+      storage.append_history(
+        instance_id: instance_id,
+        activity_id: activity_id,
+        event_type: EventType::MESSAGE_TIMEOUT,
+        event_data: { channel: channel, timed_out_at: Time.now.iso8601 }
+      )
+
+      # Remove subscription
+      storage.unsubscribe_from_channel(instance_id: instance_id, channel: channel)
+
+      # Update status to running for resumption
+      storage.update_instance_status(instance_id, Status::RUNNING)
+    rescue StandardError => e
+      log_error("handle_subscription_timeout(#{instance_id})", e)
+    end
+
+    def process_pending_messages
+      # Find channels with waiting subscriptions
+      waiting = storage.db[:channel_subscriptions]
+                       .where { Sequel.~(activity_id: nil) }
+                       .select(:channel, :instance_id, :mode, :activity_id, :cursor_message_id)
+                       .limit(100)
+                       .all
+
+      delivered_count = 0
+
+      waiting.each do |sub|
+        break unless @running
+
+        message = storage.get_next_message(
+          channel: sub[:channel],
+          mode: sub[:mode],
+          instance_id: sub[:instance_id],
+          cursor_id: sub[:cursor_message_id]
+        )
+
+        next unless message
+
+        deliver_message(sub, message)
+        delivered_count += 1
+      end
+
+      delivered_count
+    end
+
+    def deliver_message(sub, message)
+      instance_id = sub[:instance_id]
+      activity_id = sub[:activity_id]
+      mode = sub[:mode]
+
+      # For competing mode, try to claim
+      if (mode == ChannelMode::COMPETING) && !storage.claim_message(message_id: message[:message_id],
+                                                                    instance_id: instance_id)
+        return
+      end
+
+      # Record message received in history
+      storage.append_history(
+        instance_id: instance_id,
+        activity_id: activity_id,
+        event_type: EventType::CHANNEL_MESSAGE_RECEIVED,
+        event_data: {
+          channel: sub[:channel],
+          message_id: message[:message_id],
+          data: message[:data],
+          metadata: message[:metadata]
+        }
+      )
+
+      # Clear activity_id from subscription (no longer waiting)
+      storage.db[:channel_subscriptions]
+             .where(instance_id: instance_id, channel: sub[:channel])
+             .update(activity_id: nil, timeout_at: nil)
+
+      # Update status to running for resumption
+      storage.update_instance_status(instance_id, Status::RUNNING)
+    rescue StandardError => e
+      log_error("deliver_message(#{instance_id})", e)
+    end
+
+    def calculate_backoff(consecutive_empty)
+      # Exponential backoff: 2, 4, 8, 16, 32, max 60 seconds + jitter
+      base = WORKFLOW_POLL_INTERVAL
+      exp = [consecutive_empty, 5].min
+      backoff = base * (2**exp)
+      [backoff, 60].min + (rand * 3)
+    end
+
+    def interruptible_sleep(seconds)
+      deadline = Time.now + seconds
+      sleep([0.5, deadline - Time.now].min) while @running && Time.now < deadline
+    end
+
+    def log_error(context, error)
+      warn "[Shikibu::Worker] Error in #{context}: #{error.class}: #{error.message}"
+      warn error.backtrace.first(5).join("\n") if error.backtrace
+    end
+  end
+end