hatchet-sdk 0.1.1 → 0.3.0

data/lib/hatchet/worker/runner.rb

@@ -2,6 +2,7 @@
 
 require "concurrent"
 require "json"
+require "monitor"
 
 module Hatchet
   module WorkerRuntime
@@ -10,7 +11,11 @@ module Hatchet
     #
     # The runner receives actions from the action listener, looks up the
     # corresponding task block, sets up context variables, executes the task,
-    # and sends the result back to the dispatcher.
+    # and sends the result back to the dispatcher. For durable tasks, it wires
+    # the :class:`DurableContext` up to the shared
+    # :class:`DurableEventListener` and per-run
+    # :class:`DurableEviction::DurableEvictionManager` when the engine supports
+    # eviction.
     #
     # @example
     #   runner = Runner.new(
@@ -19,36 +24,76 @@ module Hatchet
     #     dispatcher_client: dispatcher_grpc,
     #     event_client: event_grpc,
     #     logger: logger,
-    #     client: hatchet_client
+    #     client: hatchet_client,
+    #     engine_version: "v0.80.0",
+    #     durable_slots: 10,
     #   )
     #   runner.execute(action)
     class Runner
+      STARTED_EVENT_RETRY_COUNT = 5
+      STARTED_EVENT_STOP = Object.new
+
       # @param workflows [Array<Workflow, Task>] Registered workflows
       # @param slots [Integer] Maximum concurrent task slots
       # @param dispatcher_client [Hatchet::Clients::Grpc::Dispatcher] gRPC dispatcher client
       # @param event_client [Hatchet::Clients::Grpc::EventClient] gRPC event client
       # @param logger [Logger] Logger instance
       # @param client [Hatchet::Client] The Hatchet client
-      def initialize(workflows:, slots:, dispatcher_client:, event_client:, logger:, client:)
+      # @param engine_version [String, nil] Engine semantic version (from GetVersion)
+      # @param durable_slots [Integer, nil] Separate slot count for durable tasks; defaults to ``slots``.
+      # @param worker_id [String, nil] Worker ID from registration; stamped onto durable calls that need it.
+      def initialize(
+        workflows:,
+        slots:,
+        dispatcher_client:,
+        event_client:,
+        logger:,
+        client:,
+        engine_version: nil,
+        durable_slots: nil,
+        worker_id: nil
+      )
         @workflows = workflows
         @slots = slots
+        @durable_slots = durable_slots || slots
         @dispatcher_client = dispatcher_client
         @event_client = event_client
         @logger = logger
         @client = client
+        @engine_version = engine_version
+        @worker_id = worker_id
 
-        # Thread pool with semaphore for slot management
         @pool = Concurrent::FixedThreadPool.new(slots)
         @semaphore = Concurrent::Semaphore.new(slots)
 
-        # Build task lookup table
         @task_map = build_task_map
+
+        @contexts_mu = Monitor.new
+        @contexts = {}
+        @task_threads = {}
+        @step_action_event_queue = Queue.new
+        @step_action_event_thread = Thread.new { process_step_action_events }
+
+        @has_durable_tasks = @task_map.values.any?(&:durable)
+        @supports_durable_eviction = supports_durable_eviction?
+
+        @durable_event_listener = build_durable_event_listener
+        @eviction_manager = nil
+        @eviction_manager_mu = Mutex.new
       end
 
+      # @return [WorkerRuntime::DurableEviction::DurableEvictionManager, nil]
+      attr_reader :eviction_manager
+
+      # @return [WorkerRuntime::DurableEventListener, nil]
+      attr_reader :durable_event_listener
+
       # Execute an action (task assignment) in the thread pool.
       #
       # @param action [AssignedAction] The action from the dispatcher
       def execute(action)
+        ensure_eviction_manager_started(action)
+
         @semaphore.acquire
 
         @pool.post do
@@ -62,12 +107,97 @@ module Hatchet
       #
       # @param timeout [Integer] Seconds to wait for in-progress tasks
       def shutdown(timeout: 30)
+        if @eviction_manager
+          begin
+            @eviction_manager.evict_all_waiting
+          rescue StandardError => e
+            @logger.warn("Runner: failed to evict waiting durable runs during shutdown: #{e.class}: #{e.message}")
+          end
+        end
+
         @pool.shutdown
         @pool.wait_for_termination(timeout)
+        stop_step_action_event_thread
+
+        @durable_event_listener&.stop
       end
 
       private
 
+      def supports_durable_eviction?
+        return false unless @engine_version
+
+        !Hatchet::EngineVersion.semver_less_than?(
+          @engine_version,
+          Hatchet::MinEngineVersion::DURABLE_EVICTION,
+        )
+      end
+
+      def build_durable_event_listener
+        return nil unless @has_durable_tasks && @supports_durable_eviction
+
+        DurableEventListener.new(
+          config: @client.config,
+          channel: @client.channel,
+          logger: @logger,
+          on_server_evict: method(:handle_server_evict),
+        )
+      end
+
+      def ensure_eviction_manager_started(_action)
+        return unless @has_durable_tasks
+        return unless @supports_durable_eviction
+
+        @durable_event_listener&.ensure_started(@worker_id) if @worker_id
+        return if @eviction_manager
+
+        @eviction_manager_mu.synchronize do
+          return if @eviction_manager
+
+          @eviction_manager = DurableEviction::DurableEvictionManager.new(
+            durable_slots: @durable_slots,
+            cancel_local: method(:eviction_cancel_local),
+            request_eviction_with_ack: method(:eviction_request_with_ack),
+            logger: @logger,
+          )
+          @eviction_manager.start
+        end
+      end
+
+      def handle_server_evict(durable_task_external_id, invocation_count)
+        return unless @eviction_manager
+
+        @eviction_manager.handle_server_eviction(durable_task_external_id, invocation_count)
+      end
+
+      def eviction_cancel_local(action_key)
+        thread, ctx = @contexts_mu.synchronize do
+          [@task_threads[action_key], @contexts[action_key]]
+        end
+
+        if @durable_event_listener && ctx.is_a?(DurableContext)
+          @durable_event_listener.cleanup_task_state(ctx.step_run_id, ctx.invocation_count || 1)
+        end
+
+        thread&.raise(Hatchet::DurableTaskEvictedError.new)
+      end
+
+      def eviction_request_with_ack(action_key, rec)
+        return unless @durable_event_listener
+
+        invocation_count = 1
+        @contexts_mu.synchronize do
+          ctx = @contexts[action_key]
+          invocation_count = ctx.invocation_count if ctx.is_a?(DurableContext) && ctx.invocation_count
+        end
+
+        @durable_event_listener.send_evict_invocation(
+          rec.step_run_id,
+          invocation_count,
+          reason: rec.eviction_reason,
+        )
+      end
+
       def build_task_map
         map = {}
 
@@ -76,7 +206,6 @@ module Hatchet
             service_name = @client.config.apply_namespace(wf.name.downcase)
 
             wf.tasks.each do |name, task|
-              # TODO: is this what we do across sdks...
               key = "#{service_name}:#{name}".downcase
               map[key] = task
             end
@@ -85,7 +214,6 @@ module Hatchet
 
             map["#{service_name}:on_success"] = wf.on_success if wf.on_success
           elsif wf.is_a?(Task)
-            # Standalone task -- the workflow wrapper has the same name
             workflow = wf.workflow
             if workflow
               service_name = @client.config.apply_namespace(workflow.name.downcase)
@@ -98,35 +226,60 @@ module Hatchet
       end
 
       def execute_task(action)
-        # Set context vars BEFORE executing the task
+        action_key = nil
+        prepare_action_execution(action)
+
+        task = find_task(action)
+        return unless task
+
+        ctx = build_context(action, task)
+        action_key = action_key_for(action)
+        configure_durable_context(task, ctx, action, action_key)
+        track_action_context(action_key, ctx)
+        run_task(action, task, ctx)
+      rescue Hatchet::DurableTaskEvictedError => e
+        @logger.info("Durable task evicted: #{action.action_id}: #{e.message}")
+      rescue NonRetryableError => e
+        @logger.error("Non-retryable error in task #{action.action_id}: #{e.message}")
+        send_failure(action, e, retryable: false)
+      rescue StandardError => e
+        @logger.error("Error in task #{action.action_id}: #{e.message}")
+        send_failure(action, e, retryable: true)
+      ensure
+        # CRITICAL: Clean up context vars to prevent leaking to next task
+        cleanup_action(action_key) if action_key
+        ContextVars.clear
+      end
+
+      def prepare_action_execution(action)
+        @logger.debug(
+          "Runner: received action action_id=#{action.action_id} step_run_id=#{action.task_run_external_id} " \
+          "retry_count=#{action.retry_count} durable_invocation_count=#{extract_invocation_count(action)}",
+        )
         ContextVars.set(
           workflow_run_id: action.workflow_run_id,
           step_run_id: action.task_run_external_id,
-          worker_id: @dispatcher_client.respond_to?(:worker_id) ? @dispatcher_client.worker_id : "",
+          worker_id: worker_id,
           action_key: action.action_id,
           additional_metadata: parse_metadata(action),
           retry_count: action.retry_count,
         )
-
-        # Send STARTED event
         send_started(action)
+      end
 
-        # Look up the task by action_id (service_name:task_name), case-insensitive
+      def find_task(action)
         task_key = action.action_id.downcase
         task = @task_map[task_key]
+        return task if task
 
-        unless task
-          @logger.error("No task found for action: #{task_key}")
-          send_failure(action, StandardError.new("No task found for action: #{task_key}"), retryable: false)
-          return
-        end
-
-        # Parse parent outputs from the action payload
-        parent_outputs = parse_parent_outputs(action)
+        @logger.error("No task found for action: #{task_key}")
+        send_failure(action, StandardError.new("No task found for action: #{task_key}"), retryable: false)
+        nil
+      end
 
-        # Build the context
+      def build_context(action, task)
         ctx_class = task.durable ? DurableContext : Context
-        ctx = ctx_class.new(
+        ctx_class.new(
           workflow_run_id: action.workflow_run_id,
           step_run_id: action.task_run_external_id,
           action: action,
@@ -135,43 +288,121 @@ module Hatchet
           event_client: @event_client,
           additional_metadata: ContextVars.additional_metadata,
           retry_count: action.retry_count,
-          parent_outputs: parent_outputs,
+          parent_outputs: parse_parent_outputs(action),
+          worker_id: ContextVars.worker_id,
         )
+      end
 
-        # Parse input from action payload
-        input = parse_input(action)
+      def configure_durable_context(task, ctx, action, action_key)
+        return unless task.durable && ctx.is_a?(DurableContext)
 
-        # Resolve dependencies if the task has any
-        ctx.deps = resolve_dependencies(task.deps, input, ctx) if task.deps && !task.deps.empty?
+        ctx.eviction_manager = @eviction_manager
+        ctx.action_key = action_key
+        ctx.durable_event_listener = @durable_event_listener
+        ctx.invocation_count = extract_invocation_count(action)
+        ctx.engine_version = @engine_version
+        register_durable_run(task, action, action_key, ctx)
+      end
 
-        # Execute the task
-        result = task.call(input, ctx)
+      def register_durable_run(task, action, action_key, ctx)
+        return unless @eviction_manager && task.eviction_policy
+
+        @eviction_manager.register_run(
+          action_key,
+          step_run_id: action.task_run_external_id,
+          invocation_count: ctx.invocation_count,
+          eviction_policy: task.eviction_policy,
+        )
+      end
+
+      def track_action_context(action_key, ctx)
+        @contexts_mu.synchronize do
+          @contexts[action_key] = ctx
+          @task_threads[action_key] = Thread.current
+        end
+      end
 
-        # Send result back to dispatcher
+      def run_task(action, task, ctx)
+        input = parse_input(action)
+        ctx.deps = resolve_dependencies(task.deps, input, ctx) if task.deps && !task.deps.empty?
+        result = task.call(input, ctx)
         send_result(action, result)
-      rescue NonRetryableError => e
-        @logger.error("Non-retryable error in task #{action.action_id}: #{e.message}")
-        send_failure(action, e, retryable: false)
-      rescue StandardError => e
-        @logger.error("Error in task #{action.action_id}: #{e.message}")
-        send_failure(action, e, retryable: true)
-      ensure
-        # CRITICAL: Clean up context vars to prevent leaking to next task
-        ContextVars.clear
       end
 
-      # Send a STARTED event to the dispatcher.
+      def action_key_for(action)
+        "#{action.task_run_external_id}/#{action.retry_count}"
+      end
+
+      def extract_invocation_count(action)
+        value = action.respond_to?(:durable_task_invocation_count) ? action.durable_task_invocation_count : nil
+        value.nil? || value.zero? ? 1 : value
+      end
+
+      def worker_id
+        @dispatcher_client.respond_to?(:worker_id) ? @dispatcher_client.worker_id : ""
+      end
+
+      def cleanup_action(action_key)
+        @contexts_mu.synchronize do
+          @contexts.delete(action_key)
+          @task_threads.delete(action_key)
+        end
+
+        @eviction_manager&.unregister_run(action_key)
+      end
+
       def send_started(action)
-        @dispatcher_client.send_step_action_event(
-          action: action,
-          event_type: :STEP_EVENT_TYPE_STARTED,
-          payload: "{}",
-        )
-      rescue StandardError => e
-        @logger.warn("Failed to send STARTED event: #{e.message}")
+        @step_action_event_queue << action
+      end
+
+      def process_step_action_events
+        loop do
+          action = @step_action_event_queue.pop
+          break if action.equal?(STARTED_EVENT_STOP)
+
+          send_started_with_retry(action)
+        end
+      rescue ClosedQueueError
+        nil
+      end
+
+      def send_started_with_retry(action)
+        attempt = 1
+
+        loop do
+          @dispatcher_client.send_step_action_event(
+            action: action,
+            event_type: :STEP_EVENT_TYPE_STARTED,
+            payload: "{}",
+            retry_count: action.retry_count,
+          )
+          return
+        rescue StandardError => e
+          @logger.warn(
+            "Failed to send STARTED event (#{attempt}/#{STARTED_EVENT_RETRY_COUNT}): #{e.message}",
+          )
+          raise e if attempt >= STARTED_EVENT_RETRY_COUNT
+
+          sleep started_event_backoff_seconds(attempt)
+          attempt += 1
+        end
+      end
+
+      def started_event_backoff_seconds(attempt)
+        base = 0.1
+        jitter = rand * base
+        [((base * (2**attempt)) + jitter), 1.0].min
+      end
+
+      def stop_step_action_event_thread
+        return unless @step_action_event_thread
+
+        @step_action_event_queue << STARTED_EVENT_STOP
+        @step_action_event_thread.join(5)
+      rescue StandardError
+        nil
       end
 
-      # Send a COMPLETED event with the task result.
       def send_result(action, result)
         payload = result.nil? ? "{}" : JSON.generate(result)
 
@@ -183,7 +414,6 @@ module Hatchet
         )
       end
 
-      # Send a FAILED event with error details.
       def send_failure(action, error, retryable:)
         payload = JSON.generate({ "error" => error.message })
 
@@ -218,7 +448,6 @@ module Hatchet
         resolved
       end
 
-      # Parse additional metadata from the action.
       def parse_metadata(action)
         raw = action.respond_to?(:additional_metadata) ? action.additional_metadata : nil
         return {} if raw.nil? || raw.to_s.empty?
@@ -228,7 +457,6 @@ module Hatchet
         {}
       end
 
-      # Parse parent task outputs from the action payload.
       def parse_parent_outputs(action)
         raw = action.respond_to?(:action_payload) ? action.action_payload : nil
         return {} if raw.nil? || raw.to_s.empty?
@@ -239,13 +467,11 @@ module Hatchet
         {}
       end
 
-      # Parse task input from the action payload.
       def parse_input(action)
         raw = action.respond_to?(:action_payload) ? action.action_payload : nil
         return {} if raw.nil? || raw.to_s.empty?
 
         parsed = JSON.parse(raw)
-        # The input is typically stored under the "input" key in the payload
         parsed.is_a?(Hash) && parsed.key?("input") ? parsed["input"] : parsed
       rescue JSON::ParserError
         {}

data/lib/hatchet/worker_obj.rb

@@ -30,19 +30,28 @@ module Hatchet
     # @return [String, nil] Worker ID assigned by the server
     attr_accessor :worker_id
 
+    # @return [Integer] Number of durable-task slots (defaults to ``slots``)
+    attr_reader :durable_slots
+
+    # @return [String, nil] Engine semantic version detected on ``start``
+    attr_reader :engine_version
+
     # @param name [String] Worker name
     # @param client [Hatchet::Client] The Hatchet client
     # @param workflows [Array<Workflow, Task>] Workflows to register
     # @param slots [Integer] Number of concurrent task slots (default: 10)
+    # @param durable_slots [Integer, nil] Number of durable-task slots; defaults to ``slots``
     # @param labels [Hash] Worker labels (default: {})
-    def initialize(name:, client:, workflows: [], slots: 10, labels: {})
+    def initialize(name:, client:, workflows: [], slots: 10, durable_slots: nil, labels: {})
       @name = name
       @client = client
       @workflows = workflows
       @slots = slots
+      @durable_slots = durable_slots || slots
       @labels = client.config.worker_preset_labels.merge(labels)
       @worker_id = nil
       @shutdown = false
+      @engine_version = nil
     end
 
     # Start the worker. This blocks until shutdown is requested.
@@ -58,7 +67,8 @@ module Hatchet
       @client.config.logger.info("Starting worker '#{@name}' with #{@slots} slots")
       @client.config.logger.info("Registering #{@workflows.length} workflow(s)")
 
-      # Register workflows with the server
+      check_engine_version
+
       register_workflows
 
       # Start the health check server if enabled
@@ -88,6 +98,46 @@ module Hatchet
 
     private
 
+    def check_engine_version
+      @engine_version = @client.dispatcher_grpc.get_version
+      if @engine_version
+        @client.config.logger.info("Connected to Hatchet engine #{@engine_version}")
+        check_eviction_support
+      else
+        @client.config.logger.debug(
+          "Engine did not report a version (GetVersion unimplemented); assuming pre-eviction compatibility mode.",
+        )
+      end
+    rescue StandardError => e
+      @client.config.logger.debug("Failed to fetch engine version: #{e.class}: #{e.message}")
+      @engine_version = nil
+    end
+
+    def check_eviction_support
+      return unless @workflows.any? { |wf| workflow_has_durable_task?(wf) }
+      return unless @engine_version
+
+      return unless Hatchet::EngineVersion.semver_less_than?(
+        @engine_version,
+        Hatchet::MinEngineVersion::DURABLE_EVICTION,
+      )
+
+      @client.config.logger.warn(
+        "Durable task eviction requires engine #{Hatchet::MinEngineVersion::DURABLE_EVICTION} or newer " \
+        "(engine reports #{@engine_version}). Falling back to legacy durable-event protocol; eviction policies will be ignored.",
+      )
+    end
+
+    def workflow_has_durable_task?(workflow)
+      if workflow.is_a?(Workflow)
+        workflow.tasks.values.any?(&:durable)
+      elsif workflow.is_a?(Task)
+        workflow.durable
+      else
+        false
+      end
+    end
+
     def setup_signal_handlers
       @main_thread = Thread.current
 
@@ -126,9 +176,13 @@ module Hatchet
 
       # Register the worker with the dispatcher
       response = @client.dispatcher_grpc.register(
-        name: @name,
+        name: @client.config.apply_namespace(@name),
         actions: action_ids,
         slots: @slots,
+        slot_config: {
+          "default" => @slots,
+          "durable" => @durable_slots,
+        },
         labels: @labels,
       )
 
@@ -188,7 +242,6 @@ module Hatchet
     def run_action_listener
       @client.config.logger.info("Worker '#{@name}' is running. Press Ctrl+C to stop.")
 
-      # Create the runner for executing tasks
       runner = WorkerRuntime::Runner.new(
         workflows: @workflows,
         slots: @slots,
@@ -196,6 +249,9 @@ module Hatchet
         event_client: @client.event_grpc,
         logger: @client.config.logger,
         client: @client,
+        engine_version: @engine_version,
+        durable_slots: @durable_slots,
+        worker_id: @worker_id,
       )
 
       # Create the action listener with retry/reconnect logic

data/lib/hatchet/workflow.rb

@@ -110,14 +110,18 @@ module Hatchet
       t
     end
 
-    # Define a durable task within this workflow
+    # Define a durable task within this workflow.
     #
     # @param name [Symbol, String] Task name
-    # @param opts [Hash] Task options
+    # @param eviction_policy [Hatchet::EvictionPolicy, nil] Eviction policy for this
+    #   durable task. Defaults to {Hatchet::DEFAULT_DURABLE_TASK_EVICTION_POLICY}
+    #   (15-minute TTL, capacity-eviction enabled). Pass ``nil`` to disable
+    #   eviction entirely for this task.
+    # @param opts [Hash] Other Task options forwarded to {#task}.
     # @yield [input, ctx] The task execution block
     # @return [Task] The created durable task
-    def durable_task(name, **opts, &)
-      task(name, durable: true, **opts, &)
+    def durable_task(name, eviction_policy: Hatchet::DEFAULT_DURABLE_TASK_EVICTION_POLICY, **opts, &)
+      task(name, durable: true, eviction_policy: eviction_policy, **opts, &)
     end
 
     # Define an on_failure task for this workflow

data/lib/hatchet-sdk.rb

@@ -25,6 +25,8 @@ require_relative "hatchet/features/scheduled"
 
 # Core classes
 require_relative "hatchet/exceptions"
+require_relative "hatchet/engine_version"
+require_relative "hatchet/eviction_policy"
 require_relative "hatchet/concurrency"
 require_relative "hatchet/conditions"
 require_relative "hatchet/condition_converter"
@@ -52,6 +54,7 @@ require_relative "hatchet/contracts/events/events_services_pb"
 require_relative "hatchet/contracts/workflows/workflows_pb"
 require_relative "hatchet/contracts/workflows/workflows_services_pb"
 require_relative "hatchet/contracts/v1/shared/condition_pb"
+require_relative "hatchet/contracts/v1/shared/trigger_pb"
 require_relative "hatchet/contracts/v1/dispatcher_pb"
 require_relative "hatchet/contracts/v1/dispatcher_services_pb"
 require_relative "hatchet/contracts/v1/workflows_pb"
@@ -65,6 +68,9 @@ require_relative "hatchet/clients/grpc/event_client"
 # Worker runtime
 require_relative "hatchet/worker/action_listener"
 require_relative "hatchet/worker/workflow_run_listener"
+require_relative "hatchet/worker/durable_eviction/cache"
+require_relative "hatchet/worker/durable_eviction/manager"
+require_relative "hatchet/worker/durable_event_listener"
 require_relative "hatchet/worker/runner"
 
 # Ruby SDK for Hatchet workflow engine
@@ -216,17 +222,21 @@ module Hatchet
       wf.task(name, **opts, &block)
     end
 
-    # Create a standalone durable task
+    # Create a standalone durable task.
     #
     # @param name [String] Task name
+    # @param eviction_policy [Hatchet::EvictionPolicy, nil] Eviction policy for this
+    #   durable task. Defaults to {Hatchet::DEFAULT_DURABLE_TASK_EVICTION_POLICY}
+    #   (15-minute TTL, capacity-eviction enabled). Pass ``nil`` to disable
+    #   eviction entirely for this task.
     # @param opts [Hash] Task options
     # @yield [input, ctx] The task execution block
     # @return [Hatchet::Task]
-    def durable_task(name:, **opts, &block)
+    def durable_task(name:, eviction_policy: Hatchet::DEFAULT_DURABLE_TASK_EVICTION_POLICY, **opts, &block)
       wf = Workflow.new(name: name, client: self,
                         on_events: opts.delete(:on_events) || [],
                         default_filters: opts.delete(:default_filters) || [],)
-      wf.durable_task(name, **opts, &block)
+      wf.durable_task(name, eviction_policy: eviction_policy, **opts, &block)
     end
 
     # Create a new worker

data/sig/hatchet/clients/grpc/dispatcher.rbs

@@ -10,6 +10,7 @@ module Hatchet
           name: String,
           actions: Array[String],
           slots: Integer,
+          ?slot_config: Hash[String, Integer]?,
           ?labels: Hash[String, String | Integer]
         ) -> untyped
 
@@ -28,6 +29,7 @@ module Hatchet
         def release_slot: (step_run_id: String) -> untyped
         def upsert_worker_labels: (worker_id: String, labels: Hash[String, String | Integer]) -> untyped
         def subscribe_to_workflow_runs: (untyped request_enum) -> untyped
+        def get_version: () -> String?
         def close: () -> void
       end
     end