turnkit 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78b1d7eae5414daf8116c999392a6d726e50360c643578d35ff7154559c525f3
-  data.tar.gz: f6e0f434323bf93ac97af5839cf08ede8d89b9aa50fe96eb0c0ed93992fd4d42
+  metadata.gz: 3db08f0653a468f22daa74070c7b1a290f02b44beaf6d35c950f3d36d86dc224
+  data.tar.gz: 18d3b2df593865ba53e0863d0e2b061f8744bdbbf78409b40d525c3ba47b5070
 SHA512:
-  metadata.gz: 8fdf2b4569248ebc409da2915663864e5891d54049c3696d827e0c2c2242b283c52e2f0cf2bbaee9adc08094d74c839b28dab2f5efaf898a03812393de28cf18
-  data.tar.gz: 3283229cf6c2e94d109437b715ee21944fbebaefe1a430670c8dc4410cbb0fbc6e670039a3d60730f4a5fd27ee2669b65eab4431cb362e342bed32dac89d4fec
+  metadata.gz: dd2216c9e2275dd737d6e1cf7dadd97053ca5afb6992939791669f29e047f3e44eb45f492a09b4494222117428baf07ffd9aa60352f9363b0a8b3eedb9c6ab54
+  data.tar.gz: 3b8f1a45bc98506dc09a9a6703aacf1f614fb7649c3c64942c98e902aff1db9232f986aad33c077be51fec1623d82abf4506d32698b34e3079ff775b9a1eb38b

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 0.5.0 - 2026-07-22
+
+### Breaking
+
+- Custom stores: `find_stale_turns` is replaced by `reconcile_stale_turns(before:)`, which must atomically transition eligible turns to `stale` and return the reconciled records. `update_tool_execution` is replaced by `claim_tool_execution(id, from:, to:, **attributes)`, an atomic compare-and-set mirroring `claim_turn`.
+
+### Fixed
+
+- Make stale-turn reconciliation atomic, so `TurnKit.reconcile_stale!` can no longer overwrite a turn that was concurrently claimed, heartbeated, or completed (#1).
+- Heartbeat while a tool executes, so tools slower than `TurnKit.timeout` are not falsely reconciled.
+
+### Added
+
+- Add an `interrupted` tool-execution status. Reconciliation marks a stale turn's unfinished tool executions `interrupted`, appends a synthetic error tool result for unresolved tool calls (keeping the transcript continuable), and emits `turn.stale` and `tool_call.interrupted` events. Late tool results arriving after interruption are dropped, never overwriting the reconciled state.
+- `TurnKit.reconcile_stale!` returns the reconciled turn records. `stale` is provisional: a worker whose heartbeats were merely late finishes its turn normally, replacing `stale` with the actual outcome.
+
 ## 0.4.2 - 2026-07-02
 
 ### Breaking

data/README.md

@@ -784,12 +784,26 @@ TurnKit.store = TurnKit::ActiveRecordStore.new(
 )
 ```
 
-Reconcile stale turns:
+Reconcile turns abandoned by a dead worker (for example after a hard-killed
+process). Run this periodically:
 
 ```ruby
 TurnKit.reconcile_stale!
 ```
 
+Reconciliation atomically marks pending and running turns whose last heartbeat
+is older than `TurnKit.timeout` as `stale`, so it never overwrites a turn that
+was concurrently claimed, heartbeated, or completed. Each stale turn's
+unfinished tool executions become `interrupted`, and a synthetic error tool
+result is appended for any unresolved tool call so the conversation can be
+continued. TurnKit never reruns an interrupted tool — whether its side effect
+happened is unknown, so the continued model is told not to assume either way.
+
+Reconciliation does not cancel or reassign the underlying process. If the
+original worker is still alive (its heartbeats were merely late), it continues
+over the synthetic interrupted result and later replaces `stale` with its
+actual `completed` or `failed` outcome — treat `turn.stale` as provisional.
+
 ## Options
 
 | Option | Description |

data/lib/turnkit/active_record_store.rb

@@ -144,18 +144,28 @@ module TurnKit
       tool_execution_hash(tool_execution_class.find_by!(uid: id))
     end
 
-    def update_tool_execution(id, attributes)
-      record = tool_execution_class.find_by!(uid: id)
-      record.update!(Record.tool_execution_update(attributes))
-      tool_execution_hash(record)
+    def claim_tool_execution(id, from: "running", to: "completed", **attributes)
+      attrs = Record.tool_execution_update(attributes.merge(status: to))
+      affected = tool_execution_class.where(uid: id, status: from).update_all(attrs.merge(updated_at: Clock.now))
+      return nil if affected.zero?
+
+      load_tool_execution(id)
     end
 
     def list_tool_executions(turn_id:)
       tool_execution_class.where(turn_uid: turn_id).order(:created_at, :uid).map { |record| tool_execution_hash(record) }
     end
 
-    def find_stale_turns(before:)
-      turn_class.where(status: %w[pending running]).where("COALESCE(heartbeat_at, started_at, created_at) < ?", before).map { |record| turn_hash(record) }
+    def reconcile_stale_turns(before:)
+      scope = turn_class.where(status: %w[pending running]).where("COALESCE(heartbeat_at, started_at, created_at) < ?", before)
+      scope.filter_map do |record|
+        now = Clock.now
+        affected = turn_class
+          .where(id: record.id, status: %w[pending running])
+          .where("COALESCE(heartbeat_at, started_at, created_at) < ?", before)
+          .update_all(status: "stale", completed_at: now, updated_at: now)
+        turn_hash(record.reload) if affected == 1
+      end
     end
 
     private

data/lib/turnkit/memory_store.rb

@@ -100,10 +100,12 @@ module TurnKit
       @mutex.synchronize { duplicate(@tool_executions.fetch(id)) }
     end
 
-    def update_tool_execution(id, attributes)
-      attrs = Record.tool_execution_update(attributes)
+    def claim_tool_execution(id, from: "running", to: "completed", **attributes)
+      attrs = Record.tool_execution_update(attributes.merge(status: to))
       @mutex.synchronize do
         record = @tool_executions.fetch(id)
+        return nil unless record["status"] == from
+
         record.merge!(attrs.merge("updated_at" => Clock.now))
         duplicate(record)
       end
@@ -118,11 +120,14 @@ module TurnKit
       end
     end
 
-    def find_stale_turns(before:)
+    def reconcile_stale_turns(before:)
       @mutex.synchronize do
-        @turns.values.select do |turn|
-          %w[pending running].include?(turn["status"]) && stale_anchor(turn) && stale_anchor(turn) < before
-        end.map { |turn| duplicate(turn) }
+        @turns.values.filter_map do |turn|
+          next unless %w[pending running].include?(turn["status"]) && stale_anchor(turn) && stale_anchor(turn) < before
+
+          turn.merge!("status" => "stale", "completed_at" => Clock.now, "updated_at" => Clock.now)
+          duplicate(turn)
+        end
       end
     end
 

data/lib/turnkit/reconciliation.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module TurnKit
+  # Reconciles turns abandoned by a dead worker: atomically marks them stale,
+  # marks their unfinished tool executions interrupted, and appends synthetic
+  # error tool results so the persisted transcript stays structurally complete
+  # for continuation. TurnKit never reruns an interrupted tool; the synthetic
+  # result tells the continued model the outcome is unknown.
+  module Reconciliation
+    INTERRUPTED_MESSAGE = "Tool execution was interrupted before a result was recorded. " \
+      "It is unknown whether the operation ran; do not assume it did or did not."
+
+    module_function
+
+    def reconcile!(before:)
+      TurnKit.store.reconcile_stale_turns(before: before).each do |turn|
+        emit("turn.stale", turn)
+        executions = interrupt_tool_executions(turn)
+        repair_transcript(turn, executions)
+      end
+    end
+
+    def interrupt_tool_executions(turn)
+      store = TurnKit.store
+      store.list_tool_executions(turn_id: turn.fetch("id")).map do |execution|
+        next execution unless %w[pending running].include?(execution.fetch("status"))
+
+        interrupted = store.claim_tool_execution(
+          execution.fetch("id"),
+          from: execution.fetch("status"),
+          to: "interrupted",
+          error: { "message" => "interrupted: worker terminated while the tool was executing" },
+          completed_at: Clock.now
+        )
+        next execution unless interrupted
+
+        emit("tool_call.interrupted", turn, id: interrupted.fetch("tool_call_id"), name: interrupted.fetch("tool_name"), tool_execution_id: interrupted.fetch("id"))
+        interrupted
+      end
+    end
+
+    def repair_transcript(turn, executions)
+      store = TurnKit.store
+      messages = store.list_messages(turn.fetch("conversation_id"))
+      resolved = messages
+        .select { |message| message["kind"] == "tool_result" }
+        .flat_map { |message| message["content"].map { |part| part["tool_call_id"] } }
+
+      messages
+        .select { |message| message["turn_id"] == turn.fetch("id") && message["kind"] == "tool_call" }
+        .flat_map { |message| message["content"].select { |part| part["type"] == "tool_call" } }
+        .reject { |part| resolved.include?(part["id"]) }
+        .each do |part|
+          execution = executions.find { |candidate| candidate["tool_call_id"] == part["id"] }
+          message = store.append_message(
+            "conversation_id" => turn.fetch("conversation_id"),
+            "turn_id" => turn.fetch("id"),
+            "role" => "tool",
+            "kind" => "tool_result",
+            "content" => [ { "type" => "tool_result", "tool_call_id" => part["id"], "text" => { "error" => true, "message" => INTERRUPTED_MESSAGE }.to_json, "error" => true } ],
+            "tool_execution_id" => execution&.fetch("id"),
+            "metadata" => { "tool_name" => part["name"], "interrupted" => true }
+          )
+          emit("message.created", turn, message_id: message.fetch("id"), role: "tool", kind: "tool_result")
+        end
+    end
+
+    def emit(type, turn, payload = {})
+      TurnKit.on_event&.call(Event.new(type: type, turn_id: turn.fetch("id"), conversation_id: turn.fetch("conversation_id"), payload: payload))
+    end
+  end
+end

data/lib/turnkit/record.rb

@@ -3,7 +3,7 @@
 module TurnKit
   module Record
     TURN_STATUSES = %w[pending running completed failed cancelled stale].freeze
-    TOOL_EXECUTION_STATUSES = %w[pending running completed failed cancelled].freeze
+    TOOL_EXECUTION_STATUSES = %w[pending running completed failed cancelled interrupted].freeze
 
     TURN_UPDATE_KEYS = %w[status options usage cost error output_text output_data started_at heartbeat_at completed_at].freeze
     TOOL_EXECUTION_UPDATE_KEYS = %w[status result error started_at completed_at].freeze

data/lib/turnkit/store.rb

@@ -20,9 +20,20 @@ module TurnKit
 
     def create_tool_execution(_attributes) = raise(NotImplementedError)
     def load_tool_execution(_id) = raise(NotImplementedError)
-    def update_tool_execution(_id, _attributes) = raise(NotImplementedError)
+    # claim_tool_execution mirrors claim_turn: an atomic compare-and-set on
+    # status, so a tool result recorded by a live worker and a reconciler
+    # marking the execution interrupted cannot overwrite each other.
+    def claim_tool_execution(_id, from: "running", to: "completed", **_attributes) = raise(NotImplementedError)
     def list_tool_executions(turn_id:) = raise(NotImplementedError)
 
-    def find_stale_turns(before:) = []
+    # reconcile_stale_turns is the other concurrency-safety point: it must
+    # atomically transition each pending/running turn whose stale anchor
+    # (heartbeat_at, else started_at, else created_at) is older than `before`
+    # to `stale`, rechecking both predicates at write time so a concurrently
+    # claimed, heartbeated, or completed turn is never overwritten. Returns
+    # the reconciled turn records. Descendant turns of a dead process stop
+    # heartbeating too and are reconciled by the same predicate, so no
+    # explicit subtree cascade is needed.
+    def reconcile_stale_turns(before:) = []
   end
 end

data/lib/turnkit/tool_runner.rb

@@ -71,7 +71,9 @@ module TurnKit
 
       def finish_success(execution, tool_call, payload)
         json = payload.to_json
-        attrs = turn.store.update_tool_execution(execution.id, "status" => "completed", "result" => payload, "completed_at" => Clock.now)
+        attrs = turn.store.claim_tool_execution(execution.id, from: "running", to: "completed", result: payload, completed_at: Clock.now)
+        return superseded_execution(execution) unless attrs
+
         append_result(execution, tool_call, payload, json: json, error: false)
         heartbeat!
         turn.emit("tool_call.completed", id: tool_call.id, name: tool_call.name, result_chars: json.length)
@@ -81,13 +83,21 @@ module TurnKit
       def finish_error(execution, tool_call, message, details: nil)
         error = { "message" => message.to_s, "details" => details }.compact
         json = error.to_json
-        attrs = turn.store.update_tool_execution(execution.id, "status" => "failed", "error" => error, "completed_at" => Clock.now)
+        attrs = turn.store.claim_tool_execution(execution.id, from: "running", to: "failed", error: error, completed_at: Clock.now)
+        return superseded_execution(execution) unless attrs
+
         append_result(execution, tool_call, error, json: json, error: true)
         heartbeat!
         turn.emit("tool_call.failed", id: tool_call.id, name: tool_call.name, error: error, result_chars: json.length)
         ToolExecution.new(attrs)
       end
 
+      # The execution was reconciled (interrupted) while the tool ran; a
+      # synthetic result message already exists, so the late result is dropped.
+      def superseded_execution(execution)
+        ToolExecution.new(turn.store.load_tool_execution(execution.id))
+      end
+
       def append_result(execution, tool_call, payload, json: payload.to_json, error: false)
         message = turn.conversation.append_message(
           role: "tool",
@@ -104,7 +114,9 @@ module TurnKit
         calls.each do |call|
           payload = { "skipped" => true, "message" => "not executed: turn ended by #{terminal.name}" }
           execution = ToolExecution.new(create_execution(call))
-          attrs = turn.store.update_tool_execution(execution.id, "status" => "cancelled", "result" => payload, "completed_at" => Clock.now)
+          attrs = turn.store.claim_tool_execution(execution.id, from: "running", to: "cancelled", result: payload, completed_at: Clock.now)
+          next unless attrs
+
           append_result(ToolExecution.new(attrs), call, payload)
           turn.emit("tool_call.skipped", id: call.id, name: call.name)
         end
@@ -118,12 +130,24 @@ module TurnKit
         turn.agent.effective_tools.find { |tool| tool.tool_name == name.to_s }
       end
 
+      # Heartbeats while the tool runs so a tool slower than TurnKit.timeout
+      # keeps its turn's stale anchor fresh and is not falsely reconciled.
       def call_tool(tool, arguments, context:)
+        interval = (TurnKit.timeout || 300) / 3.0
+        heartbeat = Thread.new do
+          loop do
+            sleep interval
+            heartbeat!
+          end
+        end
+
         if tool.is_a?(Class)
           tool.call(arguments, context: context)
         else
           tool.class.invoke(tool, arguments, context: context)
         end
+      ensure
+        heartbeat.kill
       end
 
       def normalize_payload(value)

data/lib/turnkit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TurnKit
-  VERSION = "0.4.2"
+  VERSION = "0.5.0"
 end

data/lib/turnkit.rb

@@ -33,6 +33,7 @@ require_relative "turnkit/prompt_context"
 require_relative "turnkit/system_prompt"
 require_relative "turnkit/store"
 require_relative "turnkit/memory_store"
+require_relative "turnkit/reconciliation"
 require_relative "turnkit/compaction"
 require_relative "turnkit/tool"
 require_relative "turnkit/image_tool"
@@ -89,9 +90,7 @@ module TurnKit
   end
 
   def self.reconcile_stale!(before: Clock.now - (timeout || 300))
-    store.find_stale_turns(before: before).each do |turn|
-      store.update_turn(turn.fetch("id"), "status" => "stale", "completed_at" => Clock.now)
-    end
+    Reconciliation.reconcile!(before: before)
   end
 
   def self.check_output_policy(output, constraints: [], context: {})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turnkit
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - Sam Couch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-07-02 00:00:00.000000000 Z
+date: 2026-07-22 00:00:00.000000000 Z
 dependencies: []
 description: TurnKit is a Ruby/Rails agent runtime for durable AI conversations, application
   runs, orchestrator agents, tool calling, skills, sub-agents, context compaction,
@@ -58,6 +58,7 @@ files:
 - lib/turnkit/output_policy.rb
 - lib/turnkit/prompt_context.rb
 - lib/turnkit/prompt_data.rb
+- lib/turnkit/reconciliation.rb
 - lib/turnkit/record.rb
 - lib/turnkit/result.rb
 - lib/turnkit/run.rb