lex-llm-ledger 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 84be271a1d8b2960d1eeef98a01c950ed2d44cab4e2e6d4c8dd6708980d027d4
-  data.tar.gz: 6da1a77c998891e3b2e1a5fae24b1109c51715c762b52aa308ebb73abb886188
+  metadata.gz: b4fb5492195845786b98e664192f84716d049c4144e81e28c5627799cf169b99
+  data.tar.gz: fc6b2b70fbfda9e98c6e7a6e942c81f65193e5c65f46856a50b4fa64ec681972
 SHA512:
-  metadata.gz: 51705c939e26737dfda07ac40df418887d15550e758b4db28aea1e377579b8c65108614cd5abd1ef6a309696c4b90149d1d3dacce6507011a219b90edb7d3a1e
-  data.tar.gz: 163077fbadc935cdeb751645fc59d06a59bf3cd47db7d93d427b721cbdc067f6749c0d6adf6641b1ded3c160c03a252f2382b383d778068b7f59ececcb1b6bb0
+  metadata.gz: e85919d09a173cca54608528ceb4c50a0350912a52c81a5948fff7cf483f9a428e078cbf948b7f90237191335e0e1165ff103d22864df6ffc794c2b922394b9a
+  data.tar.gz: 6de04faae342441c24754f9b186fa92e49b6ab971ad8e4935175ad7206a58da5e288fef33791e97ed785c01f2ca0198606f82b064812a085e69530cf578dfe8d

data/CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## [0.4.0] - 2026-05-17
+
+### Changed
+- Rewrite `Runners::Tools` to write tool audit events to the official `llm_tool_calls` and
+  `llm_tool_call_attempts` tables instead of the legacy `llm_tool_records` table.
+- Each tool audit event produces one `llm_tool_calls` row (linked to the parent
+  `llm_message_inference_responses` row) and one `llm_tool_call_attempts` row containing
+  the execution outcome.
+- Argument and result payloads are stored as SHA-256 fingerprints in `arguments_ref` /
+  `result_ref` (the official schema columns are 255-char refs, not full JSON blobs).
+- Idempotency is enforced via UUID derived from `tool_call.id` (or request/message context);
+  a second write for the same tool call returns `{ result: :duplicate }`.
+- Tool call writes that cannot be linked to an existing inference response are logged at
+  `warn` and dropped gracefully (returns `{ result: :ok }`).
+- Populate `identity_canonical_name` on every insert in `OfficialRecordWriter`:
+  `llm_conversations`, `llm_messages` (user and assistant), `llm_message_inference_requests`,
+  `llm_message_inference_responses`, and `llm_message_inference_metrics`.
+- Populate `identity_principal_id` and `identity_id` on inserts into `llm_messages`,
+  `llm_message_inference_responses`, and `llm_message_inference_metrics`.
+- Backfill `identity_canonical_name` in `enrich_request!` and `enrich_response!` when the
+  enrichment opportunity arrives after a metering-first write.
+- `Runners::Tools` extracts identity via `CallerIdentity.normalize` and writes
+  `identity_canonical_name`, `identity_principal_id`, and `identity_id` to both
+  `llm_tool_calls` and `llm_tool_call_attempts`.
+- `Runners::RegistryAvailability` writes `identity_canonical_name` from AMQP headers or
+  body identity fields when present; `identity_principal_id` and `identity_id` FK columns
+  are not resolved because node/service identities may not be registered in identity tables.
+
+## [0.3.3] - 2026-05-17
+
+### Fixed
+- Extract inline `<think>` / `<thinking>` tags from string responses into `response_thinking_json` at write time instead of leaving them in `response_json`.
+- Fall back to `ThinkingExtractor` when `response_thinking` is absent from the audit payload (covers Ollama, vLLM, and OpenAI-compatible gateways that pass thinking inline).
+- Guard `finish_reason` and `thinking_response` against `String#dig` TypeError when `body[:response]` is a plain string.
+
 ## [0.3.2] - 2026-05-13
 
 ### Fixed

data/lex-llm-ledger.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
   end
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'legion-data',      '>= 1.8.0'
+  spec.add_dependency 'legion-data',      '>= 1.8.7'
   spec.add_dependency 'legion-json',      '>= 1.2'
   spec.add_dependency 'legion-logging',   '>= 1.3'
   spec.add_dependency 'legion-settings',  '>= 1.3'

data/lib/legion/extensions/llm/ledger/backfill/legacy_llm_records.rb

@@ -13,9 +13,9 @@ module Legion
             extend Legion::Logging::Helper
 
             LEGACY_TABLES = %i[
-              llm_prompt_records
-              llm_metering_records
-              llm_tool_records
+              z_archive_llm_prompt_records
+              z_archive_llm_metering_records
+              z_archive_llm_tool_records
               llm_registry_availability_records
             ].freeze
 
@@ -43,11 +43,11 @@ module Legion
 
             def backfill_row(table, row)
               case table
-              when :llm_prompt_records
+              when :z_archive_llm_prompt_records
                 backfill_prompt(row)
-              when :llm_metering_records
+              when :z_archive_llm_metering_records
                 backfill_metering(row)
-              when :llm_tool_records
+              when :z_archive_llm_tool_records
                 backfill_tool(row)
               when :llm_registry_availability_records
                 backfill_registry(row)

data/lib/legion/extensions/llm/ledger/data/migrations/011_add_identity_columns_to_registry_availability.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:llm_registry_availability_records) do
+      add_column :access_scope,            String, size: 20, null: false, default: 'global'
+      add_column :identity_principal_id,   Integer, null: true
+      add_column :identity_id,             Integer, null: true
+      add_column :identity_canonical_name, String, size: 255, null: true
+    end
+
+    alter_table(:llm_registry_availability_records) do
+      add_index :identity_principal_id
+      add_index :identity_id
+      add_index :access_scope
+    end
+  end
+
+  down do
+    alter_table(:llm_registry_availability_records) do
+      drop_index :identity_principal_id
+      drop_index :identity_id
+      drop_index :access_scope
+      drop_column :access_scope
+      drop_column :identity_principal_id
+      drop_column :identity_id
+      drop_column :identity_canonical_name
+    end
+  end
+end

data/lib/legion/extensions/llm/ledger/data/migrations/012_archive_legacy_tables.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    rename_table :llm_metering_records,  :z_archive_llm_metering_records
+    rename_table :llm_prompt_records,    :z_archive_llm_prompt_records
+    rename_table :llm_tool_records,      :z_archive_llm_tool_records
+  end
+
+  down do
+    rename_table :z_archive_llm_metering_records, :llm_metering_records
+    rename_table :z_archive_llm_prompt_records,   :llm_prompt_records
+    rename_table :z_archive_llm_tool_records,     :llm_tool_records
+  end
+end

data/lib/legion/extensions/llm/ledger/runners/prompts.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'legion/extensions/llm/responses/thinking_extractor'
 require_relative '../helpers/caller_identity'
 require_relative '../helpers/json'
 
@@ -96,7 +97,30 @@ module Legion
             end
 
             def response_thinking(body)
-              body[:response_thinking] || body[:thinking] || body.dig(:response, :thinking) || {}
+              thinking = body[:response_thinking] || body[:thinking]
+              thinking ||= body.dig(:response, :thinking) if body[:response].is_a?(Hash)
+              if thinking
+                thinking.is_a?(Hash) ? thinking : { content: thinking }
+              else
+                extract_thinking_from_content(body)
+              end
+            end
+
+            def extract_thinking_from_content(body)
+              content_str = body[:response_content] || body[:response] || body[:content]
+              return {} unless content_str.is_a?(String)
+
+              _clean, extracted = extract_inline_thinking(content_str)
+              extracted ? { content: extracted } : {}
+            end
+
+            def extract_inline_thinking(text)
+              if defined?(::Legion::Extensions::Llm::Responses::ThinkingExtractor)
+                extraction = ::Legion::Extensions::Llm::Responses::ThinkingExtractor.extract(text)
+                [extraction.content, extraction.thinking]
+              else
+                [text, nil]
+              end
             end
 
             def official_prompt_payload(body, ctx, props, headers, expires_at)

data/lib/legion/extensions/llm/ledger/runners/registry_availability.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../helpers/caller_identity'
 require_relative '../helpers/json'
 require_relative '../helpers/persistence_logging'
 
@@ -13,10 +14,11 @@ module Legion
 
             def write_registry_availability_record(payload = nil, metadata = {}, **message)
               payload, metadata = normalize_runner_args(payload, metadata, message)
-              props = metadata[:properties] || {}
+              headers = Helpers::SubscriptionMessage.extract_headers(payload, metadata)
+              props   = metadata[:properties] || {}
 
               body = symbolize(payload)
-              record = build_registry_availability_record(body, props)
+              record = build_registry_availability_record(body, props, headers)
               Helpers::PersistenceLogging.insert_row(
                 ::Legion::Data.connection,
                 :llm_registry_availability_records,
@@ -38,41 +40,55 @@ module Legion
               Helpers::SubscriptionMessage.runner_args(payload, metadata, message)
             end
 
-            def build_registry_availability_record(body, props)
+            def build_registry_availability_record(body, props, headers)
               offering = body[:offering] || {}
               runtime = body[:runtime] || {}
               lane = body[:lane]
 
               {
-                event_id:          body[:event_id],
-                message_id:        props[:message_id],
-                correlation_id:    props[:correlation_id],
-                routing_key:       props[:routing_key],
-                event_type:        body[:event_type].to_s,
-                occurred_at:       body[:occurred_at],
-                offering_id:       offering[:offering_id],
-                provider_family:   offering[:provider_family]&.to_s,
-                provider_instance: offering[:provider_instance]&.to_s,
-                instance_id:       offering[:instance_id]&.to_s,
-                model_family:      offering[:model_family]&.to_s,
-                model_id:          offering[:model],
-                canonical_model:   offering[:canonical_model_alias],
-                provider_model:    offering[:provider_model],
-                usage_type:        offering[:usage_type]&.to_s,
-                transport:         offering[:transport]&.to_s,
-                lane_key:          lane_key(lane),
-                worker_id:         runtime[:worker_id] || runtime[:worker],
-                node_id:           runtime[:node_id] || runtime[:host_id],
-                offering_json:     json_dump(offering),
-                runtime_json:      json_dump(runtime),
-                capacity_json:     json_dump(body[:capacity] || {}),
-                health_json:       json_dump(body[:health] || {}),
-                lane_json:         json_dump(lane || {}),
-                metadata_json:     json_dump(body[:metadata] || {}),
-                inserted_at:       Time.now.utc
+                event_id:                body[:event_id],
+                message_id:              props[:message_id],
+                correlation_id:          props[:correlation_id],
+                routing_key:             props[:routing_key],
+                event_type:              body[:event_type].to_s,
+                occurred_at:             body[:occurred_at],
+                offering_id:             offering[:offering_id],
+                provider_family:         offering[:provider_family]&.to_s,
+                provider_instance:       offering[:provider_instance]&.to_s,
+                instance_id:             offering[:instance_id]&.to_s,
+                model_family:            offering[:model_family]&.to_s,
+                model_id:                offering[:model],
+                canonical_model:         offering[:canonical_model_alias],
+                provider_model:          offering[:provider_model],
+                usage_type:              offering[:usage_type]&.to_s,
+                transport:               offering[:transport]&.to_s,
+                lane_key:                lane_key(lane),
+                worker_id:               runtime[:worker_id] || runtime[:worker],
+                node_id:                 runtime[:node_id] || runtime[:host_id],
+                identity_canonical_name: extract_canonical_name(body, headers),
+                offering_json:           json_dump(offering),
+                runtime_json:            json_dump(runtime),
+                capacity_json:           json_dump(body[:capacity] || {}),
+                health_json:             json_dump(body[:health] || {}),
+                lane_json:               json_dump(lane || {}),
+                metadata_json:           json_dump(body[:metadata] || {}),
+                inserted_at:             Time.now.utc
               }
             end
 
+            # Extract identity_canonical_name from AMQP headers or body.
+            # No FK resolution — node/service identities may not be registered in
+            # identity tables, but the canonical name string is still valuable for
+            # tracking which identity is publishing which model availability.
+            def extract_canonical_name(body, headers)
+              raw = headers['x-legion-identity'] ||
+                    body.dig(:identity, :identity) ||
+                    body.dig(:identity, :canonical_name)
+              return nil unless raw && !raw.to_s.empty? # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              raw.to_s
+            end
+
             def lane_key(lane)
               if lane.is_a?(String)
                 lane

data/lib/legion/extensions/llm/ledger/runners/tools.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'digest'
+require 'securerandom'
 require_relative '../helpers/caller_identity'
 require_relative '../helpers/json'
 require_relative '../helpers/persistence_logging'
@@ -21,19 +23,25 @@ module Legion
               ctx  = body[:message_context] || {}
               tool = body[:tool_call]       || {}
 
-              expires_at = Helpers::Retention.resolve(
+              Helpers::Retention.resolve(
                 retention:    headers['x-legion-retention'],
                 contains_phi: headers['x-legion-contains-phi'] == 'true'
               )
 
-              record = build_tool_record(body, ctx, tool, props, headers, expires_at)
-              Helpers::PersistenceLogging.insert_row(
-                ::Legion::Data.connection,
-                :llm_tool_records,
-                record,
-                operation: 'write_tool_record'
-              )
-              { result: :ok }
+              db = ::Legion::Data.connection
+              write_result = [:ok]
+              db.transaction do
+                response                     = find_or_resolve_response(db, body, ctx, props, headers)
+                identity_attrs               = extract_identity_attrs(body, headers, db)
+                tool_call_row, new_tool_call = find_or_create_tool_call(db, response, body, ctx, tool, headers, identity_attrs)
+                if tool_call_row && !new_tool_call
+                  write_result[0] = :duplicate
+                elsif new_tool_call
+                  find_or_create_tool_call_attempt(db, tool_call_row, tool, body, props, headers, identity_attrs)
+                end
+              end
+
+              { result: write_result[0] }
             rescue Sequel::UniqueConstraintViolation => e
               log.warn("write_tool_record duplicate insert ignored: #{e.message}")
               { result: :duplicate }
@@ -54,46 +62,202 @@ module Legion
               Helpers::SubscriptionMessage.runner_args(payload, metadata, message)
             end
 
-            def build_tool_record(body, ctx, tool, props, headers, expires_at) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
-              src      = tool[:source] || {}
-              cls      = body[:classification] || {}
-              ts       = body[:timestamps] || {}
-              tracing    = body[:tracing] || {}
-              caller_raw = body[:caller] || {}
-              identity   = body[:identity] || {}
+            # Resolve the llm_message_inference_responses row this tool call belongs to.
+            # Returns nil if we cannot link at all.
+            def find_or_resolve_response(db, body, ctx, props, headers)
+              request_ref = ctx[:request_id] || body[:request_id] ||
+                            props[:correlation_id] || headers['x-legion-llm-request-id']
+              return nil unless request_ref # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              request = db[:llm_message_inference_requests].where(request_ref: request_ref).first
+              return nil unless request # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              db[:llm_message_inference_responses]
+                .where(message_inference_request_id: request[:id]).first
+            end
+
+            def find_or_create_tool_call(db, response, body, ctx, tool, headers, identity_attrs)
+              tool_uuid = derive_tool_call_uuid(body, ctx, tool, headers)
+              existing  = db[:llm_tool_calls].where(uuid: tool_uuid).first
+              return [existing, false] if existing # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              response_id = resolve_response_id(db, response, body, ctx, headers, tool_uuid)
+              return [nil, false] unless response_id # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              next_index = db[:llm_tool_calls]
+                           .where(message_inference_response_id: response_id)
+                           .max(:tool_call_index).to_i + 1
+
+              src    = tool[:source] || {}
+              status = tool[:status] || headers['x-legion-tool-status'] || 'success'
+              ts     = body[:timestamps] || {}
+
+              id = insert_with_savepoint(db, :llm_tool_calls, {
+                                           uuid:                          tool_uuid,
+                                           message_inference_response_id: response_id,
+                                           tool_call_index:               next_index,
+                                           provider_tool_call_ref:        tool[:id],
+                                           tool_name:                     tool[:name] || headers['x-legion-tool-name'],
+                                           tool_source_type:              src[:type] || headers['x-legion-tool-source-type'],
+                                           tool_source_server:            src[:server] || headers['x-legion-tool-source-server'],
+                                           status:                        status,
+                                           requested_at:                  ts[:tool_start] || tool[:started_at],
+                                           completed_at:                  ts[:tool_end] || tool[:finished_at],
+                                           **identity_attrs,
+                                           inserted_at:                   Time.now.utc
+                                         }, operation: 'write_tool_record.tool_call')
+              [db[:llm_tool_calls][id: id], true]
+            rescue Sequel::UniqueConstraintViolation => e
+              log.debug("[ledger] tool_call collision resolved uuid=#{tool_uuid} error=#{e.class}")
+              row = db[:llm_tool_calls].where(uuid: tool_uuid).first
+              raise(e) unless row
+
+              [row, false]
+            end
+
+            # Extract or fall back to find a response_id for linking the tool call.
+            def resolve_response_id(db, response, body, ctx, headers, tool_uuid)
+              return response[:id] if response # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              fallback = fallback_response_for_conversation(db, body, ctx, headers)
+              return fallback[:id] if fallback # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              log.warn("[ledger] write_tool_record: no response row found for tool call uuid=#{tool_uuid}, skipping")
+              nil
+            end
+
+            def fallback_response_for_conversation(db, body, ctx, headers)
+              conv_id = ctx[:conversation_id] || body[:conversation_id] ||
+                        headers['x-legion-llm-conversation-id']
+              return nil unless conv_id # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              conv = db[:llm_conversations].where(uuid: stable_uuid(conv_id)).first ||
+                     db[:llm_conversations].where(uuid: conv_id).first
+              return nil unless conv # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              db[:llm_message_inference_responses]
+                .join(:llm_message_inference_requests,
+                      id: :message_inference_request_id)
+                .where(Sequel[:llm_message_inference_requests][:conversation_id] => conv[:id])
+                .order(Sequel.desc(Sequel[:llm_message_inference_responses][:id]))
+                .select_all(:llm_message_inference_responses)
+                .first
+            end
+
+            def find_or_create_tool_call_attempt(db, tool_call_row, tool, body, props, headers, identity_attrs) # rubocop:disable Metrics/CyclomaticComplexity
+              return nil unless tool_call_row # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              tool_call_id = tool_call_row[:id]
+              attempt_no   = db[:llm_tool_call_attempts]
+                             .where(tool_call_id: tool_call_id).max(:attempt_no).to_i + 1
+              attempt_uuid = derive_attempt_uuid(tool_call_row[:uuid], attempt_no)
+
+              existing = db[:llm_tool_call_attempts].where(uuid: attempt_uuid).first
+              return existing if existing # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              status     = tool[:status] || headers['x-legion-tool-status'] || 'success'
+              error_info = tool[:error] || body[:error]
+              error_hash = error_info.is_a?(Hash) ? error_info : {}
+              ts         = body[:timestamps] || {}
+              runner_ref = body[:worker_id] || body[:runner_ref] || props[:app_id]
+
+              id = insert_with_savepoint(db, :llm_tool_call_attempts, {
+                                           uuid:           attempt_uuid,
+                                           tool_call_id:   tool_call_id,
+                                           attempt_no:     attempt_no,
+                                           runner_ref:     runner_ref,
+                                           status:         status,
+                                           error_category: error_hash[:category] || error_hash[:type],
+                                           error_code:     error_hash[:code],
+                                           error_message:  error_info.is_a?(String) ? error_info : error_hash[:message],
+                                           duration_ms:    tool[:duration_ms].to_i,
+                                           arguments_ref:  sha256_ref(tool[:arguments]),
+                                           result_ref:     sha256_ref(tool[:result] || body[:result]),
+                                           started_at:     ts[:tool_start] || tool[:started_at],
+                                           ended_at:       ts[:tool_end] || tool[:finished_at],
+                                           **identity_attrs,
+                                           inserted_at:    Time.now.utc
+                                         }, operation: 'write_tool_record.attempt')
+              db[:llm_tool_call_attempts][id: id]
+            rescue Sequel::UniqueConstraintViolation => e
+              log.debug("[ledger] tool_call_attempt collision resolved uuid=#{attempt_uuid} error=#{e.class}")
+              db[:llm_tool_call_attempts].where(uuid: attempt_uuid).first || raise(e)
+            end
+
+            def extract_identity_attrs(body, headers, db)
               caller_identity = Helpers::CallerIdentity.normalize(
-                caller_raw: caller_raw, identity: identity, headers: headers
+                caller_raw: body[:caller],
+                identity:   body[:identity],
+                headers:    headers
               )
-              agent = body[:agent] || {}
+              # raw_identity may carry a "type:value" prefix that OfficialRecordWriter
+              # knows how to parse; keep it intact for FK resolution.
+              raw_identity   = caller_identity[:identity]
+              canonical_name = raw_identity
+              # Strip "type:" prefix added by CallerIdentity for generic identities
+              if canonical_name&.include?(':') && !canonical_name&.include?('@')
+                _prefix, remainder = canonical_name.split(':', 2)
+                canonical_name = remainder if remainder && !remainder.empty?
+              end
+
+              refs = resolve_tool_identity(db, body, raw_identity)
 
               {
-                message_id:           props[:message_id] || body[:message_id],
-                correlation_id:       props[:correlation_id] || body[:correlation_id] || tracing[:correlation_id],
-                conversation_id:      ctx[:conversation_id] || body[:conversation_id] || headers['x-legion-llm-conversation-id'],
-                message_id_ctx:       ctx[:message_id],
-                parent_message_id:    ctx[:parent_message_id],
-                message_seq:          ctx[:message_seq],
-                request_id:           ctx[:request_id] || body[:request_id] || headers['x-legion-llm-request-id'],
-                exchange_id:          ctx[:exchange_id] || body[:exchange_id],
-                tool_call_id:         tool[:id],
-                tool_name:            tool[:name] || headers['x-legion-tool-name'],
-                tool_source_type:     src[:type] || headers['x-legion-tool-source-type'],
-                tool_source_server:   src[:server] || headers['x-legion-tool-source-server'],
-                tool_status:          tool[:status] || headers['x-legion-tool-status'],
-                tool_duration_ms:     tool[:duration_ms].to_i,
-                arguments_json:       Helpers::Json.dump(tool[:arguments] || {}),
-                result_json:          Helpers::Json.dump(tool[:result] || body[:result]),
-                error_json:           Helpers::Json.dump(tool[:error] || body[:error]),
-                caller_identity:      caller_identity[:identity],
-                agent_id:             agent[:id],
-                classification_level: cls[:level] || headers['x-legion-classification'],
-                contains_phi:         Helpers::Queries.phi_flag?(cls, headers),
-                retention_policy:     headers['x-legion-retention'] || 'default',
-                expires_at:           expires_at,
-                tool_start_at:        ts[:tool_start] || tool[:started_at],
-                tool_end_at:          ts[:tool_end] || tool[:finished_at],
-                inserted_at:          Time.now.utc
-              }
+                identity_canonical_name: canonical_name,
+                identity_principal_id:   refs[:principal_id],
+                identity_id:             refs[:identity_id]
+              }.compact
+            end
+
+            def resolve_tool_identity(db, body, raw_identity)
+              return {} unless raw_identity
+              return {} unless Writers::OfficialRecordWriter.identity_tables_available?(db)
+
+              # Merge the header-resolved identity string into the body so that
+              # OfficialRecordWriter.resolve_identity can find it via
+              # parsed_identity_descriptor even when identity came solely from
+              # AMQP headers and is absent from the payload body.
+              body_with_identity = raw_identity ? body.merge(caller_identity: raw_identity) : body
+              Writers::OfficialRecordWriter.resolve_identity(db, body_with_identity)
+            rescue StandardError => e
+              handle_exception(e, level: :warn, handled: true, operation: 'write_tool_record.identity_resolution')
+              {}
+            end
+
+            def derive_tool_call_uuid(body, ctx, tool, headers)
+              ref = tool[:id] ||
+                    ctx[:request_id] ||
+                    body[:request_id] ||
+                    headers['x-legion-llm-request-id'] ||
+                    ctx[:message_id] ||
+                    (body[:properties] || {})[:message_id]
+              stable_uuid("tool_call:#{ref || SecureRandom.uuid}")
+            end
+
+            def derive_attempt_uuid(tool_call_uuid, attempt_no)
+              stable_uuid("attempt:#{tool_call_uuid}:#{attempt_no}")
+            end
+
+            def sha256_ref(value)
+              return nil if value.nil? # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              raw = value.is_a?(String) ? value : Helpers::Json.dump(value)
+              Digest::SHA256.hexdigest(raw)[0, 64]
+            end
+
+            def stable_uuid(value)
+              raw = value.to_s
+              return raw if raw.length <= 36 # rubocop:disable Legion/Extension/RunnerReturnHash
+
+              hex = Digest::SHA256.hexdigest(raw)[0, 32]
+              "#{hex[0, 8]}-#{hex[8, 4]}-#{hex[12, 4]}-#{hex[16, 4]}-#{hex[20, 12]}"
+            end
+
+            def insert_with_savepoint(db, table, attributes, operation:)
+              db.transaction(savepoint: true) do
+                Helpers::PersistenceLogging.insert_row(db, table, attributes,
+                                                       operation: operation, warn_on_unique: false)
+              end
             end
 
             include Legion::Extensions::Helpers::Lex if Legion::Extensions.const_defined?(:Helpers, false) &&

data/lib/legion/extensions/llm/ledger/version.rb

@@ -4,7 +4,7 @@ module Legion
   module Extensions
     module Llm
       module Ledger
-        VERSION = '0.3.2'
+        VERSION = '0.4.0'
       end
     end
   end

data/lib/legion/extensions/llm/ledger/writers/official_record_writer.rb

@@ -3,6 +3,7 @@
 require 'digest'
 require 'securerandom'
 require 'legion/logging'
+require 'legion/extensions/llm/responses/thinking_extractor'
 require_relative '../helpers/json'
 require_relative '../helpers/persistence_logging'
 
@@ -57,17 +58,18 @@ module Legion
               return existing if existing
 
               id = insert_with_savepoint(db, :llm_conversations, {
-                                           uuid:                 uuid,
-                                           title:                body[:title] || body[:conversation_title],
-                                           classification_level: classification_level(body),
-                                           contains_phi:         contains_phi?(body),
-                                           contains_pii:         contains_pii?(body),
-                                           retention_policy:     body[:retention_policy] || 'default',
-                                           expires_at:           body[:expires_at],
-                                           recorded_at:          recorded_at(body),
-                                           inserted_at:          Time.now.utc,
-                                           created_at:           Time.now.utc,
-                                           updated_at:           Time.now.utc
+                                           uuid:                    uuid,
+                                           title:                   body[:title] || body[:conversation_title],
+                                           classification_level:    classification_level(body),
+                                           contains_phi:            contains_phi?(body),
+                                           contains_pii:            contains_pii?(body),
+                                           retention_policy:        body[:retention_policy] || 'default',
+                                           expires_at:              body[:expires_at],
+                                           identity_canonical_name: identity_canonical_name(body),
+                                           recorded_at:             recorded_at(body),
+                                           inserted_at:             Time.now.utc,
+                                           created_at:              Time.now.utc,
+                                           updated_at:              Time.now.utc
                                          }, operation: 'official_record_writer.conversation')
               db[:llm_conversations][id: id]
             rescue Sequel::UniqueConstraintViolation => e
@@ -86,16 +88,19 @@ module Legion
               seq = body[:message_seq] ? integer(body[:message_seq]) : next_message_seq(db, conversation)
               begin
                 id = insert_with_savepoint(db, :llm_messages, {
-                                             uuid:            uuid,
-                                             conversation_id: conversation[:id],
-                                             seq:             seq,
-                                             role:            'user',
-                                             content_type:    'text',
-                                             content:         request_content(body),
-                                             input_tokens:    tokens(body)[:input_tokens],
-                                             output_tokens:   0,
-                                             created_at:      recorded_at(body),
-                                             inserted_at:     Time.now.utc
+                                             uuid:                    uuid,
+                                             conversation_id:         conversation[:id],
+                                             seq:                     seq,
+                                             role:                    'user',
+                                             content_type:            'text',
+                                             content:                 request_content(body),
+                                             input_tokens:            tokens(body)[:input_tokens],
+                                             output_tokens:           0,
+                                             identity_principal_id:   caller_identity_refs(db, body)[:principal_id],
+                                             identity_id:             caller_identity_refs(db, body)[:identity_id],
+                                             identity_canonical_name: identity_canonical_name(body),
+                                             created_at:              recorded_at(body),
+                                             inserted_at:             Time.now.utc
                                            }, operation: 'official_record_writer.user_message')
                 db[:llm_messages][id: id]
               rescue Sequel::UniqueConstraintViolation => e
@@ -113,28 +118,31 @@ module Legion
               operation = operation(body)
               caller_refs = caller_identity_refs(db, body)
               id = insert_with_savepoint(db, :llm_message_inference_requests, {
-                                           uuid:                  stable_uuid(request_id),
-                                           conversation_id:       conversation[:id],
-                                           latest_message_id:     latest_message&.dig(:id),
-                                           caller_principal_id:   caller_refs[:principal_id],
-                                           caller_identity_id:    caller_refs[:identity_id],
-                                           runtime_caller_type:   caller_type(body),
-                                           request_ref:           request_id,
-                                           correlation_ref:       correlation_id(body),
-                                           correlation_id:        correlation_id(body),
-                                           exchange_ref:          body[:exchange_id],
-                                           request_type:          operation,
-                                           operation:             operation,
-                                           idempotency_key:       body[:idempotency_key] || request_id,
-                                           status:                'responded',
-                                           context_message_count: Array(body.dig(:request, :messages) || body[:messages]).size,
-                                           request_capture_mode:  'full',
-                                           request_json:          json_dump(request_payload(body)),
-                                           classification_level:  classification_level(body),
-                                           cost_center:           billing(body)[:cost_center],
-                                           budget_key:            billing(body)[:budget_id] || billing(body)[:budget_key],
-                                           requested_at:          recorded_at(body),
-                                           inserted_at:           Time.now.utc
+                                           uuid:                    stable_uuid(request_id),
+                                           conversation_id:         conversation[:id],
+                                           latest_message_id:       latest_message&.dig(:id),
+                                           caller_principal_id:     caller_refs[:principal_id],
+                                           caller_identity_id:      caller_refs[:identity_id],
+                                           identity_canonical_name: identity_canonical_name(body),
+                                           runtime_caller_type:     caller_type(body),
+                                           runtime_caller_class:    runtime_caller_class(body),
+                                           runtime_caller_client:   runtime_caller_client(body),
+                                           request_ref:             request_id,
+                                           correlation_ref:         correlation_id(body),
+                                           correlation_id:          correlation_id(body),
+                                           exchange_ref:            body[:exchange_id],
+                                           request_type:            operation,
+                                           operation:               operation,
+                                           idempotency_key:         body[:idempotency_key] || request_id,
+                                           status:                  'responded',
+                                           context_message_count:   Array(body.dig(:request, :messages) || body[:messages]).size,
+                                           request_capture_mode:    'full',
+                                           request_json:            json_dump(request_payload(body)),
+                                           classification_level:    classification_level(body),
+                                           cost_center:             billing(body)[:cost_center],
+                                           budget_key:              billing(body)[:budget_id] || billing(body)[:budget_key],
+                                           requested_at:            recorded_at(body),
+                                           inserted_at:             Time.now.utc
                                          }, operation: 'official_record_writer.inference_request')
               db[:llm_message_inference_requests][id: id]
             rescue Sequel::UniqueConstraintViolation => e
@@ -164,6 +172,9 @@ module Legion
                                              content:                      response_content(body),
                                              input_tokens:                 0,
                                              output_tokens:                tokens(body)[:output_tokens],
+                                             identity_principal_id:        caller_identity_refs(db, body)[:principal_id],
+                                             identity_id:                  caller_identity_refs(db, body)[:identity_id],
+                                             identity_canonical_name:      identity_canonical_name(body),
                                              created_at:                   recorded_at(body),
                                              inserted_at:                  Time.now.utc
                                            }, operation: 'official_record_writer.response_message')
@@ -201,6 +212,9 @@ module Legion
                                            response_json:                json_dump(visible_response(body)),
                                            response_thinking_json:       json_dump(thinking_response(body)),
                                            dispatch_path:                body[:dispatch_path] || body[:tier],
+                                           identity_principal_id:        caller_identity_refs(db, body)[:principal_id],
+                                           identity_id:                  caller_identity_refs(db, body)[:identity_id],
+                                           identity_canonical_name:      identity_canonical_name(body),
                                            responded_at:                 recorded_at(body),
                                            inserted_at:                  Time.now.utc
                                          }, operation: 'official_record_writer.inference_response')
@@ -223,6 +237,7 @@ module Legion
               update_if_missing(updates, existing, :provider_instance, provider_instance(body))
               update_if_missing(updates, existing, :finish_reason, finish_reason(body))
               update_if_missing(updates, existing, :dispatch_path, body[:dispatch_path] || body[:tier])
+              update_if_missing(updates, existing, :identity_canonical_name, identity_canonical_name(body))
 
               response_json = json_dump(visible_response(body))
               update_if_placeholder(updates, existing, :response_json, response_json)
@@ -267,6 +282,9 @@ module Legion
                                            currency:                      body[:currency] || 'USD',
                                            cost_center:                   billing(body)[:cost_center],
                                            budget_key:                    billing(body)[:budget_id] || billing(body)[:budget_key],
+                                           identity_principal_id:         caller_identity_refs(db, body)[:principal_id],
+                                           identity_id:                   caller_identity_refs(db, body)[:identity_id],
+                                           identity_canonical_name:       identity_canonical_name(body),
                                            recorded_at:                   recorded_at(body),
                                            inserted_at:                   Time.now.utc
                                          }, operation: 'official_record_writer.inference_metric')
@@ -309,6 +327,9 @@ module Legion
               updates[:caller_identity_id] = caller_refs[:identity_id] if existing[:caller_identity_id].nil? && caller_refs[:identity_id]
               updates[:caller_principal_id] = caller_refs[:principal_id] if existing[:caller_principal_id].nil? && caller_refs[:principal_id]
               updates[:runtime_caller_type] = caller_type(body) if existing[:runtime_caller_type].nil? && caller_type(body)
+              update_if_missing(updates, existing, :runtime_caller_class, runtime_caller_class(body))
+              update_if_missing(updates, existing, :runtime_caller_client, runtime_caller_client(body))
+              update_if_missing(updates, existing, :identity_canonical_name, identity_canonical_name(body))
 
               request_json = json_dump(request_payload(body))
               updates[:request_json] = request_json if existing[:request_json].to_s == '{}' && request_json != '{}'
@@ -341,6 +362,16 @@ module Legion
               parsed_identity_descriptor(body)[:kind]
             end
 
+            def runtime_caller_class(body)
+              body.dig(:caller, :class) || body.dig(:caller, :caller_class) ||
+                body.dig(:caller, :source_class) || body[:runtime_caller_class]
+            end
+
+            def runtime_caller_client(body)
+              body.dig(:caller, :client) || body.dig(:caller, :user_agent) ||
+                body[:runtime_caller_client]
+            end
+
             def caller_identity_refs(db, body)
               body[:__ledger_caller_identity_refs] ||= begin
                 explicit_identity_id = integer_or_nil(body[:caller_identity_id] || body.dig(:caller, :requested_by, :id))
@@ -580,18 +611,38 @@ module Legion
 
             def visible_response(body)
               response = body[:response] || body[:response_content] || body[:content] || {}
-              return { content: response } if response.is_a?(String)
+              if response.is_a?(String)
+                clean, _thinking = extract_inline_thinking(response)
+                return { content: clean }
+              end
               return { content: response[:content] } if response.is_a?(Hash) && response.key?(:content)
 
               response.is_a?(Hash) ? response.except(:thinking) : { content: response.to_s }
             end
 
             def thinking_response(body)
-              thinking = body[:response_thinking] || body[:thinking] || body.dig(:response, :thinking)
-              return {} if thinking.nil?
-              return { content: thinking } if thinking.is_a?(String)
+              thinking = body[:response_thinking] || body[:thinking]
+              thinking ||= body.dig(:response, :thinking) if body[:response].is_a?(Hash)
+              if thinking
+                return { content: thinking } if thinking.is_a?(String)
+
+                return thinking
+              end
+
+              content_str = body[:response_content] || body[:response] || body[:content]
+              return {} unless content_str.is_a?(String)
 
-              thinking
+              _clean, extracted = extract_inline_thinking(content_str)
+              extracted ? { content: extracted } : {}
+            end
+
+            def extract_inline_thinking(text)
+              if defined?(::Legion::Extensions::Llm::Responses::ThinkingExtractor)
+                extraction = ::Legion::Extensions::Llm::Responses::ThinkingExtractor.extract(text)
+                [extraction.content, extraction.thinking]
+              else
+                [text, nil]
+              end
             end
 
             def response_content(body)
@@ -599,7 +650,10 @@ module Legion
             end
 
             def finish_reason(body)
-              body[:finish_reason] || body.dig(:response, :finish_reason) || body.dig(:response, :stop, :reason)
+              return body[:finish_reason] if body[:finish_reason]
+              return nil unless body[:response].is_a?(Hash)
+
+              body.dig(:response, :finish_reason) || body.dig(:response, :stop, :reason)
             end
 
             def classification_level(body)
@@ -667,6 +721,10 @@ module Legion
               end
             end
 
+            def identity_canonical_name(body)
+              parsed_identity_descriptor(body)[:canonical_name]
+            end
+
             def present?(value)
               !value.nil? && !(value.respond_to?(:empty?) && value.empty?)
             end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-llm-ledger
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Esity
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: 1.8.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: 1.8.7
 - !ruby/object:Gem::Dependency
   name: legion-json
   requirement: !ruby/object:Gem::Requirement
@@ -207,6 +207,8 @@ files:
 - lib/legion/extensions/llm/ledger/data/migrations/008_relax_message_id_not_null.rb
 - lib/legion/extensions/llm/ledger/data/migrations/009_add_caller_to_metering.rb
 - lib/legion/extensions/llm/ledger/data/migrations/010_add_response_thinking_json_to_prompt_records.rb
+- lib/legion/extensions/llm/ledger/data/migrations/011_add_identity_columns_to_registry_availability.rb
+- lib/legion/extensions/llm/ledger/data/migrations/012_archive_legacy_tables.rb
 - lib/legion/extensions/llm/ledger/helpers/caller_identity.rb
 - lib/legion/extensions/llm/ledger/helpers/decryption.rb
 - lib/legion/extensions/llm/ledger/helpers/json.rb