lex-llm-ledger 0.2.5 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 83376ade1924f74e2c734ccf0b9e67b1a1db0ad0494a883469fb86a4eb1763a1
-  data.tar.gz: 83c5f37ff7b8d63c2865e3f758f0ecbb116746d93817a2b67aadc7a14f66f503
+  metadata.gz: bedae52b0629ce6755fc321019b9c35917e8d4f8a304c7294956cbba864fa676
+  data.tar.gz: c3f171de566906d37cb02c755f0336df1ce0f09dcdf769e84961f574e3b9d77e
 SHA512:
-  metadata.gz: dcd276003a26041f9b0e5aa373547d7d03f74010c1a7fd011cd0a6fd1a9ded022425476ff95e7efc5a466c94c8362b699e5d1e3f8468609eec2bc6391e02b016
-  data.tar.gz: 0b0168807bc938bdcba4d3a2955c3d5e86f58a6138a93970fea1addbd2bc1f38fef44ed5bbedfcd7d01c3d26e23c451920c645d79acce603b26b98ae7bfffddc
+  metadata.gz: f8623ee921c7e0b610217925aea11e9f0f5b315b89b56d2ea239aefa9b0ac9d17b298b1f95978f784f6932ab005fa906c5c32bdd78d1f3eb47dcefb73f3762f9
+  data.tar.gz: 278ed50a3f63394470ede944211cc0893e1c8c52bda2200209161040eba3a07a0d62aa47dbfbf37e913ecdaa8d474a4144f1961b60b3e0dcfdb3af936daf155e

data/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## [0.3.0] - 2026-05-08
+
+### Changed
+- Renamed all `portable_identity_*` table references to canonical identity table names (`identities`, `identity_principals`, `identity_providers`).
+- Renamed internal methods: `resolve_portable_identity` → `resolve_identity`, `find_or_create_portable_identity` → `find_or_create_identity`.
+
+## [0.2.9] - 2026-05-07
+
+### Fixed
+- Prefer current publisher identity payloads and AMQP identity headers over stale `caller.requested_by.id` values when normalizing prompt, metering, and tool audit events.
+- Resolve canonical caller identity strings into portable identity provider, principal, and identity rows before writing official inference request foreign keys.
+- Store `runtime_caller_type` from explicit type fields instead of identity strings.
+
+## [0.2.8] - 2026-05-07
+
+### Fixed
+- Extract caller identity from audit event structure (`identity.identity`, `caller.requested_by.identity`) instead of missing top-level `caller_identity_id` / `caller_principal_id` keys.
+- Enrich existing request rows when prompt audit arrives after metering (backfills `caller_identity_id`, `caller_principal_id`, `runtime_caller_type`, `request_json`, `context_message_count`).
+
+## [0.2.7] - 2026-05-07
+
+### Fixed
+- Enrich existing inference response rows when a richer payload arrives (prompt audit backfills `response_message_id`, `response_json`, `tier`, `finish_reason` that metering left null).
+
+## [0.2.6] - 2026-05-07
+
+### Fixed
+- Add `Legion::Logging::Helper` to `OfficialRecordWriter` so `log` is available in rescue blocks.
+- Wrap message inserts in savepoints so PostgreSQL unique constraint violations don't poison the parent transaction and cause `PG::InFailedSqlTransaction` on the fallback query.
+
 ## [0.2.5] - 2026-05-06
 
 ### Fixed

data/lib/legion/extensions/llm/ledger/helpers/caller_identity.rb

@@ -17,21 +17,21 @@ module Legion
               identity_hash = identity.is_a?(Hash) ? identity : {}
               extension = hash_value(caller_hash, :extension)
               type = first_present(
-                hash_value(caller, :type),
                 hash_value(identity_hash, :type),
                 header_value(headers, 'x-legion-caller-type'),
+                hash_value(caller, :type),
                 extension && 'extension'
               )
 
               raw_identity = first_present(
-                hash_value(caller, :id),
-                hash_value(caller, :canonical_name),
                 hash_value(identity_hash, :id),
                 hash_value(identity_hash, :canonical_name),
                 hash_value(identity_hash, :identity),
                 hash_value(identity_hash, :username),
                 header_value(headers, 'x-legion-identity'),
                 header_value(headers, 'x-legion-caller-identity'),
+                hash_value(caller, :id),
+                hash_value(caller, :canonical_name),
                 hash_value(caller, :identity),
                 hash_value(caller, :username),
                 extension && "extension:#{extension}"

data/lib/legion/extensions/llm/ledger/version.rb

@@ -4,7 +4,7 @@ module Legion
   module Extensions
     module Llm
       module Ledger
-        VERSION = '0.2.5'
+        VERSION = '0.3.0'
       end
     end
   end

data/lib/legion/extensions/llm/ledger/writers/official_record_writer.rb

@@ -2,6 +2,7 @@
 
 require 'digest'
 require 'securerandom'
+require 'legion/logging'
 require_relative '../helpers/json'
 require_relative '../helpers/persistence_logging'
 
@@ -11,6 +12,8 @@ module Legion
       module Ledger
         module Writers
           module OfficialRecordWriter
+            extend Legion::Logging::Helper
+
             module_function
 
             def write_prompt(payload)
@@ -76,34 +79,46 @@ module Legion
               return existing if existing
 
               seq = body[:message_seq] ? integer(body[:message_seq]) : next_message_seq(db, conversation)
-              id = insert_row(db, :llm_messages, {
-                                uuid:            uuid,
-                                conversation_id: conversation[:id],
-                                seq:             seq,
-                                role:            'user',
-                                content_type:    'text',
-                                content:         request_content(body),
-                                input_tokens:    tokens(body)[:input_tokens],
-                                output_tokens:   0,
-                                created_at:      recorded_at(body),
-                                inserted_at:     Time.now.utc
-                              }, operation: 'official_record_writer.user_message')
-              db[:llm_messages][id: id]
+              begin
+                id = db.transaction(savepoint: true) do
+                  insert_row(db, :llm_messages, {
+                               uuid:            uuid,
+                               conversation_id: conversation[:id],
+                               seq:             seq,
+                               role:            'user',
+                               content_type:    'text',
+                               content:         request_content(body),
+                               input_tokens:    tokens(body)[:input_tokens],
+                               output_tokens:   0,
+                               created_at:      recorded_at(body),
+                               inserted_at:     Time.now.utc
+                             }, operation: 'official_record_writer.user_message')
+                end
+                db[:llm_messages][id: id]
+              rescue Sequel::UniqueConstraintViolation => e
+                log.debug("[ledger] seq collision resolved uuid=#{uuid} conversation_id=#{conversation[:id]} error=#{e.class}")
+                db[:llm_messages].where(uuid: uuid).first ||
+                  db[:llm_messages].where(conversation_id: conversation[:id], seq: seq).first
+              end
             end
 
             def find_or_create_request(db, conversation, latest_message, body)
               request_id = request_ref(body)
               existing = db[:llm_message_inference_requests].where(request_ref: request_id).first
-              return existing if existing
+              if existing
+                enrich_request!(db, existing, body)
+                return existing
+              end
 
               operation = operation(body)
+              caller_refs = caller_identity_refs(db, body)
               id = insert_row(db, :llm_message_inference_requests, {
                                 uuid:                  stable_uuid(request_id),
                                 conversation_id:       conversation[:id],
                                 latest_message_id:     latest_message[:id],
-                                caller_principal_id:   body[:caller_principal_id],
-                                caller_identity_id:    body[:caller_identity_id],
-                                runtime_caller_type:   body[:caller_type],
+                                caller_principal_id:   caller_refs[:principal_id],
+                                caller_identity_id:    caller_refs[:identity_id],
+                                runtime_caller_type:   caller_type(body),
                                 request_ref:           request_id,
                                 correlation_ref:       correlation_id(body),
                                 correlation_id:        correlation_id(body),
@@ -130,27 +145,39 @@ module Legion
               return existing if existing
 
               latest = db[:llm_messages][id: request[:latest_message_id]]
-              id = insert_row(db, :llm_messages, {
-                                uuid:                         uuid,
-                                conversation_id:              conversation[:id],
-                                parent_message_id:            latest&.dig(:id),
-                                message_inference_request_id: request[:id],
-                                seq:                          (latest&.dig(:seq) || 1) + 1,
-                                role:                         'assistant',
-                                content_type:                 'text',
-                                content:                      response_content(body),
-                                input_tokens:                 0,
-                                output_tokens:                tokens(body)[:output_tokens],
-                                created_at:                   recorded_at(body),
-                                inserted_at:                  Time.now.utc
-                              }, operation: 'official_record_writer.response_message')
-              db[:llm_messages][id: id]
+              seq = (latest&.dig(:seq) || 1) + 1
+              begin
+                id = db.transaction(savepoint: true) do
+                  insert_row(db, :llm_messages, {
+                               uuid:                         uuid,
+                               conversation_id:              conversation[:id],
+                               parent_message_id:            latest&.dig(:id),
+                               message_inference_request_id: request[:id],
+                               seq:                          seq,
+                               role:                         'assistant',
+                               content_type:                 'text',
+                               content:                      response_content(body),
+                               input_tokens:                 0,
+                               output_tokens:                tokens(body)[:output_tokens],
+                               created_at:                   recorded_at(body),
+                               inserted_at:                  Time.now.utc
+                             }, operation: 'official_record_writer.response_message')
+                end
+                db[:llm_messages][id: id]
+              rescue Sequel::UniqueConstraintViolation => e
+                log.debug("[ledger] seq collision resolved uuid=#{uuid} conversation_id=#{conversation[:id]} error=#{e.class}")
+                db[:llm_messages].where(uuid: uuid).first ||
+                  db[:llm_messages].where(conversation_id: conversation[:id], seq: seq).first
+              end
             end
 
             def find_or_create_response(db, request, response_message, body)
               response_uuid = stable_uuid(reference(body, :provider_response_ref) || "response:#{request_ref(body)}")
               existing = db[:llm_message_inference_responses].where(uuid: response_uuid).first
-              return existing if existing
+              if existing
+                enrich_response!(db, existing, response_message, body)
+                return existing
+              end
 
               id = insert_row(db, :llm_message_inference_responses, {
                                 uuid:                         response_uuid,
@@ -176,6 +203,34 @@ module Legion
               db[:llm_message_inference_responses][id: id]
             end
 
+            def enrich_response!(db, existing, response_message, body)
+              updates = {}
+              update_if_missing(updates, existing, :response_message_id, response_message&.dig(:id))
+              update_if_missing(updates, existing, :tier, tier(body))
+              update_if_missing(updates, existing, :provider_instance, provider_instance(body))
+              update_if_missing(updates, existing, :finish_reason, finish_reason(body))
+              update_if_missing(updates, existing, :dispatch_path, body[:dispatch_path] || body[:tier])
+
+              response_json = json_dump(visible_response(body))
+              update_if_placeholder(updates, existing, :response_json, response_json)
+
+              thinking_json = json_dump(thinking_response(body))
+              update_if_placeholder(updates, existing, :response_thinking_json, thinking_json)
+
+              return if updates.empty?
+
+              db[:llm_message_inference_responses].where(id: existing[:id]).update(updates)
+              log.info("[ledger] enriched response id=#{existing[:id]} fields=#{updates.keys.join(',')}")
+            end
+
+            def update_if_missing(updates, existing, key, value)
+              updates[key] = value if existing[key].nil? && present?(value)
+            end
+
+            def update_if_placeholder(updates, existing, key, value)
+              updates[key] = value if existing[key].to_s == '{}' && value != '{}'
+            end
+
             def find_or_create_metric(db, request, response, body)
               metric_uuid = stable_uuid(reference(body, :message_id) || "metric:#{request_ref(body)}")
               existing = db[:llm_message_inference_metrics].where(uuid: metric_uuid).first
@@ -222,6 +277,216 @@ module Legion
               db[:llm_messages].where(id: response_message[:id]).update(message_inference_response_id: response[:id])
             end
 
+            def enrich_request!(db, existing, body)
+              updates = {}
+              caller_refs = caller_identity_refs(db, body)
+              updates[:caller_identity_id] = caller_refs[:identity_id] if existing[:caller_identity_id].nil? && caller_refs[:identity_id]
+              updates[:caller_principal_id] = caller_refs[:principal_id] if existing[:caller_principal_id].nil? && caller_refs[:principal_id]
+              updates[:runtime_caller_type] = caller_type(body) if existing[:runtime_caller_type].nil? && caller_type(body)
+
+              request_json = json_dump(request_payload(body))
+              updates[:request_json] = request_json if existing[:request_json].to_s == '{}' && request_json != '{}'
+
+              msg_count = Array(body.dig(:request, :messages) || body[:messages]).size
+              updates[:context_message_count] = msg_count if existing[:context_message_count].to_i.zero? && msg_count.positive?
+
+              return if updates.empty?
+
+              db[:llm_message_inference_requests].where(id: existing[:id]).update(updates)
+              log.info("[ledger] enriched request id=#{existing[:id]} fields=#{updates.keys.join(',')}")
+            end
+
+            def caller_identity(body)
+              caller_identity_refs(::Legion::Data.connection, body)[:identity_id]
+            end
+
+            def caller_principal(body)
+              caller_identity_refs(::Legion::Data.connection, body)[:principal_id]
+            end
+
+            def caller_type(body)
+              raw_type = body[:caller_type] ||
+                         body.dig(:identity, :type) ||
+                         body.dig(:caller, :requested_by, :type) ||
+                         body.dig(:caller, :source)
+              return normalize_caller_type(raw_type) if present?(raw_type)
+
+              parsed_identity_descriptor(body)[:kind]
+            end
+
+            def caller_identity_refs(db, body)
+              body[:__ledger_caller_identity_refs] ||= begin
+                explicit_identity_id = integer_or_nil(body[:caller_identity_id] || body.dig(:caller, :requested_by, :id))
+                explicit_principal_id = integer_or_nil(body[:caller_principal_id] ||
+                                                       body.dig(:caller, :requested_by, :principal_id))
+                refs = { principal_id: explicit_principal_id, identity_id: explicit_identity_id }.compact
+                unless refs[:principal_id] && refs[:identity_id]
+                  if explicit_identity_id && !explicit_principal_id && identity_tables_available?(db)
+                    row = db[:identities].where(id: explicit_identity_id).first
+                    refs[:principal_id] = row[:principal_id] if row
+                  end
+
+                  resolved = resolve_identity(db, body)
+                  refs[:principal_id] ||= resolved[:principal_id]
+                  refs[:identity_id] ||= resolved[:identity_id]
+                end
+                refs.compact
+              end
+            end
+
+            def resolve_identity(db, body)
+              return {} unless identity_tables_available?(db)
+
+              descriptor = parsed_identity_descriptor(body)
+              return {} unless present?(descriptor[:canonical_name])
+
+              provider = find_or_create_identity_provider(db, descriptor[:provider_name])
+              principal = find_or_create_identity_principal(db, descriptor)
+              identity = find_or_create_identity(db, principal, provider, descriptor)
+
+              { principal_id: principal[:id], identity_id: identity[:id] }
+            rescue StandardError => e
+              handle_exception(e, level: :warn, handled: true, operation: 'official_record_writer.identity_resolution')
+              {}
+            end
+
+            def parsed_identity_descriptor(body)
+              raw_identity = body[:caller_identity] ||
+                             body.dig(:identity, :identity) ||
+                             body.dig(:identity, :canonical_name) ||
+                             body.dig(:caller, :requested_by, :identity) ||
+                             body.dig(:caller, :requested_by, :canonical_name) ||
+                             body.dig(:caller, :requested_by, :id)
+              return {} unless present?(raw_identity)
+
+              raw_type = body[:caller_type] ||
+                         body.dig(:identity, :type) ||
+                         body.dig(:caller, :requested_by, :type) ||
+                         body.dig(:caller, :source)
+              provider_name = body.dig(:identity, :credential) ||
+                              body.dig(:caller, :requested_by, :credential) ||
+                              'local'
+              parse_identity_descriptor(raw_identity, raw_type, provider_name)
+            end
+
+            def parse_identity_descriptor(raw_identity, raw_type, provider_name)
+              text = raw_identity.to_s
+              kind = normalize_caller_type(raw_type)
+              canonical = text
+
+              if text.include?(':') && !text.include?('@')
+                prefix, remainder = text.split(':', 2)
+                prefix_kind = normalize_caller_type(prefix)
+                if prefix_kind && present?(remainder)
+                  kind ||= prefix_kind
+                  canonical = remainder
+                end
+              end
+
+              {
+                canonical_name:        canonical,
+                kind:                  kind || 'unknown',
+                provider_identity_key: text,
+                provider_name:         normalize_provider_name(provider_name)
+              }
+            end
+
+            def find_or_create_identity_provider(db, provider_name)
+              table = db[:identity_providers]
+              existing = table.where(name: provider_name).first
+              return existing if existing
+
+              id = insert_row(db, :identity_providers, {
+                                uuid:          deterministic_uuid("identity_provider:#{provider_name}"),
+                                name:          provider_name,
+                                provider_type: provider_name == 'local' ? 'local' : 'external',
+                                facing:        'internal',
+                                source:        'ledger',
+                                created_at:    Time.now.utc,
+                                updated_at:    Time.now.utc
+                              }, operation: 'official_record_writer.identity_provider')
+              table[id: id]
+            rescue Sequel::UniqueConstraintViolation => e
+              handle_exception(e, level: :debug, handled: true, operation: 'official_record_writer.identity_provider_race')
+              table.where(name: provider_name).first
+            end
+
+            def find_or_create_identity_principal(db, descriptor)
+              table = db[:identity_principals]
+              existing = table.where(canonical_name: descriptor[:canonical_name], kind: descriptor[:kind]).first
+              return existing if existing
+
+              id = insert_row(db, :identity_principals, {
+                                uuid:           deterministic_uuid("identity_principal:#{descriptor[:kind]}:#{descriptor[:canonical_name]}"),
+                                canonical_name: descriptor[:canonical_name],
+                                kind:           descriptor[:kind],
+                                display_name:   descriptor[:canonical_name],
+                                last_seen_at:   Time.now.utc,
+                                created_at:     Time.now.utc,
+                                updated_at:     Time.now.utc
+                              }, operation: 'official_record_writer.identity_principal')
+              table[id: id]
+            rescue Sequel::UniqueConstraintViolation => e
+              handle_exception(e, level: :debug, handled: true, operation: 'official_record_writer.identity_principal_race')
+              table.where(canonical_name: descriptor[:canonical_name], kind: descriptor[:kind]).first
+            end
+
+            def find_or_create_identity(db, principal, provider, descriptor)
+              table = db[:identities]
+              existing = table.where(
+                principal_id:          principal[:id],
+                provider_id:           provider[:id],
+                provider_identity_key: descriptor[:provider_identity_key]
+              ).first
+              return existing if existing
+
+              uuid_key = "identity:#{principal[:id]}:#{provider[:id]}:#{descriptor[:provider_identity_key]}"
+              id = insert_row(db, :identities, {
+                                uuid:                  deterministic_uuid(uuid_key),
+                                principal_id:          principal[:id],
+                                provider_id:           provider[:id],
+                                provider_identity_key: descriptor[:provider_identity_key],
+                                last_authenticated_at: Time.now.utc,
+                                account_type:          'primary',
+                                is_default:            true,
+                                created_at:            Time.now.utc,
+                                updated_at:            Time.now.utc
+                              }, operation: 'official_record_writer.identity')
+              table[id: id]
+            rescue Sequel::UniqueConstraintViolation => e
+              handle_exception(e, level: :debug, handled: true, operation: 'official_record_writer.identity_race')
+              table.where(principal_id: principal[:id], provider_id: provider[:id],
+                          provider_identity_key: descriptor[:provider_identity_key]).first
+            end
+
+            def identity_tables_available?(db)
+              db.table_exists?(:identity_providers) &&
+                db.table_exists?(:identity_principals) &&
+                db.table_exists?(:identities)
+            end
+
+            def normalize_caller_type(value)
+              return nil unless present?(value)
+
+              normalized = value.to_s.downcase.gsub(/[^a-z0-9_:-]+/, '_').split(':', 2).first
+              return 'human' if normalized == 'user'
+
+              normalized
+            end
+
+            def normalize_provider_name(value)
+              raw = present?(value) ? value.to_s : 'local'
+              raw.downcase.gsub(/[^a-z0-9_.:-]+/, '-').gsub(/\A-+|-+\z/, '')
+            end
+
+            def integer_or_nil(value)
+              return nil if value.nil?
+              return value if value.is_a?(Integer)
+
+              int = value.to_s.to_i
+              int.positive? ? int : nil
+            end
+
             def correlation_id(body)
               reference(body, :correlation_id, :correlation_ref) || body.dig(:tracing, :correlation_id)
             end
@@ -329,6 +594,11 @@ module Legion
               "#{hex[0, 8]}-#{hex[8, 4]}-#{hex[12, 4]}-#{hex[16, 4]}-#{hex[20, 12]}"
             end
 
+            def deterministic_uuid(value)
+              hex = Digest::SHA256.hexdigest(value.to_s)[0, 32]
+              "#{hex[0, 8]}-#{hex[8, 4]}-#{hex[12, 4]}-#{hex[16, 4]}-#{hex[20, 12]}"
+            end
+
             def next_message_seq(db, conversation)
               db[:llm_messages].where(conversation_id: conversation[:id]).max(:seq).to_i + 1
             end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-llm-ledger
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.3.0
 platform: ruby
 authors:
 - Esity