legion-data 1.10.3 → 1.10.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2489246ac60704ae24678b24c75ab7d2e8f4153d332f3ad1d61f062a4e5c1c9c
-  data.tar.gz: 03e7b2217887a809525bdbf10337af382b1059638ce04c65c3c6bca31f517cc2
+  metadata.gz: 7f3b320363f209f8dfae3ad1a9757bae5e17dd4d7e6327b849b0d8804df22c07
+  data.tar.gz: ac12848278ac407233eb43b6f5a72dbf503b030a9d404e88578fdb8ebd173cc5
 SHA512:
-  metadata.gz: 5bd7a8abc3a69bcf9ad7c7da89a3937057af7e10516b2070ea60eb0d893b9f09c83570b3535af5949ce0bf3138a7b4e6cd5302eac0bd0b09b259da84e432a9dd
-  data.tar.gz: b934c3058f683f0905a84590f41eb0da21db6ba8c9d221201e5a444c47e486c3cec1dedc14f058ebff7ea8c0637fe67d9bbbae66b108d211529909db3ef4e968
+  metadata.gz: 0e0331f06f446400f70581981a8f083e1baac336aee984904f56af60eb9a9d021d56af48c8b845a13beaebd4aca1ba0e7a264af1101f78426d447a4cfd3d86b2
+  data.tar.gz: 70b599bee80873cd661bdd09aacb406da9d8cdd479d270b6012032945aedb08bb82a3819f00f11f44e45b3470b8b88042491e5e892900564d334d6b4748f945a

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Legion::Data Changelog
 
+## [1.10.5] - 2026-06-16
+
+### Added
+- Migration 135: adds context token accounting columns to `llm_message_inference_metrics` — `llm_message_inference_metrics` is now the canonical source of truth for all pipeline context token metrics (request messages, loaded/curated/archived history, thinking strip savings, context-window enforcement savings, RAG injection, system/baseline prompt, tool definitions, final context estimate). Includes `context_accounting_status` and `context_accounting_json` for provenance.
+- Migration 135: creates `llm_context_accounting_events` table for drill-down evidence rows (not a second source of token truth — totals reconcile to the canonical metrics row).
+- Model: `Legion::Data::Models::LLM::ContextAccountingEvent` with foreign key associations to request, response, and metric.
+- Association: `MessageInferenceMetric#context_accounting_events`.
+
+## [1.10.4] - 2026-06-12
+
+### Added
+- Migration 134: adds `operation` (String 64), `dispatch_path` (String 32), and `idempotency_key` (String 128) columns to `llm_route_attempts` with indexes on `operation` and `idempotency_key`. Enables per-attempt tracking of the LLM operation type, routing path, and deduplication key.
+
 ## [1.10.3] - 2026-06-10
 
 ### Changed

data/lib/legion/data/migrations/135_add_llm_context_token_accounting.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:llm_message_inference_metrics) do
+      add_column :request_message_estimated_tokens, Integer, null: false, default: 0
+      add_column :loaded_history_estimated_tokens, Integer, null: false, default: 0
+      add_column :curated_history_estimated_tokens, Integer, null: false, default: 0
+      add_column :curation_saved_estimated_tokens, Integer, null: false, default: 0
+      add_column :stripped_thinking_estimated_tokens, Integer, null: false, default: 0
+      add_column :archived_history_estimated_tokens, Integer, null: false, default: 0
+      add_column :archive_saved_estimated_tokens, Integer, null: false, default: 0
+      add_column :context_window_saved_estimated_tokens, Integer, null: false, default: 0
+      add_column :rag_injected_estimated_tokens, Integer, null: false, default: 0
+      add_column :system_prompt_estimated_tokens, Integer, null: false, default: 0
+      add_column :baseline_system_estimated_tokens, Integer, null: false, default: 0
+      add_column :tool_definition_estimated_tokens, Integer, null: false, default: 0
+      add_column :final_context_estimated_tokens, Integer, null: false, default: 0
+      add_column :loaded_history_message_count, Integer, null: false, default: 0
+      add_column :curated_history_message_count, Integer, null: false, default: 0
+      add_column :archived_history_message_count, Integer, null: false, default: 0
+      add_column :stripped_thinking_message_count, Integer, null: false, default: 0
+      add_column :context_window_message_count_before, Integer, null: false, default: 0
+      add_column :context_window_message_count_after, Integer, null: false, default: 0
+      add_column :rag_entry_count, Integer, null: false, default: 0
+      add_column :tool_definition_count, Integer, null: false, default: 0
+      add_column :context_accounting_status, String, size: 64, null: false, default: 'missing'
+      add_column :context_accounting_json, String, text: true
+    end
+
+    create_table(:llm_context_accounting_events) do
+      primary_key :id
+      String :uuid, size: 36, null: false, unique: true
+      foreign_key :message_inference_request_id, :llm_message_inference_requests, null: false, on_delete: :cascade
+      foreign_key :message_inference_response_id, :llm_message_inference_responses, null: true, on_delete: :set_null
+      foreign_key :message_inference_metric_id, :llm_message_inference_metrics, null: true, on_delete: :set_null
+      String :conversation_ref, size: 128
+      String :request_ref, size: 128, null: false
+      String :event_type, size: 64, null: false
+      String :component, size: 64, null: false
+      Integer :estimated_tokens_before, null: false, default: 0
+      Integer :estimated_tokens_after, null: false, default: 0
+      Integer :estimated_tokens_delta, null: false, default: 0
+      Integer :message_count_before, null: false, default: 0
+      Integer :message_count_after, null: false, default: 0
+      String :metadata_json, text: true
+      DateTime :recorded_at
+      DateTime :inserted_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+
+      index :message_inference_request_id
+      index :message_inference_response_id
+      index :message_inference_metric_id
+      index :request_ref
+      index :conversation_ref
+      index %i[event_type component]
+      index :recorded_at
+    end
+  end
+
+  down do
+    drop_table(:llm_context_accounting_events)
+
+    alter_table(:llm_message_inference_metrics) do
+      drop_column :context_accounting_json
+      drop_column :context_accounting_status
+      drop_column :tool_definition_count
+      drop_column :rag_entry_count
+      drop_column :context_window_message_count_after
+      drop_column :context_window_message_count_before
+      drop_column :stripped_thinking_message_count
+      drop_column :archived_history_message_count
+      drop_column :curated_history_message_count
+      drop_column :loaded_history_message_count
+      drop_column :final_context_estimated_tokens
+      drop_column :tool_definition_estimated_tokens
+      drop_column :baseline_system_estimated_tokens
+      drop_column :system_prompt_estimated_tokens
+      drop_column :rag_injected_estimated_tokens
+      drop_column :context_window_saved_estimated_tokens
+      drop_column :archive_saved_estimated_tokens
+      drop_column :archived_history_estimated_tokens
+      drop_column :stripped_thinking_estimated_tokens
+      drop_column :curation_saved_estimated_tokens
+      drop_column :curated_history_estimated_tokens
+      drop_column :loaded_history_estimated_tokens
+      drop_column :request_message_estimated_tokens
+    end
+  end
+end

data/lib/legion/data/model.rb

@@ -21,7 +21,8 @@ module Legion
              rbac/role_assignments rbac/runner_grants rbac/cross_team_grants
              llm/conversation llm/message llm/message_inference_request
              llm/message_inference_response llm/route_attempt
-             llm/message_inference_metric llm/tool_call llm/tool_call_attempt
+             llm/message_inference_metric llm/context_accounting_event
+             llm/tool_call llm/tool_call_attempt
              llm/conversation_compaction llm/policy_evaluation
              llm/security_event llm/registry_event]
         end

data/lib/legion/data/models/llm/context_accounting_event.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative 'model_helpers'
+
+module Legion
+  module Data
+    module Models
+      module LLM
+        class ContextAccountingEvent < Sequel::Model(:llm_context_accounting_events)
+          include ModelHelpers
+
+          many_to_one :message_inference_request
+          many_to_one :message_inference_response
+          many_to_one :message_inference_metric
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/models/llm/message_inference_metric.rb

@@ -11,6 +11,7 @@ module Legion
 
           many_to_one :message_inference_request
           many_to_one :message_inference_response
+          one_to_many :context_accounting_events
 
           class << self
             def finance_usage_by_cost_center_model_day(cost_center: nil, model_key: nil, from: nil, to: nil)

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.10.3'
+    VERSION = '1.10.5'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.10.3
+  version: 1.10.5
 platform: ruby
 authors:
 - Esity
@@ -281,6 +281,7 @@ files:
 - lib/legion/data/migrations/132_drop_schema_version_from_llm_tool_calls.rb
 - lib/legion/data/migrations/133_allow_null_context_tokens.rb
 - lib/legion/data/migrations/134_add_route_attempt_columns.rb
+- lib/legion/data/migrations/135_add_llm_context_token_accounting.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo/access_log.rb
 - lib/legion/data/models/apollo/entries.rb
@@ -306,6 +307,7 @@ files:
 - lib/legion/data/models/identity/model_helpers.rb
 - lib/legion/data/models/identity/principal.rb
 - lib/legion/data/models/identity/providers.rb
+- lib/legion/data/models/llm/context_accounting_event.rb
 - lib/legion/data/models/llm/conversation.rb
 - lib/legion/data/models/llm/conversation_compaction.rb
 - lib/legion/data/models/llm/message.rb