legion-data 1.10.2 → 1.10.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 212e01a164b58ab2a8a4273497e98acf0d99f174f4a66a77c7c707899685f2ed
-  data.tar.gz: e3302347dd4f8d3ce634fc88110a645ad6986aa5d8b531d5a9187fe894872ae0
+  metadata.gz: 7f3b320363f209f8dfae3ad1a9757bae5e17dd4d7e6327b849b0d8804df22c07
+  data.tar.gz: ac12848278ac407233eb43b6f5a72dbf503b030a9d404e88578fdb8ebd173cc5
 SHA512:
-  metadata.gz: 6399d19fe35c9b3e975522d6d2b4654d0bc56240e508a38bcf55c4cb064fbb1214f7e2c5de7f3c4828a79f8543044da476811808d574b020609ab20adcc8d778
-  data.tar.gz: 87e1b1e58c66e464cbb1ccf41ad2a1917531edff611dcef0fc0f6b20f7dfe9331f854e684c00ace11557c714cd77e6131cd2a6b07c837f60d670c0d9fdf9b704
+  metadata.gz: 0e0331f06f446400f70581981a8f083e1baac336aee984904f56af60eb9a9d021d56af48c8b845a13beaebd4aca1ba0e7a264af1101f78426d447a4cfd3d86b2
+  data.tar.gz: 70b599bee80873cd661bdd09aacb406da9d8cdd479d270b6012032945aedb08bb82a3819f00f11f44e45b3470b8b88042491e5e892900564d334d6b4748f945a

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Legion::Data Changelog
 
+## [1.10.5] - 2026-06-16
+
+### Added
+- Migration 135: adds context token accounting columns to `llm_message_inference_metrics` — `llm_message_inference_metrics` is now the canonical source of truth for all pipeline context token metrics (request messages, loaded/curated/archived history, thinking strip savings, context-window enforcement savings, RAG injection, system/baseline prompt, tool definitions, final context estimate). Includes `context_accounting_status` and `context_accounting_json` for provenance.
+- Migration 135: creates `llm_context_accounting_events` table for drill-down evidence rows (not a second source of token truth — totals reconcile to the canonical metrics row).
+- Model: `Legion::Data::Models::LLM::ContextAccountingEvent` with foreign key associations to request, response, and metric.
+- Association: `MessageInferenceMetric#context_accounting_events`.
+
+## [1.10.4] - 2026-06-12
+
+### Added
+- Migration 134: adds `operation` (String 64), `dispatch_path` (String 32), and `idempotency_key` (String 128) columns to `llm_route_attempts` with indexes on `operation` and `idempotency_key`. Enables per-attempt tracking of the LLM operation type, routing path, and deduplication key.
+
+## [1.10.3] - 2026-06-10
+
+### Changed
+- `connection_validation` now defaults to `false` — the Sequel connection_validator extension issues a `SELECT NULL` on every pool checkout/checkin and before real queries, which measurably degrades throughput; query-time error handling already recovers stale/dead connections. Set `connection_validation: true` in settings to opt back in (timeout semantics unchanged: `-1` validates every checkout)
+- Pool timeout assignments read `connection_validation_timeout` / `connection_expiration_timeout` directly from settings — removed inline `|| 600` / `|| 14_400` shadow defaults that disagreed with the documented settings defaults
+
 ## [1.10.2] - 2026-06-02
 
 ### Fixed

data/lib/legion/data/connection.rb

@@ -581,12 +581,12 @@ module Legion
 
           if data[:connection_validation] != false
             @sequel.extension(:connection_validator)
-            @sequel.pool.connection_validation_timeout = data[:connection_validation_timeout] || 600
+            @sequel.pool.connection_validation_timeout = data[:connection_validation_timeout]
           end
 
           if data[:connection_expiration] != false
             @sequel.extension(:connection_expiration)
-            @sequel.pool.connection_expiration_timeout = data[:connection_expiration_timeout] || 14_400
+            @sequel.pool.connection_expiration_timeout = data[:connection_expiration_timeout]
           end
         rescue StandardError => e
           handle_exception(e, level: :warn, handled: true, operation: :configure_extensions, adapter: adapter)

data/lib/legion/data/migrations/134_add_route_attempt_columns.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:llm_route_attempts) do
+      add_column :operation, String, size: 64, null: true
+      add_column :dispatch_path, String, size: 32, null: true
+      add_column :idempotency_key, String, size: 128, null: true
+      add_index :operation, name: :idx_route_attempts_operation
+      add_index :idempotency_key, name: :idx_route_attempts_idempotency_key
+    end
+  end
+
+  down do
+    alter_table(:llm_route_attempts) do
+      drop_index :operation, name: :idx_route_attempts_operation
+      drop_index :idempotency_key, name: :idx_route_attempts_idempotency_key
+      drop_column :operation
+      drop_column :dispatch_path
+      drop_column :idempotency_key
+    end
+  end
+end

data/lib/legion/data/migrations/135_add_llm_context_token_accounting.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:llm_message_inference_metrics) do
+      add_column :request_message_estimated_tokens, Integer, null: false, default: 0
+      add_column :loaded_history_estimated_tokens, Integer, null: false, default: 0
+      add_column :curated_history_estimated_tokens, Integer, null: false, default: 0
+      add_column :curation_saved_estimated_tokens, Integer, null: false, default: 0
+      add_column :stripped_thinking_estimated_tokens, Integer, null: false, default: 0
+      add_column :archived_history_estimated_tokens, Integer, null: false, default: 0
+      add_column :archive_saved_estimated_tokens, Integer, null: false, default: 0
+      add_column :context_window_saved_estimated_tokens, Integer, null: false, default: 0
+      add_column :rag_injected_estimated_tokens, Integer, null: false, default: 0
+      add_column :system_prompt_estimated_tokens, Integer, null: false, default: 0
+      add_column :baseline_system_estimated_tokens, Integer, null: false, default: 0
+      add_column :tool_definition_estimated_tokens, Integer, null: false, default: 0
+      add_column :final_context_estimated_tokens, Integer, null: false, default: 0
+      add_column :loaded_history_message_count, Integer, null: false, default: 0
+      add_column :curated_history_message_count, Integer, null: false, default: 0
+      add_column :archived_history_message_count, Integer, null: false, default: 0
+      add_column :stripped_thinking_message_count, Integer, null: false, default: 0
+      add_column :context_window_message_count_before, Integer, null: false, default: 0
+      add_column :context_window_message_count_after, Integer, null: false, default: 0
+      add_column :rag_entry_count, Integer, null: false, default: 0
+      add_column :tool_definition_count, Integer, null: false, default: 0
+      add_column :context_accounting_status, String, size: 64, null: false, default: 'missing'
+      add_column :context_accounting_json, String, text: true
+    end
+
+    create_table(:llm_context_accounting_events) do
+      primary_key :id
+      String :uuid, size: 36, null: false, unique: true
+      foreign_key :message_inference_request_id, :llm_message_inference_requests, null: false, on_delete: :cascade
+      foreign_key :message_inference_response_id, :llm_message_inference_responses, null: true, on_delete: :set_null
+      foreign_key :message_inference_metric_id, :llm_message_inference_metrics, null: true, on_delete: :set_null
+      String :conversation_ref, size: 128
+      String :request_ref, size: 128, null: false
+      String :event_type, size: 64, null: false
+      String :component, size: 64, null: false
+      Integer :estimated_tokens_before, null: false, default: 0
+      Integer :estimated_tokens_after, null: false, default: 0
+      Integer :estimated_tokens_delta, null: false, default: 0
+      Integer :message_count_before, null: false, default: 0
+      Integer :message_count_after, null: false, default: 0
+      String :metadata_json, text: true
+      DateTime :recorded_at
+      DateTime :inserted_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+
+      index :message_inference_request_id
+      index :message_inference_response_id
+      index :message_inference_metric_id
+      index :request_ref
+      index :conversation_ref
+      index %i[event_type component]
+      index :recorded_at
+    end
+  end
+
+  down do
+    drop_table(:llm_context_accounting_events)
+
+    alter_table(:llm_message_inference_metrics) do
+      drop_column :context_accounting_json
+      drop_column :context_accounting_status
+      drop_column :tool_definition_count
+      drop_column :rag_entry_count
+      drop_column :context_window_message_count_after
+      drop_column :context_window_message_count_before
+      drop_column :stripped_thinking_message_count
+      drop_column :archived_history_message_count
+      drop_column :curated_history_message_count
+      drop_column :loaded_history_message_count
+      drop_column :final_context_estimated_tokens
+      drop_column :tool_definition_estimated_tokens
+      drop_column :baseline_system_estimated_tokens
+      drop_column :system_prompt_estimated_tokens
+      drop_column :rag_injected_estimated_tokens
+      drop_column :context_window_saved_estimated_tokens
+      drop_column :archive_saved_estimated_tokens
+      drop_column :archived_history_estimated_tokens
+      drop_column :stripped_thinking_estimated_tokens
+      drop_column :curation_saved_estimated_tokens
+      drop_column :curated_history_estimated_tokens
+      drop_column :loaded_history_estimated_tokens
+      drop_column :request_message_estimated_tokens
+    end
+  end
+end

data/lib/legion/data/model.rb

@@ -21,7 +21,8 @@ module Legion
              rbac/role_assignments rbac/runner_grants rbac/cross_team_grants
              llm/conversation llm/message llm/message_inference_request
              llm/message_inference_response llm/route_attempt
-             llm/message_inference_metric llm/tool_call llm/tool_call_attempt
+             llm/message_inference_metric llm/context_accounting_event
+             llm/tool_call llm/tool_call_attempt
              llm/conversation_compaction llm/policy_evaluation
              llm/security_event llm/registry_event]
         end

data/lib/legion/data/models/llm/context_accounting_event.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative 'model_helpers'
+
+module Legion
+  module Data
+    module Models
+      module LLM
+        class ContextAccountingEvent < Sequel::Model(:llm_context_accounting_events)
+          include ModelHelpers
+
+          many_to_one :message_inference_request
+          many_to_one :message_inference_response
+          many_to_one :message_inference_metric
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/models/llm/message_inference_metric.rb

@@ -11,6 +11,7 @@ module Legion
 
           many_to_one :message_inference_request
           many_to_one :message_inference_response
+          one_to_many :context_accounting_events
 
           class << self
             def finance_usage_by_cost_center_model_day(cost_center: nil, model_key: nil, from: nil, to: nil)

data/lib/legion/data/settings.rb

@@ -48,8 +48,12 @@ module Legion
           sql_log_level:                 'debug',
 
           # Connection health (network adapters only, ignored for sqlite)
-          # -1 means validate on every checkout, catching stale connections from VPN/sleep/network changes immediately
-          connection_validation:         true,
+          # Validation is disabled by default: the connection_validator extension issues a
+          # SELECT NULL on every checkout/checkin and before real queries, which kills
+          # throughput. Connection errors are already rescued and reconnected at query time.
+          # When enabled, connection_validation_timeout: -1 validates on every checkout
+          # (catches stale connections from VPN/sleep/network changes immediately).
+          connection_validation:         false,
           connection_validation_timeout: -1,
           connection_expiration:         true,
           connection_expiration_timeout: 14_400,

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.10.2'
+    VERSION = '1.10.5'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.10.2
+  version: 1.10.5
 platform: ruby
 authors:
 - Esity
@@ -280,6 +280,8 @@ files:
 - lib/legion/data/migrations/131_add_llm_tool_calls_schema_version.rb
 - lib/legion/data/migrations/132_drop_schema_version_from_llm_tool_calls.rb
 - lib/legion/data/migrations/133_allow_null_context_tokens.rb
+- lib/legion/data/migrations/134_add_route_attempt_columns.rb
+- lib/legion/data/migrations/135_add_llm_context_token_accounting.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo/access_log.rb
 - lib/legion/data/models/apollo/entries.rb
@@ -305,6 +307,7 @@ files:
 - lib/legion/data/models/identity/model_helpers.rb
 - lib/legion/data/models/identity/principal.rb
 - lib/legion/data/models/identity/providers.rb
+- lib/legion/data/models/llm/context_accounting_event.rb
 - lib/legion/data/models/llm/conversation.rb
 - lib/legion/data/models/llm/conversation_compaction.rb
 - lib/legion/data/models/llm/message.rb