legion-data 1.10.3 → 1.10.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2489246ac60704ae24678b24c75ab7d2e8f4153d332f3ad1d61f062a4e5c1c9c
-  data.tar.gz: 03e7b2217887a809525bdbf10337af382b1059638ce04c65c3c6bca31f517cc2
+  metadata.gz: 861fe6533a77e8819c03400430f32ddb420a7cfa29e042b3c86734f6d7fd57bc
+  data.tar.gz: 48fe377db280cad64c5e9331926a24319a44c78c96eeb9aa7e2c6aa038a77024
 SHA512:
-  metadata.gz: 5bd7a8abc3a69bcf9ad7c7da89a3937057af7e10516b2070ea60eb0d893b9f09c83570b3535af5949ce0bf3138a7b4e6cd5302eac0bd0b09b259da84e432a9dd
-  data.tar.gz: b934c3058f683f0905a84590f41eb0da21db6ba8c9d221201e5a444c47e486c3cec1dedc14f058ebff7ea8c0637fe67d9bbbae66b108d211529909db3ef4e968
+  metadata.gz: 3850a5774dbc1de772fd8a2b4762dddc9da3164e9cff32b5f02b946f7608dba551394bcb3b2aa90590f038199f33da655957c74896ff74419e840a4769b7008e
+  data.tar.gz: 006b1ed6fd955a876dabfa6875fa075eb8abfec1aa59467b3411e36e3ce32752735277b4013ba908db280cb98a4c152bb91e14563be55c41d6ff53778c3f944a

data/.rubocop.yml

@@ -23,6 +23,13 @@ Naming/VariableNumber:
     - lib/legion/data/connection.rb
 Legion/Framework/EagerSequelModel:
   Enabled: false
+# TaxonomyEnum validates LLM lane taxonomy (:tier/:type/:circuit_state). In
+# these paths `type:` is a Sequel column type (e.g. foreign_key type: :uuid) or
+# the extract API (type: :auto/:text) — not LLM taxonomy — so the cop misfires.
+Legion/Llm/TaxonomyEnum:
+  Exclude:
+    - 'lib/legion/data/migrations/**/*'
+    - 'spec/**/*'
 Metrics/BlockLength:
   Max: 100
   Exclude:

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Legion::Data Changelog
 
+## [1.10.6] - 2026-07-03
+
+### Changed
+- Migration 136: drops the `(conversation_id, seq)` UNIQUE constraint on `llm_messages`, replacing it with a NON-unique composite index. A conversation is a tree, not a line — a single parent can legitimately have two children at the same depth (e.g. a failed branch and the branch that continued), so two distinct messages can share `(conversation_id, seq)` and that is truth, not a duplicate. The unique constraint rejected legitimate siblings and forced the centralized ledger into a racy `MAX(seq)+1` generation that collided under concurrent writers (`PG::UniqueViolation` on `llm_messages_conversation_id_seq_key`). Message identity remains guaranteed by the unique `uuid`; `seq` becomes descriptive tree-depth. On SQLite the constraint drop rebuilds the table (dropping the inline `uuid` UNIQUE), so the migration re-asserts the uuid unique index — a no-op on PostgreSQL where `DROP CONSTRAINT` is surgical.
+
+## [1.10.5] - 2026-06-16
+
+### Added
+- Migration 135: adds context token accounting columns to `llm_message_inference_metrics` — `llm_message_inference_metrics` is now the canonical source of truth for all pipeline context token metrics (request messages, loaded/curated/archived history, thinking strip savings, context-window enforcement savings, RAG injection, system/baseline prompt, tool definitions, final context estimate). Includes `context_accounting_status` and `context_accounting_json` for provenance.
+- Migration 135: creates `llm_context_accounting_events` table for drill-down evidence rows (not a second source of token truth — totals reconcile to the canonical metrics row).
+- Model: `Legion::Data::Models::LLM::ContextAccountingEvent` with foreign key associations to request, response, and metric.
+- Association: `MessageInferenceMetric#context_accounting_events`.
+
+## [1.10.4] - 2026-06-12
+
+### Added
+- Migration 134: adds `operation` (String 64), `dispatch_path` (String 32), and `idempotency_key` (String 128) columns to `llm_route_attempts` with indexes on `operation` and `idempotency_key`. Enables per-attempt tracking of the LLM operation type, routing path, and deduplication key.
+
 ## [1.10.3] - 2026-06-10
 
 ### Changed

data/lib/legion/data/archiver.rb

@@ -97,18 +97,15 @@ module Legion
           total_rows = 0
           paths = []
 
-          loop do
-            batch_result = archive_batch(
-              conn:            conn,
-              table:           table,
-              cutoff:          cutoff,
-              batch_size:      batch_size,
-              batch_n:         batches + 1,
-              now:             now,
-              storage_backend: storage_backend
-            )
-            break unless batch_result
-
+          while (batch_result = archive_batch(
+            conn:            conn,
+            table:           table,
+            cutoff:          cutoff,
+            batch_size:      batch_size,
+            batch_n:         batches + 1,
+            now:             now,
+            storage_backend: storage_backend
+          ))
             batches += 1
             total_rows += batch_result[:row_count]
             paths << batch_result[:path]

data/lib/legion/data/extract/handlers/base.rb

@@ -53,7 +53,7 @@ module Legion
               require gem_name
               true
             rescue LoadError => e
-              handle_exception(e, level: :debug, handled: true, operation: :extract_handler_available, handler: name, gem: gem_name)
+              handle_exception(e, level: :warn, handled: true, operation: :extract_handler_available, handler: name, gem: gem_name)
               false
             end
           end

data/lib/legion/data/migrations/135_add_llm_context_token_accounting.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:llm_message_inference_metrics) do
+      add_column :request_message_estimated_tokens, Integer, null: false, default: 0
+      add_column :loaded_history_estimated_tokens, Integer, null: false, default: 0
+      add_column :curated_history_estimated_tokens, Integer, null: false, default: 0
+      add_column :curation_saved_estimated_tokens, Integer, null: false, default: 0
+      add_column :stripped_thinking_estimated_tokens, Integer, null: false, default: 0
+      add_column :archived_history_estimated_tokens, Integer, null: false, default: 0
+      add_column :archive_saved_estimated_tokens, Integer, null: false, default: 0
+      add_column :context_window_saved_estimated_tokens, Integer, null: false, default: 0
+      add_column :rag_injected_estimated_tokens, Integer, null: false, default: 0
+      add_column :system_prompt_estimated_tokens, Integer, null: false, default: 0
+      add_column :baseline_system_estimated_tokens, Integer, null: false, default: 0
+      add_column :tool_definition_estimated_tokens, Integer, null: false, default: 0
+      add_column :final_context_estimated_tokens, Integer, null: false, default: 0
+      add_column :loaded_history_message_count, Integer, null: false, default: 0
+      add_column :curated_history_message_count, Integer, null: false, default: 0
+      add_column :archived_history_message_count, Integer, null: false, default: 0
+      add_column :stripped_thinking_message_count, Integer, null: false, default: 0
+      add_column :context_window_message_count_before, Integer, null: false, default: 0
+      add_column :context_window_message_count_after, Integer, null: false, default: 0
+      add_column :rag_entry_count, Integer, null: false, default: 0
+      add_column :tool_definition_count, Integer, null: false, default: 0
+      add_column :context_accounting_status, String, size: 64, null: false, default: 'missing'
+      add_column :context_accounting_json, String, text: true
+    end
+
+    create_table(:llm_context_accounting_events) do
+      primary_key :id
+      String :uuid, size: 36, null: false, unique: true
+      foreign_key :message_inference_request_id, :llm_message_inference_requests, null: false, on_delete: :cascade
+      foreign_key :message_inference_response_id, :llm_message_inference_responses, null: true, on_delete: :set_null
+      foreign_key :message_inference_metric_id, :llm_message_inference_metrics, null: true, on_delete: :set_null
+      String :conversation_ref, size: 128
+      String :request_ref, size: 128, null: false
+      String :event_type, size: 64, null: false
+      String :component, size: 64, null: false
+      Integer :estimated_tokens_before, null: false, default: 0
+      Integer :estimated_tokens_after, null: false, default: 0
+      Integer :estimated_tokens_delta, null: false, default: 0
+      Integer :message_count_before, null: false, default: 0
+      Integer :message_count_after, null: false, default: 0
+      String :metadata_json, text: true
+      DateTime :recorded_at
+      DateTime :inserted_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+
+      index :message_inference_request_id
+      index :message_inference_response_id
+      index :message_inference_metric_id
+      index :request_ref
+      index :conversation_ref
+      index %i[event_type component]
+      index :recorded_at
+    end
+  end
+
+  down do
+    drop_table(:llm_context_accounting_events)
+
+    alter_table(:llm_message_inference_metrics) do
+      drop_column :context_accounting_json
+      drop_column :context_accounting_status
+      drop_column :tool_definition_count
+      drop_column :rag_entry_count
+      drop_column :context_window_message_count_after
+      drop_column :context_window_message_count_before
+      drop_column :stripped_thinking_message_count
+      drop_column :archived_history_message_count
+      drop_column :curated_history_message_count
+      drop_column :loaded_history_message_count
+      drop_column :final_context_estimated_tokens
+      drop_column :tool_definition_estimated_tokens
+      drop_column :baseline_system_estimated_tokens
+      drop_column :system_prompt_estimated_tokens
+      drop_column :rag_injected_estimated_tokens
+      drop_column :context_window_saved_estimated_tokens
+      drop_column :archive_saved_estimated_tokens
+      drop_column :archived_history_estimated_tokens
+      drop_column :stripped_thinking_estimated_tokens
+      drop_column :curation_saved_estimated_tokens
+      drop_column :curated_history_estimated_tokens
+      drop_column :loaded_history_estimated_tokens
+      drop_column :request_message_estimated_tokens
+    end
+  end
+end

data/lib/legion/data/migrations/136_drop_llm_messages_seq_unique.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# The (conversation_id, seq) UNIQUE constraint modeled a conversation as a flat
+# line, but a conversation is a tree: a single parent can legitimately have two
+# children at the same depth (e.g. a failed branch and the branch that
+# succeeded), so two distinct messages can share the same (conversation_id, seq)
+# and that is TRUTH, not a duplicate. The unique constraint rejected those
+# legitimate siblings and forced the ledger into a racy MAX(seq)+1 generation
+# that collided under concurrent writers.
+#
+# Message identity is guaranteed by the unique `uuid` (producer-derived, stable
+# across redelivery). seq/parent become descriptive ordering data, so we keep a
+# NON-unique composite index for conversation-ordered reads.
+Sequel.migration do
+  up do
+    alter_table(:llm_messages) do
+      drop_constraint(:llm_messages_conversation_id_seq_key, type: :unique)
+      add_index %i[conversation_id seq], name: :idx_llm_messages_conversation_seq
+    end
+    # SQLite emulates the constraint drop by rebuilding the table, which drops
+    # the inline `uuid` UNIQUE. Re-assert it so message identity/dedup is never
+    # lost. On PostgreSQL DROP CONSTRAINT is surgical and the original uuid index
+    # already exists, so this is a no-op there.
+    alter_table(:llm_messages) do
+      add_index :uuid, unique: true, name: :idx_llm_messages_uuid_unique, if_not_exists: true
+    end
+  end
+
+  down do
+    alter_table(:llm_messages) do
+      drop_index :uuid, name: :idx_llm_messages_uuid_unique, if_exists: true
+      drop_index %i[conversation_id seq], name: :idx_llm_messages_conversation_seq
+      add_unique_constraint %i[conversation_id seq], name: :llm_messages_conversation_id_seq_key
+    end
+  end
+end

data/lib/legion/data/model.rb

@@ -21,7 +21,8 @@ module Legion
              rbac/role_assignments rbac/runner_grants rbac/cross_team_grants
              llm/conversation llm/message llm/message_inference_request
              llm/message_inference_response llm/route_attempt
-             llm/message_inference_metric llm/tool_call llm/tool_call_attempt
+             llm/message_inference_metric llm/context_accounting_event
+             llm/tool_call llm/tool_call_attempt
              llm/conversation_compaction llm/policy_evaluation
              llm/security_event llm/registry_event]
         end

data/lib/legion/data/models/function.rb

@@ -18,7 +18,7 @@ module Legion
 
           ::JSON.parse(embedding)
         rescue ::JSON::ParserError => e
-          handle_exception(e, level: :debug, handled: true, operation: :embedding_vector, id: self[:id])
+          handle_exception(e, level: :warn, handled: true, operation: :embedding_vector, id: self[:id])
           nil
         end
 

data/lib/legion/data/models/llm/context_accounting_event.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative 'model_helpers'
+
+module Legion
+  module Data
+    module Models
+      module LLM
+        class ContextAccountingEvent < Sequel::Model(:llm_context_accounting_events)
+          include ModelHelpers
+
+          many_to_one :message_inference_request
+          many_to_one :message_inference_response
+          many_to_one :message_inference_metric
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/models/llm/message_inference_metric.rb

@@ -11,6 +11,7 @@ module Legion
 
           many_to_one :message_inference_request
           many_to_one :message_inference_response
+          one_to_many :context_accounting_events
 
           class << self
             def finance_usage_by_cost_center_model_day(cost_center: nil, model_key: nil, from: nil, to: nil)

data/lib/legion/data/models/node.rb

@@ -17,7 +17,7 @@ module Legion
 
           Legion::JSON.load(metrics)
         rescue StandardError => e
-          handle_exception(e, level: :debug, handled: true, operation: :parsed_metrics, id: self[:id])
+          handle_exception(e, level: :warn, handled: true, operation: :parsed_metrics, id: self[:id])
           nil
         end
 
@@ -26,7 +26,7 @@ module Legion
 
           Legion::JSON.load(hosted_worker_ids)
         rescue StandardError => e
-          handle_exception(e, level: :debug, handled: true, operation: :parsed_hosted_worker_ids, id: self[:id])
+          handle_exception(e, level: :warn, handled: true, operation: :parsed_hosted_worker_ids, id: self[:id])
           []
         end
       end

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.10.3'
+    VERSION = '1.10.6'
   end
 end

data/lib/legion/data.rb

@@ -112,7 +112,7 @@ module Legion
       def connected?
         Legion::Settings[:data][:connected] == true
       rescue StandardError => e
-        handle_exception(e, level: :debug, handled: true, operation: :connected?)
+        handle_exception(e, level: :warn, handled: true, operation: :connected?)
         false
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.10.3
+  version: 1.10.6
 platform: ruby
 authors:
 - Esity
@@ -281,6 +281,8 @@ files:
 - lib/legion/data/migrations/132_drop_schema_version_from_llm_tool_calls.rb
 - lib/legion/data/migrations/133_allow_null_context_tokens.rb
 - lib/legion/data/migrations/134_add_route_attempt_columns.rb
+- lib/legion/data/migrations/135_add_llm_context_token_accounting.rb
+- lib/legion/data/migrations/136_drop_llm_messages_seq_unique.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo/access_log.rb
 - lib/legion/data/models/apollo/entries.rb
@@ -306,6 +308,7 @@ files:
 - lib/legion/data/models/identity/model_helpers.rb
 - lib/legion/data/models/identity/principal.rb
 - lib/legion/data/models/identity/providers.rb
+- lib/legion/data/models/llm/context_accounting_event.rb
 - lib/legion/data/models/llm/conversation.rb
 - lib/legion/data/models/llm/conversation_compaction.rb
 - lib/legion/data/models/llm/message.rb