legion-data 1.8.8 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60b8f26149494282c49455aedc216eebf0db867fa359dea9404ca44d09ad9815
-  data.tar.gz: 4da8dfdb277d9ceda6807543f58c2804ae71f938056cff7f8bcd366c4bad9cae
+  metadata.gz: 92fba520187d3940bae5ac79e43f2155b16a0c7e3949e664233dd012871d56e7
+  data.tar.gz: da5ff3d58c176ccb65fc03bdc6855318acba28b4974cf76e8b986eec2b70d5c3
 SHA512:
-  metadata.gz: ef647d212100c659d28847879153403ac1dd42c55c0cb9b1529fd1e17d73f029a912d23988f6701e59e4b25d326b8512cd89f0291bcc52a9e492e6fd4c85b055
-  data.tar.gz: '018844cceb163d4100cffcfa8ea1ed06b916889ca14111e9ca8398d9c3d2a045c9e9c4d73f735e8c22f681ccc834c23f732d08d68303973251c60085f5013c1b'
+  metadata.gz: 748443ee5df5494f60e080a6ad6405df97ed4166ed71e58fc58159b540ab2c9748195447c2a05e697a5ecc00889b17d7ff84337d0b718644c27c1312a6c47ab5
+  data.tar.gz: 9417467c46a3a4b1651c68760ff682b07648cc021d51601bc844fb1afdccf68ae0a807f897823a818450658e8503feea7e409ecaacd2b80c757dd0847a037b22

data/.rubocop.yml

@@ -1,31 +1,56 @@
 inherit_gem:
   rubocop-legion: config/core.yml
-
-Metrics/ParameterLists:
-  Max: 8
-
-Metrics/BlockLength:
-  Max: 40
+AllCops:
+  TargetRubyVersion: 3.4
+  NewCops: enable
+  SuggestExtensions: false
+Layout/LineLength:
+  Max: 160
+Layout/SpaceAroundEqualsInParameterDefault:
+  EnforcedStyle: space
+Layout/HashAlignment:
+  EnforcedHashRocketStyle: table
+  EnforcedColonStyle: table
+Metrics/MethodLength:
+  Max: 50
+Metrics/ClassLength:
+  Max: 1500
+Metrics/ModuleLength:
+  Max: 1500
+Naming/VariableNumber:
   Exclude:
     - 'spec/**/*'
-    - 'lib/legion/data/migrations/**/*'
-
-Naming/VariableNumber:
+    - lib/legion/data/connection.rb
+Legion/Framework/EagerSequelModel:
   Enabled: false
-
-Style/FileOpen:
+Metrics/BlockLength:
+  Max: 100
   Exclude:
-    - 'lib/legion/data/connection.rb'
-
-# Pre-existing patterns — suppress until addressed in a dedicated cleanup PR
+    - 'spec/**/*'
+    - 'lib/legion/data/migrations/*'
 ThreadSafety/ClassInstanceVariable:
   Enabled: false
-
 ThreadSafety/ClassAndModuleAttributes:
   Enabled: false
+Metrics/AbcSize:
+  Max: 60
+  Exclude:
+    - 'spec/**/*'
+Metrics/CyclomaticComplexity:
+  Max: 15
+  Exclude:
+    - 'spec/**/*'
+Metrics/PerceivedComplexity:
+  Max: 17
+  Exclude:
+    - 'spec/**/*'
 
-Legion/RescueLogging/NoCapture:
+Style/Documentation:
   Enabled: false
-
-Legion/Framework/EagerSequelModel:
+Style/SymbolArray:
+  Enabled: true
+Style/FrozenStringLiteralComment:
+  Enabled: true
+  EnforcedStyle: always
+Naming/PredicateMethod:
   Enabled: false

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Legion::Data Changelog
 
+## [1.9.0] - 2026-06-01
+
+### Added
+
+- Migration 123: audit columns on `llm_tool_calls` — `tool_arguments_json` (TEXT), `tool_result_json` (TEXT), `tool_category` (String(64)), `data_handling_classification` (String(32)), `policy_decision` (String(32)), `requires_human_approval` (Boolean) plus indexes on `tool_category`, `data_handling_classification`, `policy_decision`.
+- Migration 124: audit columns on `llm_tool_call_attempts` — `attempt_input_json` (TEXT), `attempt_output_json` (TEXT), `error_details_json` (TEXT).
+- Migration 125: audit columns on `llm_escalation_events` — `history_json` (TEXT), `outcome` (String(32)), `total_attempts` (Integer) plus index on `outcome`.
+- Migration 126: audit columns on `llm_message_inference_responses` — `route_attempts` (Integer, default 0), `escalation_chain_ref` (String(128)) plus index on `escalation_chain_ref`. Skips `response_content_hash` (already exists since migration 080).
+- Migration 127: audit columns on `llm_message_inference_requests` — `parent_request_id` (Integer, self-referencing FK on_delete: :set_null). Skips `request_content_hash`, `curation_strategy`, `tool_policy` (all already exist since migration 079).
+
+## [1.8.9] - 2026-05-26
+
+### Changed
+- Migration 116: make `llm_tool_calls.message_inference_response_id` nullable and drop composite unique index on `[message_inference_response_id, tool_call_index]`. Eliminates 30-40% dead-letter rate on tool audit messages caused by AMQP race between response and tool call writers.
+- Migration 117: add nullable `conversation_id` FK to `llm_tool_calls` referencing `llm_conversations`, so tool call rows can track their conversation even when the response row hasn't been written yet.
+- Add `many_to_one :conversation` association to `LLM::ToolCall` model.
+
 ## [1.8.8] - 2026-05-20
 
 ### Added

data/CLAUDE.md

@@ -32,11 +32,116 @@ jq '[.examples[] | select(.status != "passed") | {file_path, line_number, full_d
 
 - Never edit published migrations. Add a new migration.
 - Do not guard migrations with `create_table?`, `drop_table?`, `table_exists?`, `if_exists`, `if_not_exists`, `next if`, or `next unless`.
-- Keep migrations small enough to diagnose and roll back. Split by domain and dependency.
+- **One change per migration file.** Each migration modifies exactly ONE table. Never loop over tables. If a migration fails, you must be able to identify exactly what broke and roll back cleanly.
+- Never use `.each`, `.map`, or any iterator in a migration. If 12 tables need the same column, that's 12 migration files.
+- Never use raw SQL (`run '...'`) when Sequel DSL supports the operation. Use `add_index`, `drop_index`, `add_column`, `drop_column`, etc.
 - Use portable Sequel DSL unless the feature truly requires adapter-specific behavior.
 - Use integer `id` primary keys for joins and public `uuid` columns for APIs/logs/external references.
 - Normalize stable fields. Use JSON only for genuinely dynamic provider payloads or evidence.
 
+### Sequel Migration DSL Reference
+
+**Create table**: https://sequel.jeremyevans.net/rdoc/classes/Sequel/Database.html#method-i-create_table
+**Column options**: https://sequel.jeremyevans.net/rdoc/classes/Sequel/Schema/CreateTableGenerator.html#method-i-column
+
+### Create Table Pattern
+
+```ruby
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:example_records) do
+      primary_key :id
+      String :uuid, size: 36, null: false, unique: true
+
+      # Identity columns (required on every table)
+      String :access_scope, size: 20, null: false, default: 'global', index: true
+      foreign_key :identity_principal_id, :identity_principals, null: true, on_delete: :set_null, on_update: :cascade
+      foreign_key :identity_id, :identities, null: true, on_delete: :set_null, on_update: :cascade
+      String :identity_canonical_name, size: 255, null: true, index: true
+
+      # Domain columns here...
+
+      # Timestamps (required on every table)
+      DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP  # reflects when the event happened (request/AMQP timestamp)
+      DateTime :inserted_at, null: false, default: Sequel::CURRENT_TIMESTAMP # when the row was physically written to the database
+      DateTime :updated_at, null: true                                        # set on row update; NULL means never updated
+
+      index :identity_principal_id
+    end
+  end
+end
+```
+
+### Alter Table Pattern (adding a column)
+
+```ruby
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:target_table) do
+      add_column :new_column, String, size: 128, null: true, index: true
+    end
+  end
+
+  down do
+    alter_table(:target_table) do
+      drop_index :new_column
+      drop_column :new_column
+    end
+  end
+end
+```
+
+### Column Option Reference
+
+| Option | Purpose |
+|--------|---------|
+| `:null` | `false` = NOT NULL, `true` = nullable |
+| `:default` | Default value (use `Sequel::CURRENT_TIMESTAMP` for timestamps) |
+| `:index` | `true` creates an index on this column; pass a Hash for index options |
+| `:unique` | `true` adds a UNIQUE constraint |
+| `:on_delete` | FK behavior: `:cascade`, `:set_null`, `:restrict`, `:no_action` |
+| `:on_update` | FK behavior: `:cascade`, `:set_null`, `:restrict`, `:no_action` |
+| `:key` | For FKs — the referenced column (unnecessary if referencing primary key) |
+| `:size` | Column width for String/Decimal |
+| `:text` | `true` for TEXT columns (unlimited length) |
+
+### Foreign Key Conventions
+
+```ruby
+# FK to identity tables — always cascade updates, set null on delete
+foreign_key :identity_principal_id, :identity_principals, null: true, on_delete: :set_null, on_update: :cascade
+foreign_key :identity_id, :identities, null: true, on_delete: :set_null, on_update: :cascade
+
+# FK to domain tables — cascade delete (child dies with parent)
+foreign_key :conversation_id, :llm_conversations, null: false, on_delete: :cascade
+
+# FK to optional parent — set null on delete (orphan is ok)
+foreign_key :parent_message_id, :llm_messages, null: true, on_delete: :set_null
+```
+
+### Timestamp Semantics
+
+| Column | Meaning | Default | Nullable |
+|--------|---------|---------|----------|
+| `created_at` | When the event/action occurred in the real world (e.g. AMQP message timestamp, API request time) | `CURRENT_TIMESTAMP` | NOT NULL |
+| `inserted_at` | When the row was physically written to this database — always DB clock time | `CURRENT_TIMESTAMP` | NOT NULL |
+| `updated_at` | Last time the row was modified after initial insert. NULL means never updated. | none | NULL |
+
+`created_at` vs `inserted_at`: a message published at 14:00:00 that gets consumed and written at 14:00:03 has `created_at = 14:00:00` and `inserted_at = 14:00:03`. For synchronous writes they will be the same.
+
+### Index Conventions
+
+- `access_scope` — always indexed (high cardinality filter for multi-tenant queries)
+- `identity_canonical_name` — always indexed (user-facing search/filter)
+- `identity_principal_id` — always indexed (join path to identity tables)
+- `uuid` — always unique index (external reference lookups)
+- Timestamp columns used in WHERE clauses — indexed
+- Composite indexes for common query patterns: `index [:provider, :model_key]`
+
 ## Sequel ORM Rules
 
 Use Sequel associations as the object graph. References:
@@ -55,26 +160,30 @@ When Sequel cannot infer names, set `:class`, `:key`, `:primary_key`, `:join_tab
 
 All new tables in legion-data should follow this column convention. Required fields must be present on every table. Optional fields are added when the domain warrants them.
 
-### Required
+### Required (every table, in this order)
 
-| Column | Type | Purpose |
-|--------|------|---------|
-| `id` | `INTEGER PRIMARY KEY` (auto-increment) | Internal join key — never exposed externally |
-| `identity_principal_id` | `INTEGER` FK → `identity_principals.id` | The principal who caused this row to exist |
-| `identity_id` | `INTEGER` FK → `identities.id` | The specific provider-bound identity credential |
-| `identity_canonical_name` | `VARCHAR(255)` | Denormalized snapshot of the identity's canonical name for fast filtering without joins. This value is a point-in-time copy — it may become stale if the principal is renamed. Use the FK join for authoritative lookups. |
-| `created_at` | `TIMESTAMPTZ` | Row creation time |
-| `updated_at` | `TIMESTAMPTZ` | Last modification time |
+| Column | Sequel DSL | Purpose |
+|--------|-----------|---------|
+| `id` | `primary_key :id` | Auto-increment integer PK — internal join key, never exposed externally |
+| `uuid` | `String :uuid, size: 36, null: false, unique: true` | External reference — used in APIs, logs, AMQP correlation |
+| `access_scope` | `String :access_scope, size: 20, null: false, default: 'global', index: true` | Multi-tenant scoping (global, personal, team, org) |
+| `identity_principal_id` | `foreign_key :identity_principal_id, :identity_principals, null: true, on_delete: :set_null, on_update: :cascade` | FK to the principal who caused this row |
+| `identity_id` | `foreign_key :identity_id, :identities, null: true, on_delete: :set_null, on_update: :cascade` | FK to the specific provider-bound identity credential |
+| `identity_canonical_name` | `String :identity_canonical_name, size: 255, null: true, index: true` | Point-in-time snapshot of the identity's canonical name. NOT a FK. May become stale if principal is renamed — use FK join for authoritative lookups. |
+| `created_at` | `DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP` | When the event/action occurred (AMQP timestamp, request time) |
+| `inserted_at` | `DateTime :inserted_at, null: false, default: Sequel::CURRENT_TIMESTAMP` | When the row was physically written to the database |
+| `updated_at` | `DateTime :updated_at, null: true` | Set on row update; NULL means never updated after insert |
 
 ### Optional (add when applicable)
 
 | Column | Type | Purpose |
 |--------|------|---------|
-| `expires_at` | `TIMESTAMPTZ` | TTL / archival eligibility |
-| `content_type` | `VARCHAR(...)` | Classifier for the row's payload kind |
-| `conversation_id` | `INTEGER` FK → `llm_conversations.id` | Links to the LLM conversation that produced this row |
-| `contains_phi` | `BOOLEAN` | Row contains Protected Health Information |
-| `contains_pii` | `BOOLEAN` | Row contains Personally Identifiable Information |
+| `expires_at` | `DateTime, null: true` | TTL / archival eligibility |
+| `content_type` | `String, size: 64` | Classifier for the row's payload kind |
+| `conversation_id` | `foreign_key ..., :llm_conversations, on_delete: :cascade` | Links to the LLM conversation that produced this row |
+| `task_id` | `foreign_key ..., :tasks, on_delete: :set_null` | Links to the task that triggered this row |
+| `contains_phi` | `TrueClass, default: false` | Row contains Protected Health Information |
+| `contains_pii` | `TrueClass, default: false` | Row contains Personally Identifiable Information |
 
 ### Naming rules
 
@@ -87,6 +196,10 @@ All new tables in legion-data should follow this column convention. Required fie
 - `074`-`076`: Apollo field width, task idempotency, extract step timings.
 - `077`-`090`: LLM lifecycle ledger.
 - `091`-`096`: portable identity companion tables.
+- `097`: LLM dispatch fields (operation, correlation_id, provider_instance, dispatch_path).
+- `098`-`099`: Legacy identity table drop + rename (portable_identity_* → identity_*).
+- `100`-`102`: Apollo identity columns + access_scope + indexes.
+- `103`-`114`: LLM table identity standardization (access_scope, identity_principal_id, identity_id, identity_canonical_name).
 - Namespaced models: `Identity::*`, `Apollo::*`, `RBAC::*`, `LLM::*`.
 
 ## Boundaries

data/lib/legion/data/connection.rb

@@ -198,7 +198,12 @@ module Legion
             connected:          Legion::Settings[:data][:connected],
             fallback_active:    @fallback_active || false,
             configured_adapter: Legion::Settings[:data][:adapter]&.to_sym || :sqlite,
-            sequel_alive:       (begin; !@sequel&.test_connection.nil?; rescue StandardError; false; end)
+            sequel_alive:       (begin
+              !@sequel&.test_connection.nil?
+            rescue StandardError => e
+              log.debug("connection health check failed: #{e.message}")
+              false
+            end)
           }
         end
 

data/lib/legion/data/local.rb

@@ -48,9 +48,11 @@ module Legion
           @connection.run('PRAGMA journal_mode=WAL')
           @connection.run('PRAGMA busy_timeout=30000')
           @connection.run('PRAGMA synchronous=NORMAL')
+          @connection.run('PRAGMA cache_size=-20000')
+          @connection.run('PRAGMA mmap_size=268435456')
           @connected = true
           run_migrations
-          log.info "Legion::Data::Local connected to #{db_file} (WAL mode, 30s busy_timeout)"
+          log.info "Legion::Data::Local connected to #{db_file} (WAL mode, 30s busy_timeout, 20MB cache, 256MB mmap)"
         rescue StandardError => e
           handle_exception(e, level: :error, handled: false, operation: :local_setup, database: db_file)
           raise
@@ -99,7 +101,7 @@ module Legion
           stats[:file_size] = File.size(@db_path) if @db_path && File.exist?(@db_path)
 
           %w[page_size page_count freelist_count journal_mode
-             wal_autocheckpoint cache_size busy_timeout].each do |pragma|
+             wal_autocheckpoint cache_size busy_timeout mmap_size].each do |pragma|
             val = begin
               @connection.fetch("PRAGMA #{pragma}").single_value
             rescue StandardError => e

data/lib/legion/data/migrations/116_make_tool_calls_response_id_nullable.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:llm_tool_calls) do
+      drop_index :identity_principal_id, name: :idx_tool_calls_identity_principal_id, if_exists: true
+      set_column_allow_null :message_inference_response_id
+      add_index :identity_principal_id, name:  :idx_tool_calls_identity_principal_id,
+                                        where: Sequel.negate(identity_principal_id: nil)
+    end
+  end
+
+  down do
+    alter_table(:llm_tool_calls) do
+      drop_index :identity_principal_id, name: :idx_tool_calls_identity_principal_id, if_exists: true
+      set_column_not_null :message_inference_response_id
+      add_index :identity_principal_id, name:  :idx_tool_calls_identity_principal_id,
+                                        where: Sequel.negate(identity_principal_id: nil)
+    end
+  end
+end

data/lib/legion/data/migrations/117_add_conversation_id_to_llm_tool_calls.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:llm_tool_calls) do
+      add_foreign_key :conversation_id, :llm_conversations, null: true, on_delete: :set_null, on_update: :cascade
+      add_index :conversation_id
+    end
+  end
+
+  down do
+    alter_table(:llm_tool_calls) do
+      drop_column :conversation_id
+      # On SQLite, drop_column triggers table recreation which silently destroys
+      # partial indexes. Recreate the one from migration 109.
+      add_index :identity_principal_id,
+                name:          :idx_tool_calls_identity_principal_id,
+                where:         Sequel.negate(identity_principal_id: nil),
+                if_not_exists: true
+    end
+  end
+end

data/lib/legion/data/migrations/118_add_entity_type_to_audit_records.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# The Great Convergence (part 1 of 2): add entity_type column to audit_records on all adapters.
+# Migration 068 added this column on PostgreSQL only.
+# Production is already at 117+, so this migration only runs on SQLite/MySQL
+# deployments that missed it due to the postgres-only guard in migration 068.
+
+Sequel.migration do
+  up do
+    return unless table_exists?(:audit_records)
+
+    existing = schema(:audit_records).map(&:first)
+    return if existing.include?(:entity_type)
+
+    alter_table(:audit_records) do
+      add_column :entity_type, String, size: 100, null: true
+    end
+
+    add_index :audit_records, :entity_type, name: :idx_audit_records_entity_type, if_not_exists: true
+  end
+
+  down do
+    return unless table_exists?(:audit_records)
+
+    alter_table(:audit_records) do
+      drop_column :entity_type if schema(:audit_records).any? { |col, _| col == :entity_type }
+    end
+  end
+end

data/lib/legion/data/migrations/119_create_missing_apollo_tables.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# The Great Convergence (part 2): create apollo_relations, apollo_expertise,
+# apollo_access_log, and apollo_operations on all adapters.
+#
+# Migration 012 (postgres-only) created apollo_relations, apollo_expertise,
+# and apollo_access_log.
+# Migration 047 (postgres-only) created apollo_operations.
+# These tables were never created on SQLite/MySQL deployments.
+
+Sequel.migration do
+  up do
+    # apollo_relations
+    unless table_exists?(:apollo_relations)
+      create_table(:apollo_relations) do
+        primary_key :id
+        String :from_entry_id, size: 36, null: false
+        String :to_entry_id, size: 36, null: false
+        String :relation_type, null: false, size: 50
+        Float :weight, default: 1.0
+        String :source_agent, size: 255
+        DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+
+        index :from_entry_id, name: :idx_apollo_rel_from
+        index :to_entry_id, name: :idx_apollo_rel_to
+        index :relation_type, name: :idx_apollo_rel_type
+        index %i[from_entry_id relation_type], name: :idx_apollo_rel_composite
+      end
+    end
+
+    # apollo_expertise
+    unless table_exists?(:apollo_expertise)
+      create_table(:apollo_expertise) do
+        primary_key :id
+        String :agent_id, null: false, size: 255, index: { name: :idx_apollo_exp_agent }
+        String :domain, null: false, size: 255, index: { name: :idx_apollo_exp_domain }
+        Float :proficiency, default: 0.0
+        Integer :entry_count, default: 0
+        DateTime :last_active_at, default: Sequel::CURRENT_TIMESTAMP
+
+        index %i[agent_id domain], name: :idx_apollo_exp_composite
+      end
+    end
+
+    # apollo_access_log
+    unless table_exists?(:apollo_access_log)
+      create_table(:apollo_access_log) do
+        primary_key :id
+        String :entry_id, size: 36, index: { name: :idx_apollo_access_entry }
+        String :agent_id, null: false, size: 255
+        String :action, null: false, size: 20
+        DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+      end
+    end
+
+    # apollo_operations
+    unless table_exists?(:apollo_operations)
+      create_table(:apollo_operations) do
+        primary_key :id
+        String :operation, size: 50, null: false
+        String :actor, size: 255, null: false
+        String :target_type, size: 50
+        String :target_ids, text: true # serialized array; PG uses INTEGER[]
+        String :summary, text: true
+        String :detail, text: true, default: '{}' # serialized json; PG uses JSONB
+        String :old_state, text: true
+        String :new_state, text: true
+        String :reason, text: true
+        String :principal_id, size: 255
+        DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+
+        index :created_at, name: :idx_apollo_ops_created
+        index :operation, name: :idx_apollo_ops_operation
+        index :actor, name: :idx_apollo_ops_actor
+      end
+    end
+  end
+
+  down do
+    drop_table :apollo_operations if table_exists?(:apollo_operations)
+    drop_table :apollo_access_log if table_exists?(:apollo_access_log)
+    drop_table :apollo_expertise if table_exists?(:apollo_expertise)
+    drop_table :apollo_relations if table_exists?(:apollo_relations)
+  end
+end

data/lib/legion/data/migrations/120_add_missing_apollo_indexes.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# The Great Convergence (part 3): add missing indexes on apollo_* tables.
+#
+# Migration 047 (postgres-only) created dozens of indexes on apollo_entries,
+# apollo_relations, apollo_expertise, apollo_operations, and
+# apollo_entries_archive. These were never created on SQLite/MySQL.
+#
+# Vector indexes (hnsw) and GIN indexes are postgres-specific and skipped.
+#
+# NOTE: Uses raw CREATE INDEX IF NOT EXISTS SQL because Sequel's add_index
+# inside alter_table does not honor if_not_exists on SQLite (it triggers
+# table recreation which fails if the index already exists).
+
+Sequel.migration do
+  up do
+    return unless table_exists?(:apollo_entries)
+
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_submitted_by ON apollo_entries (submitted_by)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_submitted_from ON apollo_entries (submitted_from)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_status ON apollo_entries (status)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_confidence ON apollo_entries (confidence)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_created ON apollo_entries (created_at)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_updated ON apollo_entries (updated_at)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_domain ON apollo_entries (knowledge_domain)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_source_agent ON apollo_entries (source_agent)'
+    run "CREATE UNIQUE INDEX IF NOT EXISTS idx_apollo_content_hash ON apollo_entries (content_hash) WHERE status != 'archived'"
+    run "CREATE INDEX IF NOT EXISTS idx_apollo_active ON apollo_entries (id) WHERE status IN ('candidate', 'confirmed', 'disputed')"
+    run "CREATE INDEX IF NOT EXISTS idx_apollo_decay_target ON apollo_entries (updated_at) WHERE status != 'archived'"
+    run "CREATE INDEX IF NOT EXISTS idx_apollo_candidates ON apollo_entries (status, source_provider, source_channel) WHERE status = 'candidate'"
+
+    return unless table_exists?(:apollo_entries_archive)
+
+    run 'CREATE INDEX IF NOT EXISTS idx_archive_content_hash ON apollo_entries_archive (content_hash)'
+    run 'CREATE INDEX IF NOT EXISTS idx_archive_source_agent ON apollo_entries_archive (source_agent)'
+    run 'CREATE INDEX IF NOT EXISTS idx_archive_archived_at ON apollo_entries_archive (archived_at)'
+
+    return unless table_exists?(:apollo_relations)
+
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_rel_from ON apollo_relations (from_entry_id)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_rel_to ON apollo_relations (to_entry_id)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_rel_type ON apollo_relations (relation_type)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_rel_composite ON apollo_relations (from_entry_id, relation_type)'
+
+    return unless table_exists?(:apollo_expertise)
+
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_exp_agent ON apollo_expertise (agent_id)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_exp_domain ON apollo_expertise (domain)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_exp_composite ON apollo_expertise (agent_id, domain)'
+
+    return unless table_exists?(:apollo_operations)
+
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_ops_created ON apollo_operations (created_at)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_ops_operation ON apollo_operations (operation)'
+    run 'CREATE INDEX IF NOT EXISTS idx_apollo_ops_actor ON apollo_operations (actor)'
+  end
+
+  down do
+    %w[
+      idx_apollo_submitted_by idx_apollo_submitted_from idx_apollo_status
+      idx_apollo_confidence idx_apollo_created idx_apollo_updated
+      idx_apollo_domain idx_apollo_source_agent idx_apollo_content_hash
+      idx_apollo_active idx_apollo_decay_target idx_apollo_candidates
+      idx_archive_content_hash idx_archive_source_agent idx_archive_archived_at
+      idx_apollo_rel_from idx_apollo_rel_to idx_apollo_rel_type
+      idx_apollo_rel_composite
+      idx_apollo_exp_agent idx_apollo_exp_domain idx_apollo_exp_composite
+      idx_apollo_ops_created idx_apollo_ops_operation idx_apollo_ops_actor
+    ].each do |name|
+      run "DROP INDEX IF EXISTS #{name}"
+    end
+  end
+end

data/lib/legion/data/migrations/121_add_cache_token_metrics.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Add cached_input_tokens and cache_creation_tokens to llm_message_inference_metrics.
+# Tracks cache hit tokens (read from cache) and cache write tokens separately from
+# standard input/output token counts.
+#
+# See: https://github.com/LegionIO/legion-data/issues/55
+
+Sequel.migration do
+  up do
+    alter_table(:llm_message_inference_metrics) do
+      add_column :cached_input_tokens, Integer, null: false, default: 0
+      add_column :cache_creation_tokens, Integer, null: false, default: 0
+    end
+  end
+
+  down do
+    alter_table(:llm_message_inference_metrics) do
+      drop_column :cache_creation_tokens
+      drop_column :cached_input_tokens
+    end
+  end
+end

data/lib/legion/data/migrations/122_ensure_runtime_caller_columns.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# Migration 115 had a bug: it guarded the up block on the :definition column
+# (added by migration 055) instead of :runtime_caller_class. This means on
+# deployments where :definition existed but :runtime_caller_class did not,
+# the columns were added correctly. But the guard was checking the wrong
+# thing, and the down block has no guard at all.
+#
+# This migration ensures the columns exist on any deployment that might have
+# skipped them due to the 115 bug.
+
+Sequel.migration do
+  up do
+    if table_exists?(:llm_message_inference_requests)
+      cols = schema(:llm_message_inference_requests).map(&:first)
+      unless cols.include?(:runtime_caller_class) && cols.include?(:runtime_caller_client)
+        alter_table(:llm_message_inference_requests) do
+          add_column :runtime_caller_class, String, size: 255, null: true, index: true unless cols.include?(:runtime_caller_class)
+          add_column :runtime_caller_client, String, size: 255, null: true unless cols.include?(:runtime_caller_client)
+        end
+      end
+    end
+  end
+
+  down do
+    if table_exists?(:llm_message_inference_requests)
+      cols = schema(:llm_message_inference_requests).map(&:first)
+      if cols.include?(:runtime_caller_class) || cols.include?(:runtime_caller_client)
+        alter_table(:llm_message_inference_requests) do
+          drop_column :runtime_caller_client if cols.include?(:runtime_caller_client)
+          drop_column :runtime_caller_class if cols.include?(:runtime_caller_class)
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/migrations/123_add_llm_tool_calls_audit_columns.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:llm_tool_calls) do
+      add_column :tool_arguments_json, :text, null: true
+      add_column :tool_result_json, :text, null: true
+      add_column :tool_category, String, size: 64, null: true
+      add_column :data_handling_classification, String, size: 32, null: true
+      add_column :policy_decision, String, size: 32, null: true
+      add_column :requires_human_approval, TrueClass, null: true
+      add_index :tool_category, name: :idx_tool_calls_tool_category
+      add_index :data_handling_classification, name: :idx_tool_calls_data_handling_classification
+      add_index :policy_decision, name: :idx_tool_calls_policy_decision
+    end
+  end
+end

data/lib/legion/data/migrations/124_add_llm_tool_call_attempts_audit_columns.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:llm_tool_call_attempts) do
+      add_column :attempt_input_json, :text, null: true
+      add_column :attempt_output_json, :text, null: true
+      add_column :error_details_json, :text, null: true
+    end
+  end
+end

data/lib/legion/data/migrations/125_add_llm_escalation_events_audit_columns.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    next unless table_exists?(:llm_escalation_events)
+
+    alter_table(:llm_escalation_events) do
+      add_column :history_json, :text, null: true
+      add_column :outcome, String, size: 32, null: true
+      add_column :total_attempts, Integer, null: true
+      add_index :outcome, name: :idx_escalation_events_outcome
+    end
+  end
+
+  down do
+    next unless table_exists?(:llm_escalation_events)
+
+    alter_table(:llm_escalation_events) do
+      drop_index :outcome, name: :idx_escalation_events_outcome
+      drop_column :total_attempts
+      drop_column :outcome
+      drop_column :history_json
+    end
+  end
+end

data/lib/legion/data/migrations/126_add_llm_message_inference_responses_audit_columns.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# NOTE: response_content_hash already exists (migration 080) — skipped.
+
+Sequel.migration do
+  change do
+    alter_table(:llm_message_inference_responses) do
+      add_column :route_attempts, Integer, null: true, default: 0
+      add_column :escalation_chain_ref, String, size: 128, null: true
+      add_index :escalation_chain_ref, name: :idx_inference_responses_escalation_chain_ref
+    end
+  end
+end

data/lib/legion/data/migrations/127_add_llm_message_inference_requests_audit_columns.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+# NOTE: request_content_hash, curation_strategy, and tool_policy already exist
+# (migration 079) — all skipped. Only parent_request_id is new.
+
+Sequel.migration do
+  change do
+    alter_table(:llm_message_inference_requests) do
+      add_foreign_key :parent_request_id, :llm_message_inference_requests, null: true, on_delete: :set_null
+    end
+  end
+end

data/lib/legion/data/migrations/128_add_identity_columns_to_shared_tables.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    %i[apollo_access_log memory_traces memory_associations audit_log audit_records].each do |table|
+      next unless table_exists?(table)
+
+      cols = schema(table).map(&:first)
+      alter_table(table) do
+        add_column :identity_principal_id, Integer, null: true unless cols.include?(:identity_principal_id)
+        add_column :identity_id, Integer, null: true unless cols.include?(:identity_id)
+        add_column :identity_canonical_name, String, size: 255, null: true unless cols.include?(:identity_canonical_name)
+      end
+    end
+  end
+
+  down do
+    %i[apollo_access_log memory_traces memory_associations audit_log audit_records].each do |table|
+      next unless table_exists?(table)
+
+      cols = schema(table).map(&:first)
+      alter_table(table) do
+        drop_column :identity_canonical_name if cols.include?(:identity_canonical_name)
+        drop_column :identity_id if cols.include?(:identity_id)
+        drop_column :identity_principal_id if cols.include?(:identity_principal_id)
+      end
+    end
+  end
+end

data/lib/legion/data/models/apollo/model_helpers.rb

@@ -7,7 +7,8 @@ module Legion
         module ModelHelpers
           def self.table_available?(table_name)
             Legion::Data::Connection.sequel&.table_exists?(table_name)
-          rescue StandardError
+          rescue StandardError => e
+            log.error("table availability check failed for #{table_name}: #{e.message}")
             false
           end
         end

data/lib/legion/data/models/llm/tool_call.rb

@@ -10,6 +10,7 @@ module Legion
           include ModelHelpers
 
           many_to_one :message_inference_response
+          many_to_one :conversation
           many_to_one :requested_by_message, class: 'Legion::Data::Models::LLM::Message', key: :requested_by_message_id
           many_to_one :result_message, class: 'Legion::Data::Models::LLM::Message', key: :result_message_id
           one_to_many :tool_call_attempts

data/lib/legion/data/spool.rb

@@ -117,7 +117,8 @@ module Legion
 
         def load_event_file(path, sub_namespace)
           ::JSON.parse(File.binread(path), symbolize_names: true)
-        rescue Errno::ENOENT
+        rescue Errno::ENOENT => e
+          log.debug("spool event file not found: #{path}: #{e.message}")
           nil
         rescue ::JSON::ParserError, EOFError, ArgumentError => e
           quarantine_corrupt_file(path, sub_namespace, e)

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.8.8'
+    VERSION = '1.9.0'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.8.8
+  version: 1.9.0
 platform: ruby
 authors:
 - Esity
@@ -262,6 +262,19 @@ files:
 - lib/legion/data/migrations/113_add_llm_security_events_identity_columns.rb
 - lib/legion/data/migrations/114_add_llm_registry_events_identity_columns.rb
 - lib/legion/data/migrations/115_add_runtime_caller_columns.rb
+- lib/legion/data/migrations/116_make_tool_calls_response_id_nullable.rb
+- lib/legion/data/migrations/117_add_conversation_id_to_llm_tool_calls.rb
+- lib/legion/data/migrations/118_add_entity_type_to_audit_records.rb
+- lib/legion/data/migrations/119_create_missing_apollo_tables.rb
+- lib/legion/data/migrations/120_add_missing_apollo_indexes.rb
+- lib/legion/data/migrations/121_add_cache_token_metrics.rb
+- lib/legion/data/migrations/122_ensure_runtime_caller_columns.rb
+- lib/legion/data/migrations/123_add_llm_tool_calls_audit_columns.rb
+- lib/legion/data/migrations/124_add_llm_tool_call_attempts_audit_columns.rb
+- lib/legion/data/migrations/125_add_llm_escalation_events_audit_columns.rb
+- lib/legion/data/migrations/126_add_llm_message_inference_responses_audit_columns.rb
+- lib/legion/data/migrations/127_add_llm_message_inference_requests_audit_columns.rb
+- lib/legion/data/migrations/128_add_identity_columns_to_shared_tables.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo/access_log.rb
 - lib/legion/data/models/apollo/entries.rb