legion-data 1.8.8 → 1.8.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60b8f26149494282c49455aedc216eebf0db867fa359dea9404ca44d09ad9815
-  data.tar.gz: 4da8dfdb277d9ceda6807543f58c2804ae71f938056cff7f8bcd366c4bad9cae
+  metadata.gz: 18c9b2db6f7648fbad03b852d4565fe1c0ad0f46173a2130a6184d4b47b59d8a
+  data.tar.gz: 2b31d060c165c7656967642d80760ff7e94968f2d069633f9b7b407ea19f8808
 SHA512:
-  metadata.gz: ef647d212100c659d28847879153403ac1dd42c55c0cb9b1529fd1e17d73f029a912d23988f6701e59e4b25d326b8512cd89f0291bcc52a9e492e6fd4c85b055
-  data.tar.gz: '018844cceb163d4100cffcfa8ea1ed06b916889ca14111e9ca8398d9c3d2a045c9e9c4d73f735e8c22f681ccc834c23f732d08d68303973251c60085f5013c1b'
+  metadata.gz: acce9df23ea4557560b5fc8b18bbb5a06acef0cdefbe75d9b9168371043575ffd8285b069b83b69d4b17f898f349702a47f780ebcd8de44fa9fd2dbdd60ff0a0
+  data.tar.gz: 5f4c6c236e41ff258f5f2489435b0d6f76072e764d242f91d0c8e735ef6b369daa778fc3f1fcccf51192f2d9ad4006702e86be68354b0b84510eba60767e5471

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Legion::Data Changelog
 
+## [1.8.9] - 2026-05-26
+
+### Changed
+- Migration 116: make `llm_tool_calls.message_inference_response_id` nullable and drop composite unique index on `[message_inference_response_id, tool_call_index]`. Eliminates 30-40% dead-letter rate on tool audit messages caused by AMQP race between response and tool call writers.
+- Migration 117: add nullable `conversation_id` FK to `llm_tool_calls` referencing `llm_conversations`, so tool call rows can track their conversation even when the response row hasn't been written yet.
+- Add `many_to_one :conversation` association to `LLM::ToolCall` model.
+
 ## [1.8.8] - 2026-05-20
 
 ### Added

data/CLAUDE.md

@@ -32,11 +32,116 @@ jq '[.examples[] | select(.status != "passed") | {file_path, line_number, full_d
 
 - Never edit published migrations. Add a new migration.
 - Do not guard migrations with `create_table?`, `drop_table?`, `table_exists?`, `if_exists`, `if_not_exists`, `next if`, or `next unless`.
-- Keep migrations small enough to diagnose and roll back. Split by domain and dependency.
+- **One change per migration file.** Each migration modifies exactly ONE table. Never loop over tables. If a migration fails, you must be able to identify exactly what broke and roll back cleanly.
+- Never use `.each`, `.map`, or any iterator in a migration. If 12 tables need the same column, that's 12 migration files.
+- Never use raw SQL (`run '...'`) when Sequel DSL supports the operation. Use `add_index`, `drop_index`, `add_column`, `drop_column`, etc.
 - Use portable Sequel DSL unless the feature truly requires adapter-specific behavior.
 - Use integer `id` primary keys for joins and public `uuid` columns for APIs/logs/external references.
 - Normalize stable fields. Use JSON only for genuinely dynamic provider payloads or evidence.
 
+### Sequel Migration DSL Reference
+
+**Create table**: https://sequel.jeremyevans.net/rdoc/classes/Sequel/Database.html#method-i-create_table
+**Column options**: https://sequel.jeremyevans.net/rdoc/classes/Sequel/Schema/CreateTableGenerator.html#method-i-column
+
+### Create Table Pattern
+
+```ruby
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:example_records) do
+      primary_key :id
+      String :uuid, size: 36, null: false, unique: true
+
+      # Identity columns (required on every table)
+      String :access_scope, size: 20, null: false, default: 'global', index: true
+      foreign_key :identity_principal_id, :identity_principals, null: true, on_delete: :set_null, on_update: :cascade
+      foreign_key :identity_id, :identities, null: true, on_delete: :set_null, on_update: :cascade
+      String :identity_canonical_name, size: 255, null: true, index: true
+
+      # Domain columns here...
+
+      # Timestamps (required on every table)
+      DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP  # reflects when the event happened (request/AMQP timestamp)
+      DateTime :inserted_at, null: false, default: Sequel::CURRENT_TIMESTAMP # when the row was physically written to the database
+      DateTime :updated_at, null: true                                        # set on row update; NULL means never updated
+
+      index :identity_principal_id
+    end
+  end
+end
+```
+
+### Alter Table Pattern (adding a column)
+
+```ruby
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:target_table) do
+      add_column :new_column, String, size: 128, null: true, index: true
+    end
+  end
+
+  down do
+    alter_table(:target_table) do
+      drop_index :new_column
+      drop_column :new_column
+    end
+  end
+end
+```
+
+### Column Option Reference
+
+| Option | Purpose |
+|--------|---------|
+| `:null` | `false` = NOT NULL, `true` = nullable |
+| `:default` | Default value (use `Sequel::CURRENT_TIMESTAMP` for timestamps) |
+| `:index` | `true` creates an index on this column; pass a Hash for index options |
+| `:unique` | `true` adds a UNIQUE constraint |
+| `:on_delete` | FK behavior: `:cascade`, `:set_null`, `:restrict`, `:no_action` |
+| `:on_update` | FK behavior: `:cascade`, `:set_null`, `:restrict`, `:no_action` |
+| `:key` | For FKs — the referenced column (unnecessary if referencing primary key) |
+| `:size` | Column width for String/Decimal |
+| `:text` | `true` for TEXT columns (unlimited length) |
+
+### Foreign Key Conventions
+
+```ruby
+# FK to identity tables — always cascade updates, set null on delete
+foreign_key :identity_principal_id, :identity_principals, null: true, on_delete: :set_null, on_update: :cascade
+foreign_key :identity_id, :identities, null: true, on_delete: :set_null, on_update: :cascade
+
+# FK to domain tables — cascade delete (child dies with parent)
+foreign_key :conversation_id, :llm_conversations, null: false, on_delete: :cascade
+
+# FK to optional parent — set null on delete (orphan is ok)
+foreign_key :parent_message_id, :llm_messages, null: true, on_delete: :set_null
+```
+
+### Timestamp Semantics
+
+| Column | Meaning | Default | Nullable |
+|--------|---------|---------|----------|
+| `created_at` | When the event/action occurred in the real world (e.g. AMQP message timestamp, API request time) | `CURRENT_TIMESTAMP` | NOT NULL |
+| `inserted_at` | When the row was physically written to this database — always DB clock time | `CURRENT_TIMESTAMP` | NOT NULL |
+| `updated_at` | Last time the row was modified after initial insert. NULL means never updated. | none | NULL |
+
+`created_at` vs `inserted_at`: a message published at 14:00:00 that gets consumed and written at 14:00:03 has `created_at = 14:00:00` and `inserted_at = 14:00:03`. For synchronous writes they will be the same.
+
+### Index Conventions
+
+- `access_scope` — always indexed (high cardinality filter for multi-tenant queries)
+- `identity_canonical_name` — always indexed (user-facing search/filter)
+- `identity_principal_id` — always indexed (join path to identity tables)
+- `uuid` — always unique index (external reference lookups)
+- Timestamp columns used in WHERE clauses — indexed
+- Composite indexes for common query patterns: `index [:provider, :model_key]`
+
 ## Sequel ORM Rules
 
 Use Sequel associations as the object graph. References:
@@ -55,26 +160,30 @@ When Sequel cannot infer names, set `:class`, `:key`, `:primary_key`, `:join_tab
 
 All new tables in legion-data should follow this column convention. Required fields must be present on every table. Optional fields are added when the domain warrants them.
 
-### Required
+### Required (every table, in this order)
 
-| Column | Type | Purpose |
-|--------|------|---------|
-| `id` | `INTEGER PRIMARY KEY` (auto-increment) | Internal join key — never exposed externally |
-| `identity_principal_id` | `INTEGER` FK → `identity_principals.id` | The principal who caused this row to exist |
-| `identity_id` | `INTEGER` FK → `identities.id` | The specific provider-bound identity credential |
-| `identity_canonical_name` | `VARCHAR(255)` | Denormalized snapshot of the identity's canonical name for fast filtering without joins. This value is a point-in-time copy — it may become stale if the principal is renamed. Use the FK join for authoritative lookups. |
-| `created_at` | `TIMESTAMPTZ` | Row creation time |
-| `updated_at` | `TIMESTAMPTZ` | Last modification time |
+| Column | Sequel DSL | Purpose |
+|--------|-----------|---------|
+| `id` | `primary_key :id` | Auto-increment integer PK — internal join key, never exposed externally |
+| `uuid` | `String :uuid, size: 36, null: false, unique: true` | External reference — used in APIs, logs, AMQP correlation |
+| `access_scope` | `String :access_scope, size: 20, null: false, default: 'global', index: true` | Multi-tenant scoping (global, personal, team, org) |
+| `identity_principal_id` | `foreign_key :identity_principal_id, :identity_principals, null: true, on_delete: :set_null, on_update: :cascade` | FK to the principal who caused this row |
+| `identity_id` | `foreign_key :identity_id, :identities, null: true, on_delete: :set_null, on_update: :cascade` | FK to the specific provider-bound identity credential |
+| `identity_canonical_name` | `String :identity_canonical_name, size: 255, null: true, index: true` | Point-in-time snapshot of the identity's canonical name. NOT a FK. May become stale if principal is renamed — use FK join for authoritative lookups. |
+| `created_at` | `DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP` | When the event/action occurred (AMQP timestamp, request time) |
+| `inserted_at` | `DateTime :inserted_at, null: false, default: Sequel::CURRENT_TIMESTAMP` | When the row was physically written to the database |
+| `updated_at` | `DateTime :updated_at, null: true` | Set on row update; NULL means never updated after insert |
 
 ### Optional (add when applicable)
 
 | Column | Type | Purpose |
 |--------|------|---------|
-| `expires_at` | `TIMESTAMPTZ` | TTL / archival eligibility |
-| `content_type` | `VARCHAR(...)` | Classifier for the row's payload kind |
-| `conversation_id` | `INTEGER` FK → `llm_conversations.id` | Links to the LLM conversation that produced this row |
-| `contains_phi` | `BOOLEAN` | Row contains Protected Health Information |
-| `contains_pii` | `BOOLEAN` | Row contains Personally Identifiable Information |
+| `expires_at` | `DateTime, null: true` | TTL / archival eligibility |
+| `content_type` | `String, size: 64` | Classifier for the row's payload kind |
+| `conversation_id` | `foreign_key ..., :llm_conversations, on_delete: :cascade` | Links to the LLM conversation that produced this row |
+| `task_id` | `foreign_key ..., :tasks, on_delete: :set_null` | Links to the task that triggered this row |
+| `contains_phi` | `TrueClass, default: false` | Row contains Protected Health Information |
+| `contains_pii` | `TrueClass, default: false` | Row contains Personally Identifiable Information |
 
 ### Naming rules
 
@@ -87,6 +196,10 @@ All new tables in legion-data should follow this column convention. Required fie
 - `074`-`076`: Apollo field width, task idempotency, extract step timings.
 - `077`-`090`: LLM lifecycle ledger.
 - `091`-`096`: portable identity companion tables.
+- `097`: LLM dispatch fields (operation, correlation_id, provider_instance, dispatch_path).
+- `098`-`099`: Legacy identity table drop + rename (portable_identity_* → identity_*).
+- `100`-`102`: Apollo identity columns + access_scope + indexes.
+- `103`-`114`: LLM table identity standardization (access_scope, identity_principal_id, identity_id, identity_canonical_name).
 - Namespaced models: `Identity::*`, `Apollo::*`, `RBAC::*`, `LLM::*`.
 
 ## Boundaries

data/lib/legion/data/migrations/116_make_tool_calls_response_id_nullable.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:llm_tool_calls) do
+      set_column_allow_null :message_inference_response_id
+    end
+
+    # SQLite's set_column_allow_null recreates the table internally, which
+    # drops partial indexes invisible to Sequel's indexes() method. Restore
+    # the partial index from migration 109 (no-op on PG where it survives).
+    alter_table(:llm_tool_calls) do
+      add_index :identity_principal_id, name:          :idx_tool_calls_identity_principal_id,
+                                        where:         Sequel.negate(identity_principal_id: nil),
+                                        ignore_errors: true
+    end
+  end
+
+  down do
+    alter_table(:llm_tool_calls) do
+      set_column_not_null :message_inference_response_id
+    end
+
+    alter_table(:llm_tool_calls) do
+      add_index :identity_principal_id, name:          :idx_tool_calls_identity_principal_id,
+                                        where:         Sequel.negate(identity_principal_id: nil),
+                                        ignore_errors: true
+    end
+  end
+end

data/lib/legion/data/migrations/117_add_conversation_id_to_llm_tool_calls.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:llm_tool_calls) do
+      add_foreign_key :conversation_id, :llm_conversations, null: true, on_delete: :set_null, on_update: :cascade
+      add_index :conversation_id
+    end
+  end
+
+  down do
+    alter_table(:llm_tool_calls) do
+      drop_index :conversation_id
+      drop_foreign_key :conversation_id
+    end
+  end
+end

data/lib/legion/data/models/llm/tool_call.rb

@@ -10,6 +10,7 @@ module Legion
           include ModelHelpers
 
           many_to_one :message_inference_response
+          many_to_one :conversation
           many_to_one :requested_by_message, class: 'Legion::Data::Models::LLM::Message', key: :requested_by_message_id
           many_to_one :result_message, class: 'Legion::Data::Models::LLM::Message', key: :result_message_id
           one_to_many :tool_call_attempts

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.8.8'
+    VERSION = '1.8.9'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.8.8
+  version: 1.8.9
 platform: ruby
 authors:
 - Esity
@@ -262,6 +262,8 @@ files:
 - lib/legion/data/migrations/113_add_llm_security_events_identity_columns.rb
 - lib/legion/data/migrations/114_add_llm_registry_events_identity_columns.rb
 - lib/legion/data/migrations/115_add_runtime_caller_columns.rb
+- lib/legion/data/migrations/116_make_tool_calls_response_id_nullable.rb
+- lib/legion/data/migrations/117_add_conversation_id_to_llm_tool_calls.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo/access_log.rb
 - lib/legion/data/models/apollo/entries.rb