legionio 1.9.50 → 1.9.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/lib/legion/api/default_settings.rb +5 -4
- data/lib/legion/api/stats.rb +10 -9
- data/lib/legion/cli/chat/tools/model_comparison.rb +1 -1
- data/lib/legion/cli/chat_command.rb +2 -2
- data/lib/legion/cli/llm_command.rb +17 -14
- data/lib/legion/helpers/context.rb +1 -1
- data/lib/legion/service.rb +3 -2
- data/lib/legion/version.rb +1 -1
- metadata +1 -21
- data/exe/replay_ledger +0 -395
- data/extensions-agentic/lex-consent/db/migrations/001_create_consent_maps.rb +0 -24
- data/extensions-agentic/lex-consent/lex-consent.gemspec +0 -26
- data/extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/actors/tier_evaluation.rb +0 -112
- data/extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/models/consent_map.rb +0 -74
- data/extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/runners/consent.rb +0 -210
- data/extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/version.rb +0 -11
- data/extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent.rb +0 -15
- data/extensions-agentic/lex-reconciliation/Gemfile +0 -10
- data/extensions-agentic/lex-reconciliation/README.md +0 -73
- data/extensions-agentic/lex-reconciliation/lex-reconciliation.gemspec +0 -26
- data/extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/actors/reconciliation_cycle.rb +0 -143
- data/extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/drift_log.rb +0 -161
- data/extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/runners/drift_checker.rb +0 -137
- data/extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/version.rb +0 -9
- data/extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation.rb +0 -13
- data/extensions-agentic/lex-reconciliation/spec/legion/extensions/reconciliation/drift_log_spec.rb +0 -40
- data/extensions-agentic/lex-reconciliation/spec/legion/extensions/reconciliation/runners/drift_checker_spec.rb +0 -107
- data/extensions-agentic/lex-reconciliation/spec/legion/extensions/reconciliation_spec.rb +0 -19
- data/extensions-agentic/lex-reconciliation/spec/spec_helper.rb +0 -15
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ede4c3b865760acda216ca62dec3963eeba1dc510fa8320438e9fc6f5d290810
|
|
4
|
+
data.tar.gz: 4428bd1a57998cf59bc887fbbcf67fea6f37c1c4b3036f142dd0bff74d87823c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 12130c29fd51936cd819fc0aa7d320a0b5455b0bd20f4693196bf3e853ff2775ad1d5654233e00469795edc1cfc2c78377292c7b290d02fc2d821818da35ffa0
|
|
7
|
+
data.tar.gz: ae8b7b9820a8735cfbf6bcaa4bf74b05d49d63a628d4ed06a6a9ed5ffb360138a77f8ea8ad71637c20abbbc3b3b50bf08cf1b0683465b1207ecddffd6386b394
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# Legion Changelog
|
|
2
2
|
|
|
3
|
+
## [1.9.52] - 2026-08-07
|
|
4
|
+
### Fixed
|
|
5
|
+
- `/api/stats` + `legion llm status` now report routing on and read circuit state from Inventory SSOT; removed dead `Discovery::System/Ollama` guards and retargeted CLI discovery at `Inventory::Discovery::System`
|
|
6
|
+
- Fixed §4 exception-handling violations: `stats.rb` and `llm_command.rb` now use `handle_exception` instead of silent swallows or unstructured logging
|
|
7
|
+
|
|
8
|
+
## [1.9.51] - 2026-07-31
|
|
9
|
+
### Fixed
|
|
10
|
+
- Puma: add `force_shutdown_after` (default 10s) to server_settings to prevent indefinite shutdown hangs when streaming connections are wedged (puma/puma#3569). Previously Puma's default of `-1` meant SIGTERM would wait forever on any open SSE/streaming body thread that could not be interrupted
|
|
11
|
+
|
|
3
12
|
## [1.9.50] - 2026-07-15
|
|
4
13
|
### Fixed
|
|
5
14
|
- Backfill extension routes with API router after boot to fix /api/extensions/* 404s
|
|
@@ -20,10 +20,11 @@ module Legion
|
|
|
20
20
|
|
|
21
21
|
def self.puma_defaults
|
|
22
22
|
{
|
|
23
|
-
min_threads:
|
|
24
|
-
max_threads:
|
|
25
|
-
persistent_timeout:
|
|
26
|
-
first_data_timeout:
|
|
23
|
+
min_threads: 10,
|
|
24
|
+
max_threads: 16,
|
|
25
|
+
persistent_timeout: 20,
|
|
26
|
+
first_data_timeout: 30,
|
|
27
|
+
force_shutdown_after: 10
|
|
27
28
|
}
|
|
28
29
|
end
|
|
29
30
|
|
data/lib/legion/api/stats.rb
CHANGED
|
@@ -4,6 +4,8 @@ module Legion
|
|
|
4
4
|
class API < Sinatra::Base
|
|
5
5
|
module Routes
|
|
6
6
|
module Stats
|
|
7
|
+
extend Legion::Logging::Helper
|
|
8
|
+
|
|
7
9
|
def self.registered(app)
|
|
8
10
|
app.get '/api/stats' do
|
|
9
11
|
result = {}
|
|
@@ -137,16 +139,14 @@ module Legion
|
|
|
137
139
|
info[:default_provider] = s[:default_provider]
|
|
138
140
|
info[:pipeline_enabled] = s[:pipeline_enabled] == true
|
|
139
141
|
|
|
140
|
-
if defined?(Legion::LLM::Router)
|
|
142
|
+
if defined?(Legion::LLM::Router) # legit optional-gem guard (standards §6): legion-llm may be absent
|
|
141
143
|
info[:routing_enabled] = true
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
nil
|
|
149
|
-
end
|
|
144
|
+
info[:provider_health] = Legion::LLM::Inventory.lanes.each_with_object({}) do |lane, h|
|
|
145
|
+
key = "#{lane[:provider_family]}/#{lane[:instance_id]}"
|
|
146
|
+
h[key] = { circuit: lane.dig(:health, :circuit_state)&.to_s }
|
|
147
|
+
rescue StandardError => e
|
|
148
|
+
handle_exception(e, level: :warn, operation: 'api.stats.provider_health', lane: lane[:id])
|
|
149
|
+
next # skip the malformed lane, but the failure is logged + published, not swallowed
|
|
150
150
|
end
|
|
151
151
|
else
|
|
152
152
|
info[:routing_enabled] = false
|
|
@@ -158,6 +158,7 @@ module Legion
|
|
|
158
158
|
end
|
|
159
159
|
info
|
|
160
160
|
rescue StandardError => e
|
|
161
|
+
handle_exception(e, level: :warn, operation: 'api.stats.collect_llm')
|
|
161
162
|
{ error: e.message }
|
|
162
163
|
end
|
|
163
164
|
|
|
@@ -56,7 +56,7 @@ module Legion
|
|
|
56
56
|
return pricing if models_str.nil? || models_str.strip.empty?
|
|
57
57
|
|
|
58
58
|
names = models_str.split(',').map(&:strip).map(&:downcase)
|
|
59
|
-
pricing.select { |k, _| names.any? { |n| k.downcase.include?(n) } }
|
|
59
|
+
pricing.select { |k, _| names.any? { |n| k.downcase.include?(n) } }
|
|
60
60
|
end
|
|
61
61
|
|
|
62
62
|
def self.format_comparison(selected, tokens)
|
|
@@ -1045,8 +1045,8 @@ module Legion
|
|
|
1045
1045
|
def handle_review_in_session(scope, out)
|
|
1046
1046
|
scope = (scope || '').strip
|
|
1047
1047
|
diff = case scope
|
|
1048
|
-
when 'staged'
|
|
1049
|
-
when 'branch'
|
|
1048
|
+
when 'staged' then `git diff --staged 2>/dev/null`
|
|
1049
|
+
when 'branch' then `git diff main...HEAD 2>/dev/null`
|
|
1050
1050
|
when '', 'uncommitted' then `git diff 2>/dev/null`
|
|
1051
1051
|
else
|
|
1052
1052
|
out.error('Usage: /review [staged|uncommitted|branch]')
|
|
@@ -5,6 +5,8 @@ require 'thor'
|
|
|
5
5
|
module Legion
|
|
6
6
|
module CLI
|
|
7
7
|
class Llm < Thor
|
|
8
|
+
include Legion::Logging::Helper
|
|
9
|
+
|
|
8
10
|
namespace 'llm'
|
|
9
11
|
|
|
10
12
|
def self.exit_on_failure?
|
|
@@ -163,31 +165,31 @@ module Legion
|
|
|
163
165
|
end
|
|
164
166
|
|
|
165
167
|
def collect_routing
|
|
166
|
-
return { enabled: false } unless defined?(Legion::LLM::Router)
|
|
168
|
+
return { enabled: false } unless defined?(Legion::LLM::Router) # legit optional-gem guard (standards §6)
|
|
167
169
|
|
|
168
170
|
{
|
|
169
|
-
enabled:
|
|
171
|
+
enabled: true,
|
|
170
172
|
local_tier: Legion::LLM::Router.tier_available?(:local),
|
|
171
173
|
fleet_tier: Legion::LLM::Router.tier_available?(:fleet),
|
|
172
174
|
cloud_tier: Legion::LLM::Router.tier_available?(:cloud)
|
|
173
175
|
}
|
|
174
176
|
rescue StandardError => e
|
|
175
|
-
|
|
177
|
+
handle_exception(e, level: :warn, operation: 'cli.llm.collect_routing')
|
|
176
178
|
{ enabled: false }
|
|
177
179
|
end
|
|
178
180
|
|
|
179
181
|
def collect_system
|
|
180
|
-
return {} unless defined?(Legion::LLM::Discovery::System)
|
|
182
|
+
return {} unless defined?(Legion::LLM::Inventory::Discovery::System) # legit optional-gem guard (standards §6)
|
|
181
183
|
|
|
182
|
-
Legion::LLM::Discovery::System.refresh! if Legion::LLM::Discovery::System.stale?
|
|
184
|
+
Legion::LLM::Inventory::Discovery::System.refresh! if Legion::LLM::Inventory::Discovery::System.stale?
|
|
183
185
|
{
|
|
184
|
-
platform: Legion::LLM::Discovery::System.platform,
|
|
185
|
-
total_memory_mb: Legion::LLM::Discovery::System.total_memory_mb,
|
|
186
|
-
avail_memory_mb: Legion::LLM::Discovery::System.available_memory_mb,
|
|
187
|
-
memory_pressure: Legion::LLM::Discovery::System.memory_pressure?
|
|
186
|
+
platform: Legion::LLM::Inventory::Discovery::System.platform,
|
|
187
|
+
total_memory_mb: Legion::LLM::Inventory::Discovery::System.total_memory_mb,
|
|
188
|
+
avail_memory_mb: Legion::LLM::Inventory::Discovery::System.available_memory_mb,
|
|
189
|
+
memory_pressure: Legion::LLM::Inventory::Discovery::System.memory_pressure?
|
|
188
190
|
}
|
|
189
191
|
rescue StandardError => e
|
|
190
|
-
|
|
192
|
+
handle_exception(e, level: :warn, operation: 'cli.llm.collect_system')
|
|
191
193
|
{}
|
|
192
194
|
end
|
|
193
195
|
|
|
@@ -199,13 +201,14 @@ module Legion
|
|
|
199
201
|
next unless cfg[:enabled]
|
|
200
202
|
|
|
201
203
|
models = [cfg[:default_model]].compact
|
|
202
|
-
if name == :ollama && defined?(Legion::LLM::
|
|
204
|
+
if name == :ollama && defined?(Legion::LLM::Inventory) # legit optional-gem guard (standards §6)
|
|
203
205
|
begin
|
|
204
|
-
|
|
205
|
-
|
|
206
|
+
discovered = Legion::LLM::Inventory.lanes
|
|
207
|
+
.select { |l| l[:provider_family].to_s == 'ollama' }
|
|
208
|
+
.map { |l| l[:model] }.compact.uniq
|
|
206
209
|
models = discovered unless discovered.empty?
|
|
207
210
|
rescue StandardError => e
|
|
208
|
-
|
|
211
|
+
handle_exception(e, level: :warn, operation: 'cli.llm.collect_models')
|
|
209
212
|
end
|
|
210
213
|
end
|
|
211
214
|
result[name] = models
|
data/lib/legion/service.rb
CHANGED
|
@@ -382,8 +382,9 @@ module Legion
|
|
|
382
382
|
max_threads = puma_cfg[:max_threads]
|
|
383
383
|
thread_spec = "#{min_threads}:#{max_threads}"
|
|
384
384
|
puma_timeouts = {
|
|
385
|
-
persistent_timeout:
|
|
386
|
-
first_data_timeout:
|
|
385
|
+
persistent_timeout: puma_cfg[:persistent_timeout],
|
|
386
|
+
first_data_timeout: puma_cfg[:first_data_timeout],
|
|
387
|
+
force_shutdown_after: puma_cfg[:force_shutdown_after]
|
|
387
388
|
}.compact
|
|
388
389
|
|
|
389
390
|
tls_cfg = build_api_tls_config(api_settings)
|
data/lib/legion/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: legionio
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.9.
|
|
4
|
+
version: 1.9.52
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Esity
|
|
@@ -434,27 +434,7 @@ files:
|
|
|
434
434
|
- docker_deploy.rb
|
|
435
435
|
- exe/legion
|
|
436
436
|
- exe/legionio
|
|
437
|
-
- exe/replay_ledger
|
|
438
437
|
- exe/rollback_ledger
|
|
439
|
-
- extensions-agentic/lex-consent/db/migrations/001_create_consent_maps.rb
|
|
440
|
-
- extensions-agentic/lex-consent/lex-consent.gemspec
|
|
441
|
-
- extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent.rb
|
|
442
|
-
- extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/actors/tier_evaluation.rb
|
|
443
|
-
- extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/models/consent_map.rb
|
|
444
|
-
- extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/runners/consent.rb
|
|
445
|
-
- extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/version.rb
|
|
446
|
-
- extensions-agentic/lex-reconciliation/Gemfile
|
|
447
|
-
- extensions-agentic/lex-reconciliation/README.md
|
|
448
|
-
- extensions-agentic/lex-reconciliation/lex-reconciliation.gemspec
|
|
449
|
-
- extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation.rb
|
|
450
|
-
- extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/actors/reconciliation_cycle.rb
|
|
451
|
-
- extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/drift_log.rb
|
|
452
|
-
- extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/runners/drift_checker.rb
|
|
453
|
-
- extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/version.rb
|
|
454
|
-
- extensions-agentic/lex-reconciliation/spec/legion/extensions/reconciliation/drift_log_spec.rb
|
|
455
|
-
- extensions-agentic/lex-reconciliation/spec/legion/extensions/reconciliation/runners/drift_checker_spec.rb
|
|
456
|
-
- extensions-agentic/lex-reconciliation/spec/legion/extensions/reconciliation_spec.rb
|
|
457
|
-
- extensions-agentic/lex-reconciliation/spec/spec_helper.rb
|
|
458
438
|
- integration/self_generate_spec.rb
|
|
459
439
|
- legionio.gemspec
|
|
460
440
|
- lib/legion.rb
|
data/exe/replay_ledger
DELETED
|
@@ -1,395 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
# frozen_string_literal: true
|
|
3
|
-
|
|
4
|
-
# One-off script to migrate local ledger data to prod database.
|
|
5
|
-
# Handles FK remapping for identity tables by matching on natural keys.
|
|
6
|
-
#
|
|
7
|
-
# Usage:
|
|
8
|
-
# LOCAL_DB_URL="postgres://localhost/legionio" PROD_DB_URL="postgres://user:pass@prod/legionio" bundle exec exe/replay_ledger
|
|
9
|
-
#
|
|
10
|
-
# Optional:
|
|
11
|
-
# REPLAY_DRY_RUN=true (show counts, don't write)
|
|
12
|
-
|
|
13
|
-
require 'sequel'
|
|
14
|
-
require 'logger'
|
|
15
|
-
require 'uri'
|
|
16
|
-
require 'json'
|
|
17
|
-
require 'fileutils'
|
|
18
|
-
|
|
19
|
-
log = Logger.new($stdout)
|
|
20
|
-
log.level = Logger::INFO
|
|
21
|
-
|
|
22
|
-
# --- Signal handling for clean exit ---
|
|
23
|
-
module MigrationState
|
|
24
|
-
@shutdown = false
|
|
25
|
-
class << self
|
|
26
|
-
attr_accessor :shutdown
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
%w[INT TERM HUP].each do |sig|
|
|
31
|
-
Signal.trap(sig) do
|
|
32
|
-
if MigrationState.shutdown
|
|
33
|
-
warn "\nForce quit."
|
|
34
|
-
exit!(1)
|
|
35
|
-
end
|
|
36
|
-
MigrationState.shutdown = true
|
|
37
|
-
warn "\nShutdown requested — finishing current row, then saving manifests and exiting cleanly..."
|
|
38
|
-
end
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
local_url = ENV.fetch('LOCAL_DB_URL', 'postgres://localhost/legionio')
|
|
42
|
-
dry_run = ENV['REPLAY_DRY_RUN'] == 'true'
|
|
43
|
-
|
|
44
|
-
# Read prod creds from ~/.legionio/settings/z_data_override.json
|
|
45
|
-
prod_settings_path = File.expand_path('~/.legionio/settings/z_data_override.json')
|
|
46
|
-
abort "Missing #{prod_settings_path} — create it with host/database/user/password" unless File.exist?(prod_settings_path)
|
|
47
|
-
prod_config = JSON.parse(File.read(prod_settings_path))
|
|
48
|
-
prod_creds = prod_config.dig('data', 'creds') || abort("No data.creds in #{prod_settings_path}")
|
|
49
|
-
prod_url = "postgres://#{prod_creds['user']}:#{URI.encode_www_form_component(prod_creds['password'])}@#{prod_creds['host']}/#{prod_creds['database']}"
|
|
50
|
-
|
|
51
|
-
LOCAL = Sequel.connect(local_url)
|
|
52
|
-
PROD = Sequel.connect(prod_url)
|
|
53
|
-
|
|
54
|
-
log.info "Local: #{local_url.sub(/:[^:@]+@/, ':***@')}"
|
|
55
|
-
log.info "Prod: postgres://#{prod_creds['user']}:***@#{prod_creds['host']}/#{prod_creds['database']}"
|
|
56
|
-
log.info "Mode: #{dry_run ? 'DRY RUN' : 'LIVE'}"
|
|
57
|
-
|
|
58
|
-
# ============================================================
|
|
59
|
-
# PHASE 1: Sync identity_providers (match on name)
|
|
60
|
-
# ============================================================
|
|
61
|
-
|
|
62
|
-
log.info '--- Phase 1: identity_providers ---'
|
|
63
|
-
local_providers = LOCAL[:identity_providers].all
|
|
64
|
-
prod_providers = PROD[:identity_providers].all
|
|
65
|
-
prod_provider_by_name = prod_providers.to_h { |p| [p[:name], p] }
|
|
66
|
-
provider_id_map = {} # local_id → prod_id
|
|
67
|
-
|
|
68
|
-
local_providers.each do |lp|
|
|
69
|
-
prod_match = prod_provider_by_name[lp[:name]]
|
|
70
|
-
if prod_match
|
|
71
|
-
provider_id_map[lp[:id]] = prod_match[:id]
|
|
72
|
-
else
|
|
73
|
-
row = lp.except(:id)
|
|
74
|
-
if dry_run
|
|
75
|
-
log.info " [DRY] Would insert provider: #{lp[:name]}"
|
|
76
|
-
provider_id_map[lp[:id]] = -1
|
|
77
|
-
else
|
|
78
|
-
new_id = PROD[:identity_providers].insert(row)
|
|
79
|
-
provider_id_map[lp[:id]] = new_id
|
|
80
|
-
log.info " Inserted provider: #{lp[:name]} → id=#{new_id}"
|
|
81
|
-
end
|
|
82
|
-
end
|
|
83
|
-
end
|
|
84
|
-
log.info " Provider map: #{provider_id_map.size} entries (#{provider_id_map.count { |_, v| v != -1 }} matched)"
|
|
85
|
-
|
|
86
|
-
# ============================================================
|
|
87
|
-
# PHASE 2: Sync identity_principals (match on canonical_name)
|
|
88
|
-
# ============================================================
|
|
89
|
-
|
|
90
|
-
log.info '--- Phase 2: identity_principals ---'
|
|
91
|
-
local_principals = LOCAL[:identity_principals].all
|
|
92
|
-
prod_principals = PROD[:identity_principals].all
|
|
93
|
-
prod_principal_by_name = prod_principals.to_h { |p| [p[:canonical_name], p] }
|
|
94
|
-
principal_id_map = {} # local_id → prod_id
|
|
95
|
-
|
|
96
|
-
local_principals.each do |lp|
|
|
97
|
-
prod_match = prod_principal_by_name[lp[:canonical_name]]
|
|
98
|
-
if prod_match
|
|
99
|
-
principal_id_map[lp[:id]] = prod_match[:id]
|
|
100
|
-
else
|
|
101
|
-
row = lp.except(:id)
|
|
102
|
-
if dry_run
|
|
103
|
-
log.info " [DRY] Would insert principal: #{lp[:canonical_name]}"
|
|
104
|
-
principal_id_map[lp[:id]] = -1
|
|
105
|
-
else
|
|
106
|
-
new_id = PROD[:identity_principals].insert(row)
|
|
107
|
-
principal_id_map[lp[:id]] = new_id
|
|
108
|
-
log.info " Inserted principal: #{lp[:canonical_name]} → id=#{new_id}"
|
|
109
|
-
end
|
|
110
|
-
end
|
|
111
|
-
end
|
|
112
|
-
log.info " Principal map: #{principal_id_map.size} entries (#{principal_id_map.count { |_, v| v != -1 }} matched)"
|
|
113
|
-
|
|
114
|
-
# ============================================================
|
|
115
|
-
# PHASE 3: Sync identities (match on principal_id + provider_identity_key)
|
|
116
|
-
# ============================================================
|
|
117
|
-
|
|
118
|
-
log.info '--- Phase 3: identities ---'
|
|
119
|
-
local_identities = LOCAL[:identities].all
|
|
120
|
-
prod_identities = PROD[:identities].all
|
|
121
|
-
prod_identity_by_key = prod_identities.to_h { |i| ["#{i[:principal_id]}:#{i[:provider_identity_key]}", i] }
|
|
122
|
-
identity_id_map = {} # local_id → prod_id
|
|
123
|
-
|
|
124
|
-
local_identities.each do |li|
|
|
125
|
-
mapped_principal = principal_id_map[li[:principal_id]]
|
|
126
|
-
mapped_provider = provider_id_map[li[:provider_id]]
|
|
127
|
-
prod_key = "#{mapped_principal}:#{li[:provider_identity_key]}"
|
|
128
|
-
prod_match = prod_identity_by_key[prod_key]
|
|
129
|
-
|
|
130
|
-
if prod_match
|
|
131
|
-
identity_id_map[li[:id]] = prod_match[:id]
|
|
132
|
-
else
|
|
133
|
-
row = li.except(:id)
|
|
134
|
-
row[:principal_id] = mapped_principal
|
|
135
|
-
row[:provider_id] = mapped_provider
|
|
136
|
-
if dry_run
|
|
137
|
-
log.info " [DRY] Would insert identity: #{li[:provider_identity_key]} (principal=#{mapped_principal})"
|
|
138
|
-
identity_id_map[li[:id]] = -1
|
|
139
|
-
else
|
|
140
|
-
new_id = PROD[:identities].insert(row)
|
|
141
|
-
identity_id_map[li[:id]] = new_id
|
|
142
|
-
log.info " Inserted identity: #{li[:provider_identity_key]} → id=#{new_id}"
|
|
143
|
-
end
|
|
144
|
-
end
|
|
145
|
-
end
|
|
146
|
-
log.info " Identity map: #{identity_id_map.size} entries (#{identity_id_map.count { |_, v| v != -1 }} matched)"
|
|
147
|
-
|
|
148
|
-
# ============================================================
|
|
149
|
-
# PHASE 4: LLM tables — remap identity FKs, preserve UUID links
|
|
150
|
-
# ============================================================
|
|
151
|
-
|
|
152
|
-
def remap_identity_columns(row, principal_map, identity_map)
|
|
153
|
-
result = row.dup
|
|
154
|
-
# Different tables use different column names
|
|
155
|
-
%i[principal_id caller_principal_id].each do |col|
|
|
156
|
-
result[col] = principal_map[result[col]] if result.key?(col) && result[col] && principal_map.key?(result[col])
|
|
157
|
-
end
|
|
158
|
-
%i[identity_id caller_identity_id].each do |col|
|
|
159
|
-
result[col] = identity_map[result[col]] if result.key?(col) && result[col] && identity_map.key?(result[col])
|
|
160
|
-
end
|
|
161
|
-
result
|
|
162
|
-
end
|
|
163
|
-
|
|
164
|
-
MANIFEST_DIR = '/tmp/legion_migration_manifests'
|
|
165
|
-
|
|
166
|
-
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
|
|
167
|
-
def migrate_table(local_db, prod_db, table, id_map_out, log:, dry_run:, fk_maps: {}, identity_maps: {})
|
|
168
|
-
count = local_db[table].count
|
|
169
|
-
if count.zero?
|
|
170
|
-
log.info " #{table}: 0 rows, skipping"
|
|
171
|
-
return
|
|
172
|
-
end
|
|
173
|
-
|
|
174
|
-
log.info " #{table}: #{count} rows to migrate..."
|
|
175
|
-
|
|
176
|
-
# Check for UUID column to detect duplicates
|
|
177
|
-
columns = local_db[table].columns
|
|
178
|
-
has_uuid = columns.include?(:uuid)
|
|
179
|
-
log.info " #{table}: loading existing UUIDs from prod..." if has_uuid
|
|
180
|
-
prod_uuids = if has_uuid
|
|
181
|
-
PROD[table].select_map(:uuid).to_set
|
|
182
|
-
else
|
|
183
|
-
Set.new
|
|
184
|
-
end
|
|
185
|
-
log.info " #{table}: #{prod_uuids.size} existing UUIDs on prod" if has_uuid
|
|
186
|
-
|
|
187
|
-
FileUtils.mkdir_p(MANIFEST_DIR)
|
|
188
|
-
manifest_path = File.join(MANIFEST_DIR, "#{table}.json")
|
|
189
|
-
ids_path = File.join(MANIFEST_DIR, "#{table}_ids.jsonl")
|
|
190
|
-
|
|
191
|
-
inserted = 0
|
|
192
|
-
skipped = 0
|
|
193
|
-
start_time = Time.now
|
|
194
|
-
|
|
195
|
-
# Use block form to avoid file descriptor leaks
|
|
196
|
-
File.open(ids_path, 'a') do |ids_file|
|
|
197
|
-
local_db[table].order(:id).each do |row|
|
|
198
|
-
if MigrationState.shutdown
|
|
199
|
-
log.warn " #{table}: INTERRUPTED at row #{inserted + skipped}/#{count} — exiting cleanly"
|
|
200
|
-
break
|
|
201
|
-
end
|
|
202
|
-
|
|
203
|
-
local_id = row[:id]
|
|
204
|
-
|
|
205
|
-
# Skip if already exists on prod (by UUID)
|
|
206
|
-
if has_uuid && row[:uuid] && prod_uuids.include?(row[:uuid])
|
|
207
|
-
prod_row = prod_db[table].where(uuid: row[:uuid]).first
|
|
208
|
-
id_map_out[local_id] = prod_row[:id] if prod_row
|
|
209
|
-
skipped += 1
|
|
210
|
-
next
|
|
211
|
-
end
|
|
212
|
-
|
|
213
|
-
new_row = row.except(:id)
|
|
214
|
-
|
|
215
|
-
# Remap FK columns — if map is empty, NULL the column (deferred backfill)
|
|
216
|
-
fk_maps.each do |column, map|
|
|
217
|
-
next unless new_row.key?(column) && new_row[column]
|
|
218
|
-
|
|
219
|
-
new_row[column] = if map.empty?
|
|
220
|
-
nil
|
|
221
|
-
elsif map.key?(new_row[column])
|
|
222
|
-
map[new_row[column]]
|
|
223
|
-
end
|
|
224
|
-
end
|
|
225
|
-
|
|
226
|
-
# Remap identity columns
|
|
227
|
-
new_row = remap_identity_columns(new_row, identity_maps[:principals] || {}, identity_maps[:identities] || {}) unless identity_maps.empty?
|
|
228
|
-
|
|
229
|
-
if dry_run
|
|
230
|
-
inserted += 1
|
|
231
|
-
id_map_out[local_id] = -1
|
|
232
|
-
else
|
|
233
|
-
begin
|
|
234
|
-
new_id = prod_db[table].insert(new_row)
|
|
235
|
-
id_map_out[local_id] = new_id
|
|
236
|
-
ids_file.puts("#{local_id},#{new_id}")
|
|
237
|
-
ids_file.flush
|
|
238
|
-
inserted += 1
|
|
239
|
-
rescue Sequel::ForeignKeyConstraintViolation => e
|
|
240
|
-
log.warn " #{table}: FK violation on local_id=#{local_id} — #{e.message.lines.first.strip} — skipping"
|
|
241
|
-
skipped += 1
|
|
242
|
-
rescue Sequel::UniqueConstraintViolation => e
|
|
243
|
-
log.warn " #{table}: duplicate on local_id=#{local_id} — #{e.message.lines.first.strip} — skipping"
|
|
244
|
-
skipped += 1
|
|
245
|
-
rescue Sequel::DatabaseError => e
|
|
246
|
-
log.error " #{table}: DB error on local_id=#{local_id} — #{e.message.lines.first.strip} — skipping"
|
|
247
|
-
skipped += 1
|
|
248
|
-
end
|
|
249
|
-
end
|
|
250
|
-
|
|
251
|
-
# Progress logging every 100 rows
|
|
252
|
-
next unless ((inserted + skipped) % 100).zero?
|
|
253
|
-
|
|
254
|
-
elapsed = (Time.now - start_time).round(1)
|
|
255
|
-
rate = ((inserted + skipped) / [elapsed, 0.1].max).round(1)
|
|
256
|
-
log.info " #{table}: progress #{inserted + skipped}/#{count} (inserted=#{inserted} skipped=#{skipped}) #{elapsed}s #{rate} rows/s"
|
|
257
|
-
end
|
|
258
|
-
end
|
|
259
|
-
|
|
260
|
-
# Write final summary manifest
|
|
261
|
-
manifest = {
|
|
262
|
-
table: table.to_s,
|
|
263
|
-
started_at: start_time.iso8601,
|
|
264
|
-
completed_at: Time.now.iso8601,
|
|
265
|
-
inserted_count: inserted,
|
|
266
|
-
skipped_count: skipped,
|
|
267
|
-
interrupted: MigrationState.shutdown,
|
|
268
|
-
ids_file: ids_path
|
|
269
|
-
}
|
|
270
|
-
File.write(manifest_path, JSON.pretty_generate(manifest))
|
|
271
|
-
|
|
272
|
-
elapsed = (Time.now - start_time).round(1)
|
|
273
|
-
log.info " #{table}: done inserted=#{inserted} skipped=#{skipped} elapsed=#{elapsed}s"
|
|
274
|
-
log.info " #{table}: manifest → #{manifest_path}"
|
|
275
|
-
log.info " #{table}: IDs → #{ids_path}"
|
|
276
|
-
end
|
|
277
|
-
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
|
|
278
|
-
|
|
279
|
-
identity_maps = { principals: principal_id_map, identities: identity_id_map }
|
|
280
|
-
|
|
281
|
-
# --- llm_conversations ---
|
|
282
|
-
conversation_id_map = {}
|
|
283
|
-
unless MigrationState.shutdown
|
|
284
|
-
log.info '--- Phase 4: llm_conversations ---'
|
|
285
|
-
migrate_table(LOCAL, PROD, :llm_conversations, conversation_id_map,
|
|
286
|
-
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
287
|
-
end
|
|
288
|
-
|
|
289
|
-
# --- llm_message_inference_requests ---
|
|
290
|
-
request_id_map = {}
|
|
291
|
-
unless MigrationState.shutdown
|
|
292
|
-
log.info '--- Phase 5: llm_message_inference_requests ---'
|
|
293
|
-
migrate_table(LOCAL, PROD, :llm_message_inference_requests, request_id_map,
|
|
294
|
-
fk_maps: { conversation_id: conversation_id_map, latest_message_id: {} },
|
|
295
|
-
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
296
|
-
end
|
|
297
|
-
|
|
298
|
-
# --- llm_message_inference_responses ---
|
|
299
|
-
response_id_map = {}
|
|
300
|
-
unless MigrationState.shutdown
|
|
301
|
-
log.info '--- Phase 6: llm_message_inference_responses ---'
|
|
302
|
-
migrate_table(LOCAL, PROD, :llm_message_inference_responses, response_id_map,
|
|
303
|
-
fk_maps: { message_inference_request_id: request_id_map, response_message_id: {} },
|
|
304
|
-
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
305
|
-
end
|
|
306
|
-
|
|
307
|
-
# --- llm_message_inference_metrics ---
|
|
308
|
-
metrics_id_map = {}
|
|
309
|
-
unless MigrationState.shutdown
|
|
310
|
-
log.info '--- Phase 7: llm_message_inference_metrics ---'
|
|
311
|
-
migrate_table(LOCAL, PROD, :llm_message_inference_metrics, metrics_id_map,
|
|
312
|
-
fk_maps: { message_inference_request_id: request_id_map,
|
|
313
|
-
message_inference_response_id: response_id_map },
|
|
314
|
-
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
315
|
-
end
|
|
316
|
-
|
|
317
|
-
# --- llm_messages ---
|
|
318
|
-
message_id_map = {}
|
|
319
|
-
unless MigrationState.shutdown
|
|
320
|
-
log.info '--- Phase 8: llm_messages ---'
|
|
321
|
-
migrate_table(LOCAL, PROD, :llm_messages, message_id_map,
|
|
322
|
-
fk_maps: { conversation_id: conversation_id_map,
|
|
323
|
-
message_inference_request_id: request_id_map,
|
|
324
|
-
message_inference_response_id: response_id_map,
|
|
325
|
-
parent_message_id: message_id_map },
|
|
326
|
-
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
327
|
-
end
|
|
328
|
-
|
|
329
|
-
# --- llm_tool_calls ---
|
|
330
|
-
tool_call_id_map = {}
|
|
331
|
-
unless MigrationState.shutdown
|
|
332
|
-
log.info '--- Phase 9: llm_tool_calls ---'
|
|
333
|
-
migrate_table(LOCAL, PROD, :llm_tool_calls, tool_call_id_map,
|
|
334
|
-
fk_maps: { conversation_id: conversation_id_map,
|
|
335
|
-
message_inference_response_id: response_id_map,
|
|
336
|
-
requested_by_message_id: message_id_map,
|
|
337
|
-
result_message_id: message_id_map },
|
|
338
|
-
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
339
|
-
end
|
|
340
|
-
|
|
341
|
-
# --- llm_tool_call_attempts ---
|
|
342
|
-
tool_attempt_id_map = {}
|
|
343
|
-
unless MigrationState.shutdown
|
|
344
|
-
log.info '--- Phase 10: llm_tool_call_attempts ---'
|
|
345
|
-
migrate_table(LOCAL, PROD, :llm_tool_call_attempts, tool_attempt_id_map,
|
|
346
|
-
fk_maps: { tool_call_id: tool_call_id_map },
|
|
347
|
-
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
348
|
-
end
|
|
349
|
-
|
|
350
|
-
# --- Update latest_message_id and response_message_id now that messages exist ---
|
|
351
|
-
unless dry_run || MigrationState.shutdown
|
|
352
|
-
log.info '--- Phase 11: Backfill deferred FK columns ---'
|
|
353
|
-
|
|
354
|
-
backfilled = 0
|
|
355
|
-
LOCAL[:llm_message_inference_requests].where(Sequel.~(latest_message_id: nil)).each do |row|
|
|
356
|
-
break if MigrationState.shutdown
|
|
357
|
-
|
|
358
|
-
prod_req_id = request_id_map[row[:id]]
|
|
359
|
-
prod_msg_id = message_id_map[row[:latest_message_id]]
|
|
360
|
-
next unless prod_req_id && prod_msg_id
|
|
361
|
-
|
|
362
|
-
PROD[:llm_message_inference_requests].where(id: prod_req_id).update(latest_message_id: prod_msg_id)
|
|
363
|
-
backfilled += 1
|
|
364
|
-
end
|
|
365
|
-
|
|
366
|
-
LOCAL[:llm_message_inference_responses].where(Sequel.~(response_message_id: nil)).each do |row|
|
|
367
|
-
break if MigrationState.shutdown
|
|
368
|
-
|
|
369
|
-
prod_resp_id = response_id_map[row[:id]]
|
|
370
|
-
prod_msg_id = message_id_map[row[:response_message_id]]
|
|
371
|
-
next unless prod_resp_id && prod_msg_id
|
|
372
|
-
|
|
373
|
-
PROD[:llm_message_inference_responses].where(id: prod_resp_id).update(response_message_id: prod_msg_id)
|
|
374
|
-
backfilled += 1
|
|
375
|
-
end
|
|
376
|
-
|
|
377
|
-
log.info " Deferred FK backfill complete: #{backfilled} updates"
|
|
378
|
-
end
|
|
379
|
-
|
|
380
|
-
# --- Summary ---
|
|
381
|
-
log.info '=== Migration Summary ==='
|
|
382
|
-
log.info " Providers: #{provider_id_map.size}"
|
|
383
|
-
log.info " Principals: #{principal_id_map.size}"
|
|
384
|
-
log.info " Identities: #{identity_id_map.size}"
|
|
385
|
-
log.info " Conversations: #{conversation_id_map.size}"
|
|
386
|
-
log.info " Requests: #{request_id_map.size}"
|
|
387
|
-
log.info " Responses: #{response_id_map.size}"
|
|
388
|
-
log.info " Metrics: #{metrics_id_map.size}"
|
|
389
|
-
log.info " Messages: #{message_id_map.size}"
|
|
390
|
-
log.info " Tool calls: #{tool_call_id_map.size}"
|
|
391
|
-
log.info " Tool attempts: #{tool_attempt_id_map.size}"
|
|
392
|
-
log.info " Manifests: #{MANIFEST_DIR}/"
|
|
393
|
-
log.info dry_run ? '[DRY RUN COMPLETE]' : '[MIGRATION COMPLETE]'
|
|
394
|
-
|
|
395
|
-
log.info ' To rollback: bundle exec exe/rollback_ledger'
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
Sequel.migration do
|
|
4
|
-
change do
|
|
5
|
-
create_table(:consent_maps) do
|
|
6
|
-
primary_key :id
|
|
7
|
-
String :worker_id, null: false
|
|
8
|
-
String :from_tier, null: false
|
|
9
|
-
String :to_tier, null: false
|
|
10
|
-
String :requested_by, null: false
|
|
11
|
-
String :state, null: false, default: 'pending_approval'
|
|
12
|
-
String :resolved_by
|
|
13
|
-
Time :resolved_at
|
|
14
|
-
String :notes, text: true
|
|
15
|
-
String :context, text: true
|
|
16
|
-
Time :created_at
|
|
17
|
-
Time :updated_at
|
|
18
|
-
|
|
19
|
-
index :worker_id
|
|
20
|
-
index :state
|
|
21
|
-
index %i[worker_id state]
|
|
22
|
-
end
|
|
23
|
-
end
|
|
24
|
-
end
|