legionio 1.9.50 → 1.9.52

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +9 -0
  3. data/lib/legion/api/default_settings.rb +5 -4
  4. data/lib/legion/api/stats.rb +10 -9
  5. data/lib/legion/cli/chat/tools/model_comparison.rb +1 -1
  6. data/lib/legion/cli/chat_command.rb +2 -2
  7. data/lib/legion/cli/llm_command.rb +17 -14
  8. data/lib/legion/helpers/context.rb +1 -1
  9. data/lib/legion/service.rb +3 -2
  10. data/lib/legion/version.rb +1 -1
  11. metadata +1 -21
  12. data/exe/replay_ledger +0 -395
  13. data/extensions-agentic/lex-consent/db/migrations/001_create_consent_maps.rb +0 -24
  14. data/extensions-agentic/lex-consent/lex-consent.gemspec +0 -26
  15. data/extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/actors/tier_evaluation.rb +0 -112
  16. data/extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/models/consent_map.rb +0 -74
  17. data/extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/runners/consent.rb +0 -210
  18. data/extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/version.rb +0 -11
  19. data/extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent.rb +0 -15
  20. data/extensions-agentic/lex-reconciliation/Gemfile +0 -10
  21. data/extensions-agentic/lex-reconciliation/README.md +0 -73
  22. data/extensions-agentic/lex-reconciliation/lex-reconciliation.gemspec +0 -26
  23. data/extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/actors/reconciliation_cycle.rb +0 -143
  24. data/extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/drift_log.rb +0 -161
  25. data/extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/runners/drift_checker.rb +0 -137
  26. data/extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/version.rb +0 -9
  27. data/extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation.rb +0 -13
  28. data/extensions-agentic/lex-reconciliation/spec/legion/extensions/reconciliation/drift_log_spec.rb +0 -40
  29. data/extensions-agentic/lex-reconciliation/spec/legion/extensions/reconciliation/runners/drift_checker_spec.rb +0 -107
  30. data/extensions-agentic/lex-reconciliation/spec/legion/extensions/reconciliation_spec.rb +0 -19
  31. data/extensions-agentic/lex-reconciliation/spec/spec_helper.rb +0 -15
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 836093ff32350c705da43ab5a36cf9b0e5e135e24556bf5b2c10c50df421ee11
4
- data.tar.gz: ac682d87908c360cc90409e9fa773a5a8937245104394dbcd940fea67ca10141
3
+ metadata.gz: ede4c3b865760acda216ca62dec3963eeba1dc510fa8320438e9fc6f5d290810
4
+ data.tar.gz: 4428bd1a57998cf59bc887fbbcf67fea6f37c1c4b3036f142dd0bff74d87823c
5
5
  SHA512:
6
- metadata.gz: f350c4a919b2746bd5fcc72dfa22e03b4e30217a5d934a6fb50411903708473d394e8652b7bb09b27672ee1c93ed222c586a241d93f5aa40eaddae690521a312
7
- data.tar.gz: e80c66310e5e90fc93c19cc3c672e4075dac237b27105a0ea3cc5c8cad5d295841bfeb0a893edaf34a1615b44b7fe843c6c060bdc69ef03086ccc200ced7e609
6
+ metadata.gz: 12130c29fd51936cd819fc0aa7d320a0b5455b0bd20f4693196bf3e853ff2775ad1d5654233e00469795edc1cfc2c78377292c7b290d02fc2d821818da35ffa0
7
+ data.tar.gz: ae8b7b9820a8735cfbf6bcaa4bf74b05d49d63a628d4ed06a6a9ed5ffb360138a77f8ea8ad71637c20abbbc3b3b50bf08cf1b0683465b1207ecddffd6386b394
data/CHANGELOG.md CHANGED
@@ -1,5 +1,14 @@
1
1
  # Legion Changelog
2
2
 
3
+ ## [1.9.52] - 2026-08-07
4
+ ### Fixed
5
+ - `/api/stats` + `legion llm status` now report routing on and read circuit state from Inventory SSOT; removed dead `Discovery::System/Ollama` guards and retargeted CLI discovery at `Inventory::Discovery::System`
6
+ - Fixed §4 exception-handling violations: `stats.rb` and `llm_command.rb` now use `handle_exception` instead of silent swallows or unstructured logging
7
+
8
+ ## [1.9.51] - 2026-07-31
9
+ ### Fixed
10
+ - Puma: add `force_shutdown_after` (default 10s) to server_settings to prevent indefinite shutdown hangs when streaming connections are wedged (puma/puma#3569). Previously Puma's default of `-1` meant SIGTERM would wait forever on any open SSE/streaming body thread that could not be interrupted
11
+
3
12
  ## [1.9.50] - 2026-07-15
4
13
  ### Fixed
5
14
  - Backfill extension routes with API router after boot to fix /api/extensions/* 404s
@@ -20,10 +20,11 @@ module Legion
20
20
 
21
21
  def self.puma_defaults
22
22
  {
23
- min_threads: 10,
24
- max_threads: 16,
25
- persistent_timeout: 20,
26
- first_data_timeout: 30
23
+ min_threads: 10,
24
+ max_threads: 16,
25
+ persistent_timeout: 20,
26
+ first_data_timeout: 30,
27
+ force_shutdown_after: 10
27
28
  }
28
29
  end
29
30
 
@@ -4,6 +4,8 @@ module Legion
4
4
  class API < Sinatra::Base
5
5
  module Routes
6
6
  module Stats
7
+ extend Legion::Logging::Helper
8
+
7
9
  def self.registered(app)
8
10
  app.get '/api/stats' do
9
11
  result = {}
@@ -137,16 +139,14 @@ module Legion
137
139
  info[:default_provider] = s[:default_provider]
138
140
  info[:pipeline_enabled] = s[:pipeline_enabled] == true
139
141
 
140
- if defined?(Legion::LLM::Router) && Legion::LLM::Router.routing_enabled?
142
+ if defined?(Legion::LLM::Router) # legit optional-gem guard (standards §6): legion-llm may be absent
141
143
  info[:routing_enabled] = true
142
- tracker = Legion::LLM::Router.health_tracker
143
- if tracker
144
- providers = s[:providers] || {}
145
- info[:provider_health] = providers.each_with_object({}) do |(name, _cfg), h|
146
- h[name] = { circuit: tracker.circuit_state(name)&.to_s }
147
- rescue StandardError
148
- nil
149
- end
144
+ info[:provider_health] = Legion::LLM::Inventory.lanes.each_with_object({}) do |lane, h|
145
+ key = "#{lane[:provider_family]}/#{lane[:instance_id]}"
146
+ h[key] = { circuit: lane.dig(:health, :circuit_state)&.to_s }
147
+ rescue StandardError => e
148
+ handle_exception(e, level: :warn, operation: 'api.stats.provider_health', lane: lane[:id])
149
+ next # skip the malformed lane, but the failure is logged + published, not swallowed
150
150
  end
151
151
  else
152
152
  info[:routing_enabled] = false
@@ -158,6 +158,7 @@ module Legion
158
158
  end
159
159
  info
160
160
  rescue StandardError => e
161
+ handle_exception(e, level: :warn, operation: 'api.stats.collect_llm')
161
162
  { error: e.message }
162
163
  end
163
164
 
@@ -56,7 +56,7 @@ module Legion
56
56
  return pricing if models_str.nil? || models_str.strip.empty?
57
57
 
58
58
  names = models_str.split(',').map(&:strip).map(&:downcase)
59
- pricing.select { |k, _| names.any? { |n| k.downcase.include?(n) } } # rubocop:disable Style/ArrayIntersect
59
+ pricing.select { |k, _| names.any? { |n| k.downcase.include?(n) } }
60
60
  end
61
61
 
62
62
  def self.format_comparison(selected, tokens)
@@ -1045,8 +1045,8 @@ module Legion
1045
1045
  def handle_review_in_session(scope, out)
1046
1046
  scope = (scope || '').strip
1047
1047
  diff = case scope
1048
- when 'staged' then `git diff --staged 2>/dev/null`
1049
- when 'branch' then `git diff main...HEAD 2>/dev/null`
1048
+ when 'staged' then `git diff --staged 2>/dev/null`
1049
+ when 'branch' then `git diff main...HEAD 2>/dev/null`
1050
1050
  when '', 'uncommitted' then `git diff 2>/dev/null`
1051
1051
  else
1052
1052
  out.error('Usage: /review [staged|uncommitted|branch]')
@@ -5,6 +5,8 @@ require 'thor'
5
5
  module Legion
6
6
  module CLI
7
7
  class Llm < Thor
8
+ include Legion::Logging::Helper
9
+
8
10
  namespace 'llm'
9
11
 
10
12
  def self.exit_on_failure?
@@ -163,31 +165,31 @@ module Legion
163
165
  end
164
166
 
165
167
  def collect_routing
166
- return { enabled: false } unless defined?(Legion::LLM::Router)
168
+ return { enabled: false } unless defined?(Legion::LLM::Router) # legit optional-gem guard (standards §6)
167
169
 
168
170
  {
169
- enabled: Legion::LLM::Router.routing_enabled?,
171
+ enabled: true,
170
172
  local_tier: Legion::LLM::Router.tier_available?(:local),
171
173
  fleet_tier: Legion::LLM::Router.tier_available?(:fleet),
172
174
  cloud_tier: Legion::LLM::Router.tier_available?(:cloud)
173
175
  }
174
176
  rescue StandardError => e
175
- Legion::Logging.warn("LlmCommand#collect_routing failed: #{e.message}") if defined?(Legion::Logging)
177
+ handle_exception(e, level: :warn, operation: 'cli.llm.collect_routing')
176
178
  { enabled: false }
177
179
  end
178
180
 
179
181
  def collect_system
180
- return {} unless defined?(Legion::LLM::Discovery::System)
182
+ return {} unless defined?(Legion::LLM::Inventory::Discovery::System) # legit optional-gem guard (standards §6)
181
183
 
182
- Legion::LLM::Discovery::System.refresh! if Legion::LLM::Discovery::System.stale?
184
+ Legion::LLM::Inventory::Discovery::System.refresh! if Legion::LLM::Inventory::Discovery::System.stale?
183
185
  {
184
- platform: Legion::LLM::Discovery::System.platform,
185
- total_memory_mb: Legion::LLM::Discovery::System.total_memory_mb,
186
- avail_memory_mb: Legion::LLM::Discovery::System.available_memory_mb,
187
- memory_pressure: Legion::LLM::Discovery::System.memory_pressure?
186
+ platform: Legion::LLM::Inventory::Discovery::System.platform,
187
+ total_memory_mb: Legion::LLM::Inventory::Discovery::System.total_memory_mb,
188
+ avail_memory_mb: Legion::LLM::Inventory::Discovery::System.available_memory_mb,
189
+ memory_pressure: Legion::LLM::Inventory::Discovery::System.memory_pressure?
188
190
  }
189
191
  rescue StandardError => e
190
- Legion::Logging.warn("LlmCommand#collect_system failed: #{e.message}") if defined?(Legion::Logging)
192
+ handle_exception(e, level: :warn, operation: 'cli.llm.collect_system')
191
193
  {}
192
194
  end
193
195
 
@@ -199,13 +201,14 @@ module Legion
199
201
  next unless cfg[:enabled]
200
202
 
201
203
  models = [cfg[:default_model]].compact
202
- if name == :ollama && defined?(Legion::LLM::Discovery::Ollama)
204
+ if name == :ollama && defined?(Legion::LLM::Inventory) # legit optional-gem guard (standards §6)
203
205
  begin
204
- Legion::LLM::Discovery::Ollama.refresh! if Legion::LLM::Discovery::Ollama.stale?
205
- discovered = Legion::LLM::Discovery::Ollama.model_names
206
+ discovered = Legion::LLM::Inventory.lanes
207
+ .select { |l| l[:provider_family].to_s == 'ollama' }
208
+ .map { |l| l[:model] }.compact.uniq
206
209
  models = discovered unless discovered.empty?
207
210
  rescue StandardError => e
208
- Legion::Logging.debug("LlmCommand#collect_models ollama discovery failed: #{e.message}") if defined?(Legion::Logging)
211
+ handle_exception(e, level: :warn, operation: 'cli.llm.collect_models')
209
212
  end
210
213
  end
211
214
  result[name] = models
@@ -25,7 +25,7 @@ module Legion
25
25
  return { success: true, files: [] } unless Dir.exist?(base)
26
26
 
27
27
  files = Dir.glob(File.join(base, '**', '*')).select { |f| File.file?(f) }
28
- .map do |f|
28
+ .map do |f|
29
29
  f.sub("#{context_dir}/",
30
30
  '')
31
31
  end
@@ -382,8 +382,9 @@ module Legion
382
382
  max_threads = puma_cfg[:max_threads]
383
383
  thread_spec = "#{min_threads}:#{max_threads}"
384
384
  puma_timeouts = {
385
- persistent_timeout: puma_cfg[:persistent_timeout],
386
- first_data_timeout: puma_cfg[:first_data_timeout]
385
+ persistent_timeout: puma_cfg[:persistent_timeout],
386
+ first_data_timeout: puma_cfg[:first_data_timeout],
387
+ force_shutdown_after: puma_cfg[:force_shutdown_after]
387
388
  }.compact
388
389
 
389
390
  tls_cfg = build_api_tls_config(api_settings)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Legion
4
- VERSION = '1.9.50'
4
+ VERSION = '1.9.52'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: legionio
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.9.50
4
+ version: 1.9.52
5
5
  platform: ruby
6
6
  authors:
7
7
  - Esity
@@ -434,27 +434,7 @@ files:
434
434
  - docker_deploy.rb
435
435
  - exe/legion
436
436
  - exe/legionio
437
- - exe/replay_ledger
438
437
  - exe/rollback_ledger
439
- - extensions-agentic/lex-consent/db/migrations/001_create_consent_maps.rb
440
- - extensions-agentic/lex-consent/lex-consent.gemspec
441
- - extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent.rb
442
- - extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/actors/tier_evaluation.rb
443
- - extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/models/consent_map.rb
444
- - extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/runners/consent.rb
445
- - extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent/version.rb
446
- - extensions-agentic/lex-reconciliation/Gemfile
447
- - extensions-agentic/lex-reconciliation/README.md
448
- - extensions-agentic/lex-reconciliation/lex-reconciliation.gemspec
449
- - extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation.rb
450
- - extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/actors/reconciliation_cycle.rb
451
- - extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/drift_log.rb
452
- - extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/runners/drift_checker.rb
453
- - extensions-agentic/lex-reconciliation/lib/legion/extensions/reconciliation/version.rb
454
- - extensions-agentic/lex-reconciliation/spec/legion/extensions/reconciliation/drift_log_spec.rb
455
- - extensions-agentic/lex-reconciliation/spec/legion/extensions/reconciliation/runners/drift_checker_spec.rb
456
- - extensions-agentic/lex-reconciliation/spec/legion/extensions/reconciliation_spec.rb
457
- - extensions-agentic/lex-reconciliation/spec/spec_helper.rb
458
438
  - integration/self_generate_spec.rb
459
439
  - legionio.gemspec
460
440
  - lib/legion.rb
data/exe/replay_ledger DELETED
@@ -1,395 +0,0 @@
1
- #!/usr/bin/env ruby
2
- # frozen_string_literal: true
3
-
4
- # One-off script to migrate local ledger data to prod database.
5
- # Handles FK remapping for identity tables by matching on natural keys.
6
- #
7
- # Usage:
8
- # LOCAL_DB_URL="postgres://localhost/legionio" PROD_DB_URL="postgres://user:pass@prod/legionio" bundle exec exe/replay_ledger
9
- #
10
- # Optional:
11
- # REPLAY_DRY_RUN=true (show counts, don't write)
12
-
13
- require 'sequel'
14
- require 'logger'
15
- require 'uri'
16
- require 'json'
17
- require 'fileutils'
18
-
19
- log = Logger.new($stdout)
20
- log.level = Logger::INFO
21
-
22
- # --- Signal handling for clean exit ---
23
- module MigrationState
24
- @shutdown = false
25
- class << self
26
- attr_accessor :shutdown
27
- end
28
- end
29
-
30
- %w[INT TERM HUP].each do |sig|
31
- Signal.trap(sig) do
32
- if MigrationState.shutdown
33
- warn "\nForce quit."
34
- exit!(1)
35
- end
36
- MigrationState.shutdown = true
37
- warn "\nShutdown requested — finishing current row, then saving manifests and exiting cleanly..."
38
- end
39
- end
40
-
41
- local_url = ENV.fetch('LOCAL_DB_URL', 'postgres://localhost/legionio')
42
- dry_run = ENV['REPLAY_DRY_RUN'] == 'true'
43
-
44
- # Read prod creds from ~/.legionio/settings/z_data_override.json
45
- prod_settings_path = File.expand_path('~/.legionio/settings/z_data_override.json')
46
- abort "Missing #{prod_settings_path} — create it with host/database/user/password" unless File.exist?(prod_settings_path)
47
- prod_config = JSON.parse(File.read(prod_settings_path))
48
- prod_creds = prod_config.dig('data', 'creds') || abort("No data.creds in #{prod_settings_path}")
49
- prod_url = "postgres://#{prod_creds['user']}:#{URI.encode_www_form_component(prod_creds['password'])}@#{prod_creds['host']}/#{prod_creds['database']}"
50
-
51
- LOCAL = Sequel.connect(local_url)
52
- PROD = Sequel.connect(prod_url)
53
-
54
- log.info "Local: #{local_url.sub(/:[^:@]+@/, ':***@')}"
55
- log.info "Prod: postgres://#{prod_creds['user']}:***@#{prod_creds['host']}/#{prod_creds['database']}"
56
- log.info "Mode: #{dry_run ? 'DRY RUN' : 'LIVE'}"
57
-
58
- # ============================================================
59
- # PHASE 1: Sync identity_providers (match on name)
60
- # ============================================================
61
-
62
- log.info '--- Phase 1: identity_providers ---'
63
- local_providers = LOCAL[:identity_providers].all
64
- prod_providers = PROD[:identity_providers].all
65
- prod_provider_by_name = prod_providers.to_h { |p| [p[:name], p] }
66
- provider_id_map = {} # local_id → prod_id
67
-
68
- local_providers.each do |lp|
69
- prod_match = prod_provider_by_name[lp[:name]]
70
- if prod_match
71
- provider_id_map[lp[:id]] = prod_match[:id]
72
- else
73
- row = lp.except(:id)
74
- if dry_run
75
- log.info " [DRY] Would insert provider: #{lp[:name]}"
76
- provider_id_map[lp[:id]] = -1
77
- else
78
- new_id = PROD[:identity_providers].insert(row)
79
- provider_id_map[lp[:id]] = new_id
80
- log.info " Inserted provider: #{lp[:name]} → id=#{new_id}"
81
- end
82
- end
83
- end
84
- log.info " Provider map: #{provider_id_map.size} entries (#{provider_id_map.count { |_, v| v != -1 }} matched)"
85
-
86
- # ============================================================
87
- # PHASE 2: Sync identity_principals (match on canonical_name)
88
- # ============================================================
89
-
90
- log.info '--- Phase 2: identity_principals ---'
91
- local_principals = LOCAL[:identity_principals].all
92
- prod_principals = PROD[:identity_principals].all
93
- prod_principal_by_name = prod_principals.to_h { |p| [p[:canonical_name], p] }
94
- principal_id_map = {} # local_id → prod_id
95
-
96
- local_principals.each do |lp|
97
- prod_match = prod_principal_by_name[lp[:canonical_name]]
98
- if prod_match
99
- principal_id_map[lp[:id]] = prod_match[:id]
100
- else
101
- row = lp.except(:id)
102
- if dry_run
103
- log.info " [DRY] Would insert principal: #{lp[:canonical_name]}"
104
- principal_id_map[lp[:id]] = -1
105
- else
106
- new_id = PROD[:identity_principals].insert(row)
107
- principal_id_map[lp[:id]] = new_id
108
- log.info " Inserted principal: #{lp[:canonical_name]} → id=#{new_id}"
109
- end
110
- end
111
- end
112
- log.info " Principal map: #{principal_id_map.size} entries (#{principal_id_map.count { |_, v| v != -1 }} matched)"
113
-
114
- # ============================================================
115
- # PHASE 3: Sync identities (match on principal_id + provider_identity_key)
116
- # ============================================================
117
-
118
- log.info '--- Phase 3: identities ---'
119
- local_identities = LOCAL[:identities].all
120
- prod_identities = PROD[:identities].all
121
- prod_identity_by_key = prod_identities.to_h { |i| ["#{i[:principal_id]}:#{i[:provider_identity_key]}", i] }
122
- identity_id_map = {} # local_id → prod_id
123
-
124
- local_identities.each do |li|
125
- mapped_principal = principal_id_map[li[:principal_id]]
126
- mapped_provider = provider_id_map[li[:provider_id]]
127
- prod_key = "#{mapped_principal}:#{li[:provider_identity_key]}"
128
- prod_match = prod_identity_by_key[prod_key]
129
-
130
- if prod_match
131
- identity_id_map[li[:id]] = prod_match[:id]
132
- else
133
- row = li.except(:id)
134
- row[:principal_id] = mapped_principal
135
- row[:provider_id] = mapped_provider
136
- if dry_run
137
- log.info " [DRY] Would insert identity: #{li[:provider_identity_key]} (principal=#{mapped_principal})"
138
- identity_id_map[li[:id]] = -1
139
- else
140
- new_id = PROD[:identities].insert(row)
141
- identity_id_map[li[:id]] = new_id
142
- log.info " Inserted identity: #{li[:provider_identity_key]} → id=#{new_id}"
143
- end
144
- end
145
- end
146
- log.info " Identity map: #{identity_id_map.size} entries (#{identity_id_map.count { |_, v| v != -1 }} matched)"
147
-
148
- # ============================================================
149
- # PHASE 4: LLM tables — remap identity FKs, preserve UUID links
150
- # ============================================================
151
-
152
- def remap_identity_columns(row, principal_map, identity_map)
153
- result = row.dup
154
- # Different tables use different column names
155
- %i[principal_id caller_principal_id].each do |col|
156
- result[col] = principal_map[result[col]] if result.key?(col) && result[col] && principal_map.key?(result[col])
157
- end
158
- %i[identity_id caller_identity_id].each do |col|
159
- result[col] = identity_map[result[col]] if result.key?(col) && result[col] && identity_map.key?(result[col])
160
- end
161
- result
162
- end
163
-
164
- MANIFEST_DIR = '/tmp/legion_migration_manifests'
165
-
166
- # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
167
- def migrate_table(local_db, prod_db, table, id_map_out, log:, dry_run:, fk_maps: {}, identity_maps: {})
168
- count = local_db[table].count
169
- if count.zero?
170
- log.info " #{table}: 0 rows, skipping"
171
- return
172
- end
173
-
174
- log.info " #{table}: #{count} rows to migrate..."
175
-
176
- # Check for UUID column to detect duplicates
177
- columns = local_db[table].columns
178
- has_uuid = columns.include?(:uuid)
179
- log.info " #{table}: loading existing UUIDs from prod..." if has_uuid
180
- prod_uuids = if has_uuid
181
- PROD[table].select_map(:uuid).to_set
182
- else
183
- Set.new
184
- end
185
- log.info " #{table}: #{prod_uuids.size} existing UUIDs on prod" if has_uuid
186
-
187
- FileUtils.mkdir_p(MANIFEST_DIR)
188
- manifest_path = File.join(MANIFEST_DIR, "#{table}.json")
189
- ids_path = File.join(MANIFEST_DIR, "#{table}_ids.jsonl")
190
-
191
- inserted = 0
192
- skipped = 0
193
- start_time = Time.now
194
-
195
- # Use block form to avoid file descriptor leaks
196
- File.open(ids_path, 'a') do |ids_file|
197
- local_db[table].order(:id).each do |row|
198
- if MigrationState.shutdown
199
- log.warn " #{table}: INTERRUPTED at row #{inserted + skipped}/#{count} — exiting cleanly"
200
- break
201
- end
202
-
203
- local_id = row[:id]
204
-
205
- # Skip if already exists on prod (by UUID)
206
- if has_uuid && row[:uuid] && prod_uuids.include?(row[:uuid])
207
- prod_row = prod_db[table].where(uuid: row[:uuid]).first
208
- id_map_out[local_id] = prod_row[:id] if prod_row
209
- skipped += 1
210
- next
211
- end
212
-
213
- new_row = row.except(:id)
214
-
215
- # Remap FK columns — if map is empty, NULL the column (deferred backfill)
216
- fk_maps.each do |column, map|
217
- next unless new_row.key?(column) && new_row[column]
218
-
219
- new_row[column] = if map.empty?
220
- nil
221
- elsif map.key?(new_row[column])
222
- map[new_row[column]]
223
- end
224
- end
225
-
226
- # Remap identity columns
227
- new_row = remap_identity_columns(new_row, identity_maps[:principals] || {}, identity_maps[:identities] || {}) unless identity_maps.empty?
228
-
229
- if dry_run
230
- inserted += 1
231
- id_map_out[local_id] = -1
232
- else
233
- begin
234
- new_id = prod_db[table].insert(new_row)
235
- id_map_out[local_id] = new_id
236
- ids_file.puts("#{local_id},#{new_id}")
237
- ids_file.flush
238
- inserted += 1
239
- rescue Sequel::ForeignKeyConstraintViolation => e
240
- log.warn " #{table}: FK violation on local_id=#{local_id} — #{e.message.lines.first.strip} — skipping"
241
- skipped += 1
242
- rescue Sequel::UniqueConstraintViolation => e
243
- log.warn " #{table}: duplicate on local_id=#{local_id} — #{e.message.lines.first.strip} — skipping"
244
- skipped += 1
245
- rescue Sequel::DatabaseError => e
246
- log.error " #{table}: DB error on local_id=#{local_id} — #{e.message.lines.first.strip} — skipping"
247
- skipped += 1
248
- end
249
- end
250
-
251
- # Progress logging every 100 rows
252
- next unless ((inserted + skipped) % 100).zero?
253
-
254
- elapsed = (Time.now - start_time).round(1)
255
- rate = ((inserted + skipped) / [elapsed, 0.1].max).round(1)
256
- log.info " #{table}: progress #{inserted + skipped}/#{count} (inserted=#{inserted} skipped=#{skipped}) #{elapsed}s #{rate} rows/s"
257
- end
258
- end
259
-
260
- # Write final summary manifest
261
- manifest = {
262
- table: table.to_s,
263
- started_at: start_time.iso8601,
264
- completed_at: Time.now.iso8601,
265
- inserted_count: inserted,
266
- skipped_count: skipped,
267
- interrupted: MigrationState.shutdown,
268
- ids_file: ids_path
269
- }
270
- File.write(manifest_path, JSON.pretty_generate(manifest))
271
-
272
- elapsed = (Time.now - start_time).round(1)
273
- log.info " #{table}: done inserted=#{inserted} skipped=#{skipped} elapsed=#{elapsed}s"
274
- log.info " #{table}: manifest → #{manifest_path}"
275
- log.info " #{table}: IDs → #{ids_path}"
276
- end
277
- # rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
278
-
279
- identity_maps = { principals: principal_id_map, identities: identity_id_map }
280
-
281
- # --- llm_conversations ---
282
- conversation_id_map = {}
283
- unless MigrationState.shutdown
284
- log.info '--- Phase 4: llm_conversations ---'
285
- migrate_table(LOCAL, PROD, :llm_conversations, conversation_id_map,
286
- identity_maps: identity_maps, log: log, dry_run: dry_run)
287
- end
288
-
289
- # --- llm_message_inference_requests ---
290
- request_id_map = {}
291
- unless MigrationState.shutdown
292
- log.info '--- Phase 5: llm_message_inference_requests ---'
293
- migrate_table(LOCAL, PROD, :llm_message_inference_requests, request_id_map,
294
- fk_maps: { conversation_id: conversation_id_map, latest_message_id: {} },
295
- identity_maps: identity_maps, log: log, dry_run: dry_run)
296
- end
297
-
298
- # --- llm_message_inference_responses ---
299
- response_id_map = {}
300
- unless MigrationState.shutdown
301
- log.info '--- Phase 6: llm_message_inference_responses ---'
302
- migrate_table(LOCAL, PROD, :llm_message_inference_responses, response_id_map,
303
- fk_maps: { message_inference_request_id: request_id_map, response_message_id: {} },
304
- identity_maps: identity_maps, log: log, dry_run: dry_run)
305
- end
306
-
307
- # --- llm_message_inference_metrics ---
308
- metrics_id_map = {}
309
- unless MigrationState.shutdown
310
- log.info '--- Phase 7: llm_message_inference_metrics ---'
311
- migrate_table(LOCAL, PROD, :llm_message_inference_metrics, metrics_id_map,
312
- fk_maps: { message_inference_request_id: request_id_map,
313
- message_inference_response_id: response_id_map },
314
- identity_maps: identity_maps, log: log, dry_run: dry_run)
315
- end
316
-
317
- # --- llm_messages ---
318
- message_id_map = {}
319
- unless MigrationState.shutdown
320
- log.info '--- Phase 8: llm_messages ---'
321
- migrate_table(LOCAL, PROD, :llm_messages, message_id_map,
322
- fk_maps: { conversation_id: conversation_id_map,
323
- message_inference_request_id: request_id_map,
324
- message_inference_response_id: response_id_map,
325
- parent_message_id: message_id_map },
326
- identity_maps: identity_maps, log: log, dry_run: dry_run)
327
- end
328
-
329
- # --- llm_tool_calls ---
330
- tool_call_id_map = {}
331
- unless MigrationState.shutdown
332
- log.info '--- Phase 9: llm_tool_calls ---'
333
- migrate_table(LOCAL, PROD, :llm_tool_calls, tool_call_id_map,
334
- fk_maps: { conversation_id: conversation_id_map,
335
- message_inference_response_id: response_id_map,
336
- requested_by_message_id: message_id_map,
337
- result_message_id: message_id_map },
338
- identity_maps: identity_maps, log: log, dry_run: dry_run)
339
- end
340
-
341
- # --- llm_tool_call_attempts ---
342
- tool_attempt_id_map = {}
343
- unless MigrationState.shutdown
344
- log.info '--- Phase 10: llm_tool_call_attempts ---'
345
- migrate_table(LOCAL, PROD, :llm_tool_call_attempts, tool_attempt_id_map,
346
- fk_maps: { tool_call_id: tool_call_id_map },
347
- identity_maps: identity_maps, log: log, dry_run: dry_run)
348
- end
349
-
350
- # --- Update latest_message_id and response_message_id now that messages exist ---
351
- unless dry_run || MigrationState.shutdown
352
- log.info '--- Phase 11: Backfill deferred FK columns ---'
353
-
354
- backfilled = 0
355
- LOCAL[:llm_message_inference_requests].where(Sequel.~(latest_message_id: nil)).each do |row|
356
- break if MigrationState.shutdown
357
-
358
- prod_req_id = request_id_map[row[:id]]
359
- prod_msg_id = message_id_map[row[:latest_message_id]]
360
- next unless prod_req_id && prod_msg_id
361
-
362
- PROD[:llm_message_inference_requests].where(id: prod_req_id).update(latest_message_id: prod_msg_id)
363
- backfilled += 1
364
- end
365
-
366
- LOCAL[:llm_message_inference_responses].where(Sequel.~(response_message_id: nil)).each do |row|
367
- break if MigrationState.shutdown
368
-
369
- prod_resp_id = response_id_map[row[:id]]
370
- prod_msg_id = message_id_map[row[:response_message_id]]
371
- next unless prod_resp_id && prod_msg_id
372
-
373
- PROD[:llm_message_inference_responses].where(id: prod_resp_id).update(response_message_id: prod_msg_id)
374
- backfilled += 1
375
- end
376
-
377
- log.info " Deferred FK backfill complete: #{backfilled} updates"
378
- end
379
-
380
- # --- Summary ---
381
- log.info '=== Migration Summary ==='
382
- log.info " Providers: #{provider_id_map.size}"
383
- log.info " Principals: #{principal_id_map.size}"
384
- log.info " Identities: #{identity_id_map.size}"
385
- log.info " Conversations: #{conversation_id_map.size}"
386
- log.info " Requests: #{request_id_map.size}"
387
- log.info " Responses: #{response_id_map.size}"
388
- log.info " Metrics: #{metrics_id_map.size}"
389
- log.info " Messages: #{message_id_map.size}"
390
- log.info " Tool calls: #{tool_call_id_map.size}"
391
- log.info " Tool attempts: #{tool_attempt_id_map.size}"
392
- log.info " Manifests: #{MANIFEST_DIR}/"
393
- log.info dry_run ? '[DRY RUN COMPLETE]' : '[MIGRATION COMPLETE]'
394
-
395
- log.info ' To rollback: bundle exec exe/rollback_ledger'
@@ -1,24 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- Sequel.migration do
4
- change do
5
- create_table(:consent_maps) do
6
- primary_key :id
7
- String :worker_id, null: false
8
- String :from_tier, null: false
9
- String :to_tier, null: false
10
- String :requested_by, null: false
11
- String :state, null: false, default: 'pending_approval'
12
- String :resolved_by
13
- Time :resolved_at
14
- String :notes, text: true
15
- String :context, text: true
16
- Time :created_at
17
- Time :updated_at
18
-
19
- index :worker_id
20
- index :state
21
- index %i[worker_id state]
22
- end
23
- end
24
- end