legionio 1.9.39 → 1.9.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/exe/replay_ledger +395 -0
- data/exe/rollback_ledger +91 -0
- data/lib/legion/cli/setup_command.rb +135 -15
- data/lib/legion/trace_search.rb +1 -1
- data/lib/legion/version.rb +1 -1
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 13eae4c67a25af55b4f7b02423e493a1c102831f204a34368ea16e04f529d5ff
|
|
4
|
+
data.tar.gz: 44219061da324461fc19a69fc23b7ad1108fd29f696f27133e15b47e3de98e33
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a92ef53aba4416fb0a777fa0be7c4a8b822f1c102cbe45b0fa6d46b7c052db1501fa7e9b05922c3e9188ef589c15f202632b2b4efc9a53476cea6843c62cfa4d
|
|
7
|
+
data.tar.gz: dc10e34337589079aae562d8db6557c95e9c43b246b8a9f66c05d7b153413947d06bd28f26603aea95ba95987db9ba214b7ac97de40701d2b75b6190032fbcb0
|
data/CHANGELOG.md
CHANGED
data/exe/replay_ledger
ADDED
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# One-off script to migrate local ledger data to prod database.
|
|
5
|
+
# Handles FK remapping for identity tables by matching on natural keys.
|
|
6
|
+
#
|
|
7
|
+
# Usage:
|
|
8
|
+
# LOCAL_DB_URL="postgres://localhost/legionio" PROD_DB_URL="postgres://user:pass@prod/legionio" bundle exec exe/replay_ledger
|
|
9
|
+
#
|
|
10
|
+
# Optional:
|
|
11
|
+
# REPLAY_DRY_RUN=true (show counts, don't write)
|
|
12
|
+
|
|
13
|
+
require 'sequel'
|
|
14
|
+
require 'logger'
|
|
15
|
+
require 'uri'
|
|
16
|
+
require 'json'
|
|
17
|
+
require 'fileutils'
|
|
18
|
+
|
|
19
|
+
log = Logger.new($stdout)
|
|
20
|
+
log.level = Logger::INFO
|
|
21
|
+
|
|
22
|
+
# --- Signal handling for clean exit ---
|
|
23
|
+
module MigrationState
|
|
24
|
+
@shutdown = false
|
|
25
|
+
class << self
|
|
26
|
+
attr_accessor :shutdown
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
%w[INT TERM HUP].each do |sig|
|
|
31
|
+
Signal.trap(sig) do
|
|
32
|
+
if MigrationState.shutdown
|
|
33
|
+
warn "\nForce quit."
|
|
34
|
+
exit!(1)
|
|
35
|
+
end
|
|
36
|
+
MigrationState.shutdown = true
|
|
37
|
+
warn "\nShutdown requested — finishing current row, then saving manifests and exiting cleanly..."
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
local_url = ENV.fetch('LOCAL_DB_URL', 'postgres://localhost/legionio')
|
|
42
|
+
dry_run = ENV['REPLAY_DRY_RUN'] == 'true'
|
|
43
|
+
|
|
44
|
+
# Read prod creds from ~/.legionio/settings/z_data_override.json
|
|
45
|
+
prod_settings_path = File.expand_path('~/.legionio/settings/z_data_override.json')
|
|
46
|
+
abort "Missing #{prod_settings_path} — create it with host/database/user/password" unless File.exist?(prod_settings_path)
|
|
47
|
+
prod_config = JSON.parse(File.read(prod_settings_path))
|
|
48
|
+
prod_creds = prod_config.dig('data', 'creds') || abort("No data.creds in #{prod_settings_path}")
|
|
49
|
+
prod_url = "postgres://#{prod_creds['user']}:#{URI.encode_www_form_component(prod_creds['password'])}@#{prod_creds['host']}/#{prod_creds['database']}"
|
|
50
|
+
|
|
51
|
+
LOCAL = Sequel.connect(local_url)
|
|
52
|
+
PROD = Sequel.connect(prod_url)
|
|
53
|
+
|
|
54
|
+
log.info "Local: #{local_url.sub(/:[^:@]+@/, ':***@')}"
|
|
55
|
+
log.info "Prod: postgres://#{prod_creds['user']}:***@#{prod_creds['host']}/#{prod_creds['database']}"
|
|
56
|
+
log.info "Mode: #{dry_run ? 'DRY RUN' : 'LIVE'}"
|
|
57
|
+
|
|
58
|
+
# ============================================================
|
|
59
|
+
# PHASE 1: Sync identity_providers (match on name)
|
|
60
|
+
# ============================================================
|
|
61
|
+
|
|
62
|
+
log.info '--- Phase 1: identity_providers ---'
|
|
63
|
+
local_providers = LOCAL[:identity_providers].all
|
|
64
|
+
prod_providers = PROD[:identity_providers].all
|
|
65
|
+
prod_provider_by_name = prod_providers.to_h { |p| [p[:name], p] }
|
|
66
|
+
provider_id_map = {} # local_id → prod_id
|
|
67
|
+
|
|
68
|
+
local_providers.each do |lp|
|
|
69
|
+
prod_match = prod_provider_by_name[lp[:name]]
|
|
70
|
+
if prod_match
|
|
71
|
+
provider_id_map[lp[:id]] = prod_match[:id]
|
|
72
|
+
else
|
|
73
|
+
row = lp.except(:id)
|
|
74
|
+
if dry_run
|
|
75
|
+
log.info " [DRY] Would insert provider: #{lp[:name]}"
|
|
76
|
+
provider_id_map[lp[:id]] = -1
|
|
77
|
+
else
|
|
78
|
+
new_id = PROD[:identity_providers].insert(row)
|
|
79
|
+
provider_id_map[lp[:id]] = new_id
|
|
80
|
+
log.info " Inserted provider: #{lp[:name]} → id=#{new_id}"
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
log.info " Provider map: #{provider_id_map.size} entries (#{provider_id_map.count { |_, v| v != -1 }} matched)"
|
|
85
|
+
|
|
86
|
+
# ============================================================
|
|
87
|
+
# PHASE 2: Sync identity_principals (match on canonical_name)
|
|
88
|
+
# ============================================================
|
|
89
|
+
|
|
90
|
+
log.info '--- Phase 2: identity_principals ---'
|
|
91
|
+
local_principals = LOCAL[:identity_principals].all
|
|
92
|
+
prod_principals = PROD[:identity_principals].all
|
|
93
|
+
prod_principal_by_name = prod_principals.to_h { |p| [p[:canonical_name], p] }
|
|
94
|
+
principal_id_map = {} # local_id → prod_id
|
|
95
|
+
|
|
96
|
+
local_principals.each do |lp|
|
|
97
|
+
prod_match = prod_principal_by_name[lp[:canonical_name]]
|
|
98
|
+
if prod_match
|
|
99
|
+
principal_id_map[lp[:id]] = prod_match[:id]
|
|
100
|
+
else
|
|
101
|
+
row = lp.except(:id)
|
|
102
|
+
if dry_run
|
|
103
|
+
log.info " [DRY] Would insert principal: #{lp[:canonical_name]}"
|
|
104
|
+
principal_id_map[lp[:id]] = -1
|
|
105
|
+
else
|
|
106
|
+
new_id = PROD[:identity_principals].insert(row)
|
|
107
|
+
principal_id_map[lp[:id]] = new_id
|
|
108
|
+
log.info " Inserted principal: #{lp[:canonical_name]} → id=#{new_id}"
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
log.info " Principal map: #{principal_id_map.size} entries (#{principal_id_map.count { |_, v| v != -1 }} matched)"
|
|
113
|
+
|
|
114
|
+
# ============================================================
|
|
115
|
+
# PHASE 3: Sync identities (match on principal_id + provider_identity_key)
|
|
116
|
+
# ============================================================
|
|
117
|
+
|
|
118
|
+
log.info '--- Phase 3: identities ---'
|
|
119
|
+
local_identities = LOCAL[:identities].all
|
|
120
|
+
prod_identities = PROD[:identities].all
|
|
121
|
+
prod_identity_by_key = prod_identities.to_h { |i| ["#{i[:principal_id]}:#{i[:provider_identity_key]}", i] }
|
|
122
|
+
identity_id_map = {} # local_id → prod_id
|
|
123
|
+
|
|
124
|
+
local_identities.each do |li|
|
|
125
|
+
mapped_principal = principal_id_map[li[:principal_id]]
|
|
126
|
+
mapped_provider = provider_id_map[li[:provider_id]]
|
|
127
|
+
prod_key = "#{mapped_principal}:#{li[:provider_identity_key]}"
|
|
128
|
+
prod_match = prod_identity_by_key[prod_key]
|
|
129
|
+
|
|
130
|
+
if prod_match
|
|
131
|
+
identity_id_map[li[:id]] = prod_match[:id]
|
|
132
|
+
else
|
|
133
|
+
row = li.except(:id)
|
|
134
|
+
row[:principal_id] = mapped_principal
|
|
135
|
+
row[:provider_id] = mapped_provider
|
|
136
|
+
if dry_run
|
|
137
|
+
log.info " [DRY] Would insert identity: #{li[:provider_identity_key]} (principal=#{mapped_principal})"
|
|
138
|
+
identity_id_map[li[:id]] = -1
|
|
139
|
+
else
|
|
140
|
+
new_id = PROD[:identities].insert(row)
|
|
141
|
+
identity_id_map[li[:id]] = new_id
|
|
142
|
+
log.info " Inserted identity: #{li[:provider_identity_key]} → id=#{new_id}"
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
log.info " Identity map: #{identity_id_map.size} entries (#{identity_id_map.count { |_, v| v != -1 }} matched)"
|
|
147
|
+
|
|
148
|
+
# ============================================================
|
|
149
|
+
# PHASE 4: LLM tables — remap identity FKs, preserve UUID links
|
|
150
|
+
# ============================================================
|
|
151
|
+
|
|
152
|
+
def remap_identity_columns(row, principal_map, identity_map)
|
|
153
|
+
result = row.dup
|
|
154
|
+
# Different tables use different column names
|
|
155
|
+
%i[principal_id caller_principal_id].each do |col|
|
|
156
|
+
result[col] = principal_map[result[col]] if result.key?(col) && result[col] && principal_map.key?(result[col])
|
|
157
|
+
end
|
|
158
|
+
%i[identity_id caller_identity_id].each do |col|
|
|
159
|
+
result[col] = identity_map[result[col]] if result.key?(col) && result[col] && identity_map.key?(result[col])
|
|
160
|
+
end
|
|
161
|
+
result
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
MANIFEST_DIR = '/tmp/legion_migration_manifests'
|
|
165
|
+
|
|
166
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity, Metrics/ParameterLists
|
|
167
|
+
def migrate_table(local_db, prod_db, table, id_map_out, log:, dry_run:, fk_maps: {}, identity_maps: {})
|
|
168
|
+
count = local_db[table].count
|
|
169
|
+
if count.zero?
|
|
170
|
+
log.info " #{table}: 0 rows, skipping"
|
|
171
|
+
return
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
log.info " #{table}: #{count} rows to migrate..."
|
|
175
|
+
|
|
176
|
+
# Check for UUID column to detect duplicates
|
|
177
|
+
columns = local_db[table].columns
|
|
178
|
+
has_uuid = columns.include?(:uuid)
|
|
179
|
+
log.info " #{table}: loading existing UUIDs from prod..." if has_uuid
|
|
180
|
+
prod_uuids = if has_uuid
|
|
181
|
+
PROD[table].select_map(:uuid).to_set
|
|
182
|
+
else
|
|
183
|
+
Set.new
|
|
184
|
+
end
|
|
185
|
+
log.info " #{table}: #{prod_uuids.size} existing UUIDs on prod" if has_uuid
|
|
186
|
+
|
|
187
|
+
FileUtils.mkdir_p(MANIFEST_DIR)
|
|
188
|
+
manifest_path = File.join(MANIFEST_DIR, "#{table}.json")
|
|
189
|
+
ids_path = File.join(MANIFEST_DIR, "#{table}_ids.jsonl")
|
|
190
|
+
|
|
191
|
+
inserted = 0
|
|
192
|
+
skipped = 0
|
|
193
|
+
start_time = Time.now
|
|
194
|
+
|
|
195
|
+
# Use block form to avoid file descriptor leaks
|
|
196
|
+
File.open(ids_path, 'a') do |ids_file|
|
|
197
|
+
local_db[table].order(:id).each do |row|
|
|
198
|
+
if MigrationState.shutdown
|
|
199
|
+
log.warn " #{table}: INTERRUPTED at row #{inserted + skipped}/#{count} — exiting cleanly"
|
|
200
|
+
break
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
local_id = row[:id]
|
|
204
|
+
|
|
205
|
+
# Skip if already exists on prod (by UUID)
|
|
206
|
+
if has_uuid && row[:uuid] && prod_uuids.include?(row[:uuid])
|
|
207
|
+
prod_row = prod_db[table].where(uuid: row[:uuid]).first
|
|
208
|
+
id_map_out[local_id] = prod_row[:id] if prod_row
|
|
209
|
+
skipped += 1
|
|
210
|
+
next
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
new_row = row.except(:id)
|
|
214
|
+
|
|
215
|
+
# Remap FK columns — if map is empty, NULL the column (deferred backfill)
|
|
216
|
+
fk_maps.each do |column, map|
|
|
217
|
+
next unless new_row.key?(column) && new_row[column]
|
|
218
|
+
|
|
219
|
+
new_row[column] = if map.empty?
|
|
220
|
+
nil
|
|
221
|
+
elsif map.key?(new_row[column])
|
|
222
|
+
map[new_row[column]]
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# Remap identity columns
|
|
227
|
+
new_row = remap_identity_columns(new_row, identity_maps[:principals] || {}, identity_maps[:identities] || {}) unless identity_maps.empty?
|
|
228
|
+
|
|
229
|
+
if dry_run
|
|
230
|
+
inserted += 1
|
|
231
|
+
id_map_out[local_id] = -1
|
|
232
|
+
else
|
|
233
|
+
begin
|
|
234
|
+
new_id = prod_db[table].insert(new_row)
|
|
235
|
+
id_map_out[local_id] = new_id
|
|
236
|
+
ids_file.puts("#{local_id},#{new_id}")
|
|
237
|
+
ids_file.flush
|
|
238
|
+
inserted += 1
|
|
239
|
+
rescue Sequel::ForeignKeyConstraintViolation => e
|
|
240
|
+
log.warn " #{table}: FK violation on local_id=#{local_id} — #{e.message.lines.first.strip} — skipping"
|
|
241
|
+
skipped += 1
|
|
242
|
+
rescue Sequel::UniqueConstraintViolation => e
|
|
243
|
+
log.warn " #{table}: duplicate on local_id=#{local_id} — #{e.message.lines.first.strip} — skipping"
|
|
244
|
+
skipped += 1
|
|
245
|
+
rescue Sequel::DatabaseError => e
|
|
246
|
+
log.error " #{table}: DB error on local_id=#{local_id} — #{e.message.lines.first.strip} — skipping"
|
|
247
|
+
skipped += 1
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# Progress logging every 100 rows
|
|
252
|
+
next unless ((inserted + skipped) % 100).zero?
|
|
253
|
+
|
|
254
|
+
elapsed = (Time.now - start_time).round(1)
|
|
255
|
+
rate = ((inserted + skipped) / [elapsed, 0.1].max).round(1)
|
|
256
|
+
log.info " #{table}: progress #{inserted + skipped}/#{count} (inserted=#{inserted} skipped=#{skipped}) #{elapsed}s #{rate} rows/s"
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# Write final summary manifest
|
|
261
|
+
manifest = {
|
|
262
|
+
table: table.to_s,
|
|
263
|
+
started_at: start_time.iso8601,
|
|
264
|
+
completed_at: Time.now.iso8601,
|
|
265
|
+
inserted_count: inserted,
|
|
266
|
+
skipped_count: skipped,
|
|
267
|
+
interrupted: MigrationState.shutdown,
|
|
268
|
+
ids_file: ids_path
|
|
269
|
+
}
|
|
270
|
+
File.write(manifest_path, JSON.pretty_generate(manifest))
|
|
271
|
+
|
|
272
|
+
elapsed = (Time.now - start_time).round(1)
|
|
273
|
+
log.info " #{table}: done inserted=#{inserted} skipped=#{skipped} elapsed=#{elapsed}s"
|
|
274
|
+
log.info " #{table}: manifest → #{manifest_path}"
|
|
275
|
+
log.info " #{table}: IDs → #{ids_path}"
|
|
276
|
+
end
|
|
277
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity, Metrics/ParameterLists
|
|
278
|
+
|
|
279
|
+
identity_maps = { principals: principal_id_map, identities: identity_id_map }
|
|
280
|
+
|
|
281
|
+
# --- llm_conversations ---
|
|
282
|
+
conversation_id_map = {}
|
|
283
|
+
unless MigrationState.shutdown
|
|
284
|
+
log.info '--- Phase 4: llm_conversations ---'
|
|
285
|
+
migrate_table(LOCAL, PROD, :llm_conversations, conversation_id_map,
|
|
286
|
+
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# --- llm_message_inference_requests ---
|
|
290
|
+
request_id_map = {}
|
|
291
|
+
unless MigrationState.shutdown
|
|
292
|
+
log.info '--- Phase 5: llm_message_inference_requests ---'
|
|
293
|
+
migrate_table(LOCAL, PROD, :llm_message_inference_requests, request_id_map,
|
|
294
|
+
fk_maps: { conversation_id: conversation_id_map, latest_message_id: {} },
|
|
295
|
+
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
# --- llm_message_inference_responses ---
|
|
299
|
+
response_id_map = {}
|
|
300
|
+
unless MigrationState.shutdown
|
|
301
|
+
log.info '--- Phase 6: llm_message_inference_responses ---'
|
|
302
|
+
migrate_table(LOCAL, PROD, :llm_message_inference_responses, response_id_map,
|
|
303
|
+
fk_maps: { message_inference_request_id: request_id_map, response_message_id: {} },
|
|
304
|
+
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
# --- llm_message_inference_metrics ---
|
|
308
|
+
metrics_id_map = {}
|
|
309
|
+
unless MigrationState.shutdown
|
|
310
|
+
log.info '--- Phase 7: llm_message_inference_metrics ---'
|
|
311
|
+
migrate_table(LOCAL, PROD, :llm_message_inference_metrics, metrics_id_map,
|
|
312
|
+
fk_maps: { message_inference_request_id: request_id_map,
|
|
313
|
+
message_inference_response_id: response_id_map },
|
|
314
|
+
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# --- llm_messages ---
|
|
318
|
+
message_id_map = {}
|
|
319
|
+
unless MigrationState.shutdown
|
|
320
|
+
log.info '--- Phase 8: llm_messages ---'
|
|
321
|
+
migrate_table(LOCAL, PROD, :llm_messages, message_id_map,
|
|
322
|
+
fk_maps: { conversation_id: conversation_id_map,
|
|
323
|
+
message_inference_request_id: request_id_map,
|
|
324
|
+
message_inference_response_id: response_id_map,
|
|
325
|
+
parent_message_id: message_id_map },
|
|
326
|
+
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
# --- llm_tool_calls ---
|
|
330
|
+
tool_call_id_map = {}
|
|
331
|
+
unless MigrationState.shutdown
|
|
332
|
+
log.info '--- Phase 9: llm_tool_calls ---'
|
|
333
|
+
migrate_table(LOCAL, PROD, :llm_tool_calls, tool_call_id_map,
|
|
334
|
+
fk_maps: { conversation_id: conversation_id_map,
|
|
335
|
+
message_inference_response_id: response_id_map,
|
|
336
|
+
requested_by_message_id: message_id_map,
|
|
337
|
+
result_message_id: message_id_map },
|
|
338
|
+
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
# --- llm_tool_call_attempts ---
|
|
342
|
+
tool_attempt_id_map = {}
|
|
343
|
+
unless MigrationState.shutdown
|
|
344
|
+
log.info '--- Phase 10: llm_tool_call_attempts ---'
|
|
345
|
+
migrate_table(LOCAL, PROD, :llm_tool_call_attempts, tool_attempt_id_map,
|
|
346
|
+
fk_maps: { tool_call_id: tool_call_id_map },
|
|
347
|
+
identity_maps: identity_maps, log: log, dry_run: dry_run)
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
# --- Update latest_message_id and response_message_id now that messages exist ---
|
|
351
|
+
unless dry_run || MigrationState.shutdown
|
|
352
|
+
log.info '--- Phase 11: Backfill deferred FK columns ---'
|
|
353
|
+
|
|
354
|
+
backfilled = 0
|
|
355
|
+
LOCAL[:llm_message_inference_requests].where(Sequel.~(latest_message_id: nil)).each do |row|
|
|
356
|
+
break if MigrationState.shutdown
|
|
357
|
+
|
|
358
|
+
prod_req_id = request_id_map[row[:id]]
|
|
359
|
+
prod_msg_id = message_id_map[row[:latest_message_id]]
|
|
360
|
+
next unless prod_req_id && prod_msg_id
|
|
361
|
+
|
|
362
|
+
PROD[:llm_message_inference_requests].where(id: prod_req_id).update(latest_message_id: prod_msg_id)
|
|
363
|
+
backfilled += 1
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
LOCAL[:llm_message_inference_responses].where(Sequel.~(response_message_id: nil)).each do |row|
|
|
367
|
+
break if MigrationState.shutdown
|
|
368
|
+
|
|
369
|
+
prod_resp_id = response_id_map[row[:id]]
|
|
370
|
+
prod_msg_id = message_id_map[row[:response_message_id]]
|
|
371
|
+
next unless prod_resp_id && prod_msg_id
|
|
372
|
+
|
|
373
|
+
PROD[:llm_message_inference_responses].where(id: prod_resp_id).update(response_message_id: prod_msg_id)
|
|
374
|
+
backfilled += 1
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
log.info " Deferred FK backfill complete: #{backfilled} updates"
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
# --- Summary ---
|
|
381
|
+
log.info '=== Migration Summary ==='
|
|
382
|
+
log.info " Providers: #{provider_id_map.size}"
|
|
383
|
+
log.info " Principals: #{principal_id_map.size}"
|
|
384
|
+
log.info " Identities: #{identity_id_map.size}"
|
|
385
|
+
log.info " Conversations: #{conversation_id_map.size}"
|
|
386
|
+
log.info " Requests: #{request_id_map.size}"
|
|
387
|
+
log.info " Responses: #{response_id_map.size}"
|
|
388
|
+
log.info " Metrics: #{metrics_id_map.size}"
|
|
389
|
+
log.info " Messages: #{message_id_map.size}"
|
|
390
|
+
log.info " Tool calls: #{tool_call_id_map.size}"
|
|
391
|
+
log.info " Tool attempts: #{tool_attempt_id_map.size}"
|
|
392
|
+
log.info " Manifests: #{MANIFEST_DIR}/"
|
|
393
|
+
log.info dry_run ? '[DRY RUN COMPLETE]' : '[MIGRATION COMPLETE]'
|
|
394
|
+
|
|
395
|
+
log.info ' To rollback: bundle exec exe/rollback_ledger'
|
data/exe/rollback_ledger
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# Rollback a ledger migration using the manifests generated by replay_ledger.
|
|
5
|
+
#
|
|
6
|
+
# Usage:
|
|
7
|
+
# bundle exec exe/rollback_ledger
|
|
8
|
+
# ROLLBACK_DRY_RUN=true bundle exec exe/rollback_ledger (show what would be deleted)
|
|
9
|
+
|
|
10
|
+
require 'sequel'
|
|
11
|
+
require 'logger'
|
|
12
|
+
require 'uri'
|
|
13
|
+
require 'json'
|
|
14
|
+
|
|
15
|
+
log = Logger.new($stdout)
|
|
16
|
+
log.level = Logger::INFO
|
|
17
|
+
|
|
18
|
+
dry_run = ENV['ROLLBACK_DRY_RUN'] == 'true'
|
|
19
|
+
manifest_dir = ENV.fetch('MANIFEST_DIR', '/tmp/legion_migration_manifests')
|
|
20
|
+
|
|
21
|
+
abort "No manifest directory at #{manifest_dir} — nothing to rollback" unless File.directory?(manifest_dir)
|
|
22
|
+
|
|
23
|
+
# Read prod creds
|
|
24
|
+
prod_settings_path = File.expand_path('~/.legionio/settings/z_data_override.json')
|
|
25
|
+
abort "Missing #{prod_settings_path}" unless File.exist?(prod_settings_path)
|
|
26
|
+
prod_config = JSON.parse(File.read(prod_settings_path))
|
|
27
|
+
prod_creds = prod_config.dig('data', 'creds') || abort("No data.creds in #{prod_settings_path}")
|
|
28
|
+
prod_url = "postgres://#{prod_creds['user']}:#{URI.encode_www_form_component(prod_creds['password'])}@#{prod_creds['host']}/#{prod_creds['database']}"
|
|
29
|
+
|
|
30
|
+
PROD = Sequel.connect(prod_url)
|
|
31
|
+
log.info "Prod: postgres://#{prod_creds['user']}:***@#{prod_creds['host']}/#{prod_creds['database']}"
|
|
32
|
+
log.info "Mode: #{dry_run ? 'DRY RUN' : 'LIVE ROLLBACK'}"
|
|
33
|
+
log.info "Manifests: #{manifest_dir}"
|
|
34
|
+
|
|
35
|
+
# Delete in reverse FK order
|
|
36
|
+
TABLES_REVERSE = %w[
|
|
37
|
+
llm_tool_call_attempts
|
|
38
|
+
llm_tool_calls
|
|
39
|
+
llm_messages
|
|
40
|
+
llm_message_inference_metrics
|
|
41
|
+
llm_message_inference_responses
|
|
42
|
+
llm_message_inference_requests
|
|
43
|
+
llm_conversations
|
|
44
|
+
].freeze
|
|
45
|
+
|
|
46
|
+
total_deleted = 0
|
|
47
|
+
|
|
48
|
+
TABLES_REVERSE.each do |table|
|
|
49
|
+
ids_file = File.join(manifest_dir, "#{table}_ids.jsonl")
|
|
50
|
+
unless File.exist?(ids_file)
|
|
51
|
+
log.info " #{table}: no IDs file, skipping"
|
|
52
|
+
next
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Read prod IDs (second column in "local_id,prod_id" format)
|
|
56
|
+
ids = File.readlines(ids_file, chomp: true).filter_map do |line|
|
|
57
|
+
next if line.strip.empty?
|
|
58
|
+
|
|
59
|
+
line.split(',')[1]&.to_i
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
if ids.empty?
|
|
63
|
+
log.info " #{table}: 0 inserted IDs, skipping"
|
|
64
|
+
next
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
log.info " #{table}: #{ids.size} rows to delete..."
|
|
68
|
+
|
|
69
|
+
if dry_run
|
|
70
|
+
log.info " [DRY] Would delete #{ids.size} rows from #{table} (first 5 IDs: #{ids.first(5).join(',')})"
|
|
71
|
+
else
|
|
72
|
+
ids.each_slice(500) do |batch|
|
|
73
|
+
deleted = PROD[table.to_sym].where(id: batch).delete
|
|
74
|
+
total_deleted += deleted
|
|
75
|
+
end
|
|
76
|
+
log.info " #{table}: deleted #{ids.size} rows"
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Also handle deferred FK backfills (latest_message_id, response_message_id)
|
|
81
|
+
# These were UPDATEs, not INSERTs — set them back to NULL
|
|
82
|
+
unless dry_run
|
|
83
|
+
requests_manifest = File.join(manifest_dir, 'llm_message_inference_requests.json')
|
|
84
|
+
if File.exist?(requests_manifest)
|
|
85
|
+
JSON.parse(File.read(requests_manifest))['inserted_ids'] || []
|
|
86
|
+
# These were inserted rows that got deleted above, so no backfill undo needed
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
log.info "=== Rollback #{dry_run ? 'Preview' : 'Complete'} ==="
|
|
91
|
+
log.info " Total deleted: #{dry_run ? '(dry run)' : total_deleted}"
|
|
@@ -166,11 +166,16 @@ module Legion
|
|
|
166
166
|
written = []
|
|
167
167
|
skipped = []
|
|
168
168
|
|
|
169
|
+
llm_installed, = partition_gems(PACKS[:llm][:gems])
|
|
170
|
+
out.warn('LLM pack not installed. Run: legionio setup llm') if llm_installed.empty? && !options[:json]
|
|
171
|
+
|
|
169
172
|
write_codex_config(base_url, written, skipped)
|
|
170
|
-
write_claude_code_proxy_config(base_url, written, skipped)
|
|
173
|
+
# write_claude_code_proxy_config(base_url, written, skipped) # too destructive for enterprise users
|
|
174
|
+
write_zsh_legionio(base_url, written, skipped)
|
|
175
|
+
write_pack_marker(:'proxy-mode')
|
|
171
176
|
|
|
172
177
|
if options[:json]
|
|
173
|
-
out.json(written: written, skipped: skipped, base_url: base_url)
|
|
178
|
+
out.json(written: written, skipped: skipped, base_url: base_url, profile: 'legionio')
|
|
174
179
|
else
|
|
175
180
|
out.spacer
|
|
176
181
|
out.success("LegionIO proxy mode configured (#{written.size} written, #{skipped.size} skipped)")
|
|
@@ -178,8 +183,13 @@ module Legion
|
|
|
178
183
|
skipped.each { |f| puts " Skipped (already exists, use --force to overwrite): #{f}" }
|
|
179
184
|
out.spacer
|
|
180
185
|
puts " LegionIO API: #{base_url.sub('/v1', '')}"
|
|
181
|
-
puts ' Codex CLI:
|
|
186
|
+
puts ' Codex CLI: codex --profile legionio'
|
|
182
187
|
puts ' Claude Code: set ANTHROPIC_BASE_URL in your shell or ~/.claude/settings.json'
|
|
188
|
+
if written.any? { |f| f.end_with?('.zsh_legionio') }
|
|
189
|
+
out.spacer
|
|
190
|
+
puts ' To activate shell functions in this session, run:'
|
|
191
|
+
puts ' source ~/.zsh_legionio'
|
|
192
|
+
end
|
|
183
193
|
out.spacer
|
|
184
194
|
end
|
|
185
195
|
end
|
|
@@ -313,6 +323,7 @@ module Legion
|
|
|
313
323
|
end
|
|
314
324
|
|
|
315
325
|
write_python_marker(python3, packages)
|
|
326
|
+
write_pack_marker(:python)
|
|
316
327
|
|
|
317
328
|
if options[:json]
|
|
318
329
|
out.json(venv: PYTHON_VENV_DIR, python: python_version(python3), results: results)
|
|
@@ -339,8 +350,15 @@ module Legion
|
|
|
339
350
|
missing: missing }
|
|
340
351
|
end
|
|
341
352
|
|
|
353
|
+
python_status = { name: :python, description: 'Python venv + document/data packages',
|
|
354
|
+
installed: File.exist?(PYTHON_MARKER), missing: [] }
|
|
355
|
+
proxy_status = { name: :'proxy-mode', description: 'Codex CLI + shell helper functions for LegionIO proxy',
|
|
356
|
+
installed: File.exist?(File.expand_path('~/.legionio/.packs/proxy-mode')), missing: [] }
|
|
357
|
+
|
|
342
358
|
if options[:json]
|
|
343
|
-
out.json(packs:
|
|
359
|
+
out.json(packs: pack_statuses,
|
|
360
|
+
python: python_status.slice(:name, :description, :installed),
|
|
361
|
+
proxy_mode: proxy_status.slice(:name, :description, :installed))
|
|
344
362
|
else
|
|
345
363
|
out.header('Feature Packs')
|
|
346
364
|
out.spacer
|
|
@@ -355,6 +373,10 @@ module Legion
|
|
|
355
373
|
puts " #{out.colorize(g, :muted)} (missing)"
|
|
356
374
|
end
|
|
357
375
|
end
|
|
376
|
+
[python_status, proxy_status].each do |ps|
|
|
377
|
+
icon = ps[:installed] ? out.colorize('installed', :success) : out.colorize('not installed', :muted)
|
|
378
|
+
puts " #{out.colorize(ps[:name].to_s.ljust(12), :label)} #{icon} #{ps[:description]}"
|
|
379
|
+
end
|
|
358
380
|
out.spacer
|
|
359
381
|
end
|
|
360
382
|
end
|
|
@@ -742,31 +764,129 @@ module Legion
|
|
|
742
764
|
end
|
|
743
765
|
|
|
744
766
|
def write_codex_config(base_url, written, skipped)
|
|
745
|
-
codex_dir
|
|
746
|
-
|
|
767
|
+
codex_dir = File.expand_path('~/.codex')
|
|
768
|
+
FileUtils.mkdir_p(codex_dir)
|
|
769
|
+
|
|
770
|
+
write_codex_profile(codex_dir, base_url, written, skipped)
|
|
771
|
+
write_codex_catalog(codex_dir, written, skipped)
|
|
772
|
+
write_codex_main_config(codex_dir, base_url, written, skipped)
|
|
773
|
+
end
|
|
747
774
|
|
|
748
|
-
|
|
749
|
-
|
|
775
|
+
def write_codex_profile(codex_dir, base_url, written, skipped)
|
|
776
|
+
profile_path = File.join(codex_dir, 'legionio.config.toml')
|
|
777
|
+
|
|
778
|
+
if File.exist?(profile_path) && !options[:force]
|
|
779
|
+
skipped << profile_path
|
|
750
780
|
return
|
|
751
781
|
end
|
|
752
782
|
|
|
753
|
-
|
|
754
|
-
|
|
783
|
+
catalog_path = File.join(codex_dir, 'legionio-catalog.json')
|
|
755
784
|
content = <<~TOML
|
|
756
785
|
model = "legionio"
|
|
757
|
-
model_provider = "
|
|
786
|
+
model_provider = "legionio"
|
|
787
|
+
model_catalog_json = "#{catalog_path}"
|
|
758
788
|
|
|
759
|
-
[model_providers.
|
|
789
|
+
[model_providers.legionio]
|
|
760
790
|
name = "LegionIO"
|
|
761
791
|
env_key = "LEGION_API_KEY"
|
|
762
792
|
base_url = "#{base_url}"
|
|
763
793
|
wire_api = "responses"
|
|
764
794
|
TOML
|
|
765
795
|
|
|
766
|
-
File.write(
|
|
767
|
-
written <<
|
|
796
|
+
File.write(profile_path, content)
|
|
797
|
+
written << profile_path
|
|
798
|
+
rescue StandardError => e
|
|
799
|
+
raise Thor::Error, "Failed to write #{profile_path}: #{e.message}"
|
|
800
|
+
end
|
|
801
|
+
|
|
802
|
+
def write_codex_catalog(codex_dir, written, skipped)
|
|
803
|
+
catalog_path = File.join(codex_dir, 'legionio-catalog.json')
|
|
804
|
+
|
|
805
|
+
if File.exist?(catalog_path) && !options[:force]
|
|
806
|
+
skipped << catalog_path
|
|
807
|
+
return
|
|
808
|
+
end
|
|
809
|
+
|
|
810
|
+
catalog = {
|
|
811
|
+
models: [
|
|
812
|
+
{
|
|
813
|
+
id: 'legionio',
|
|
814
|
+
name: 'LegionIO',
|
|
815
|
+
context_size: 262_144,
|
|
816
|
+
context_window: 262_144
|
|
817
|
+
},
|
|
818
|
+
{
|
|
819
|
+
id: 'auto',
|
|
820
|
+
name: 'LegionIO (auto)',
|
|
821
|
+
context_size: 262_144,
|
|
822
|
+
context_window: 262_144
|
|
823
|
+
}
|
|
824
|
+
]
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
File.write(catalog_path, ::JSON.pretty_generate(catalog))
|
|
828
|
+
written << catalog_path
|
|
829
|
+
rescue StandardError => e
|
|
830
|
+
raise Thor::Error, "Failed to write #{catalog_path}: #{e.message}"
|
|
831
|
+
end
|
|
832
|
+
|
|
833
|
+
def write_codex_main_config(codex_dir, _base_url, written, _skipped)
|
|
834
|
+
config_path = File.join(codex_dir, 'config.toml')
|
|
835
|
+
existing = File.exist?(config_path) ? File.read(config_path) : ''
|
|
836
|
+
|
|
837
|
+
if existing.match?(/^\s*profile\s*=\s*"legionio"/)
|
|
838
|
+
written << config_path
|
|
839
|
+
return
|
|
840
|
+
end
|
|
841
|
+
|
|
842
|
+
updated = existing.empty? ? "profile = \"legionio\"\n" : "profile = \"legionio\"\n\n#{existing.lstrip}"
|
|
843
|
+
File.write(config_path, updated)
|
|
844
|
+
written << config_path
|
|
845
|
+
rescue StandardError => e
|
|
846
|
+
raise Thor::Error, "Failed to write #{config_path}: #{e.message}"
|
|
847
|
+
end
|
|
848
|
+
|
|
849
|
+
def write_zsh_legionio(base_url, written, _skipped)
|
|
850
|
+
zshrc_path = File.expand_path('~/.zshrc')
|
|
851
|
+
return unless File.exist?(zshrc_path)
|
|
852
|
+
|
|
853
|
+
host_base = base_url.sub(%r{/v1$}, '')
|
|
854
|
+
zsh_file = File.expand_path('~/.zsh_legionio')
|
|
855
|
+
|
|
856
|
+
content = <<~ZSH
|
|
857
|
+
# LegionIO shell helpers — generated by `legionio setup proxy-mode`
|
|
858
|
+
# Re-run to update; do not edit manually.
|
|
859
|
+
|
|
860
|
+
claude-legionio() {
|
|
861
|
+
export ANTHROPIC_BASE_URL=#{host_base}
|
|
862
|
+
export ANTHROPIC_API_KEY=legion
|
|
863
|
+
export ANTHROPIC_AUTH_TOKEN=
|
|
864
|
+
export CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1
|
|
865
|
+
export CLAUDE_CODE_USE_BEDROCK=
|
|
866
|
+
export AWS_PROFILE=
|
|
867
|
+
export AWS_REGION=
|
|
868
|
+
unset ANTHROPIC_DEFAULT_OPUS_MODEL
|
|
869
|
+
unset ANTHROPIC_DEFAULT_SONNET_MODEL
|
|
870
|
+
unset ANTHROPIC_DEFAULT_HAIKU_MODEL
|
|
871
|
+
claude --model legionio "$@"
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
codex-legionio() {
|
|
875
|
+
codex --provider legionio "$@"
|
|
876
|
+
}
|
|
877
|
+
ZSH
|
|
878
|
+
|
|
879
|
+
File.write(zsh_file, content)
|
|
880
|
+
written << zsh_file
|
|
881
|
+
|
|
882
|
+
source_line = '[ -f ~/.zsh_legionio ] && source ~/.zsh_legionio'
|
|
883
|
+
zshrc = File.read(zshrc_path)
|
|
884
|
+
unless zshrc.include?(source_line)
|
|
885
|
+
File.write(zshrc_path, "#{zshrc.rstrip}\n\n#{source_line}\n")
|
|
886
|
+
written << zshrc_path
|
|
887
|
+
end
|
|
768
888
|
rescue StandardError => e
|
|
769
|
-
raise Thor::Error, "Failed to write
|
|
889
|
+
raise Thor::Error, "Failed to write zsh config: #{e.message}"
|
|
770
890
|
end
|
|
771
891
|
|
|
772
892
|
def write_claude_code_proxy_config(base_url, written, skipped)
|
data/lib/legion/trace_search.rb
CHANGED
|
@@ -72,7 +72,7 @@ module Legion
|
|
|
72
72
|
)
|
|
73
73
|
Legion::Logging.error "[TraceSearch] LLM filter generation failed for query: #{query.inspect}" if !result[:valid] && defined?(Legion::Logging)
|
|
74
74
|
result[:data] if result[:valid]
|
|
75
|
-
rescue
|
|
75
|
+
rescue StandardError => e
|
|
76
76
|
handle_exception(e, level: :debug, handled: true, operation: 'trace_search.generate_filter') if respond_to?(:handle_exception)
|
|
77
77
|
nil
|
|
78
78
|
end
|
data/lib/legion/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: legionio
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.9.
|
|
4
|
+
version: 1.9.40
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Esity
|
|
@@ -431,6 +431,8 @@ files:
|
|
|
431
431
|
- docker_deploy.rb
|
|
432
432
|
- exe/legion
|
|
433
433
|
- exe/legionio
|
|
434
|
+
- exe/replay_ledger
|
|
435
|
+
- exe/rollback_ledger
|
|
434
436
|
- extensions-agentic/lex-consent/db/migrations/001_create_consent_maps.rb
|
|
435
437
|
- extensions-agentic/lex-consent/lex-consent.gemspec
|
|
436
438
|
- extensions-agentic/lex-consent/lib/legion/extensions/agentic/consent.rb
|