pcrd 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +24 -0
- data/LICENSE +21 -0
- data/README.md +614 -0
- data/bin/pcrd +7 -0
- data/lib/pcrd/advisory_lock.rb +50 -0
- data/lib/pcrd/apply/engine.rb +184 -0
- data/lib/pcrd/apply/worker.rb +97 -0
- data/lib/pcrd/backfill/batch.rb +158 -0
- data/lib/pcrd/backfill/engine.rb +153 -0
- data/lib/pcrd/checkpoint/store.rb +217 -0
- data/lib/pcrd/cli.rb +274 -0
- data/lib/pcrd/commands/analyze.rb +125 -0
- data/lib/pcrd/commands/cleanup.rb +112 -0
- data/lib/pcrd/commands/demo.rb +152 -0
- data/lib/pcrd/commands/readiness.rb +30 -0
- data/lib/pcrd/commands/status.rb +129 -0
- data/lib/pcrd/commands/verify.rb +172 -0
- data/lib/pcrd/config/add_column.rb +7 -0
- data/lib/pcrd/config/analyze_config.rb +8 -0
- data/lib/pcrd/config/column_spec.rb +10 -0
- data/lib/pcrd/config/connection.rb +7 -0
- data/lib/pcrd/config/cutover_config.rb +7 -0
- data/lib/pcrd/config/load_error.rb +7 -0
- data/lib/pcrd/config/loader.rb +158 -0
- data/lib/pcrd/config/migrate_config.rb +21 -0
- data/lib/pcrd/config/root.rb +9 -0
- data/lib/pcrd/config/schema.rb +62 -0
- data/lib/pcrd/config/table.rb +9 -0
- data/lib/pcrd/config/verify_config.rb +7 -0
- data/lib/pcrd/config.rb +7 -0
- data/lib/pcrd/connection/client.rb +129 -0
- data/lib/pcrd/connection/error.rb +7 -0
- data/lib/pcrd/connection/replication.rb +108 -0
- data/lib/pcrd/cutover/orchestrator.rb +108 -0
- data/lib/pcrd/cutover/sequences.rb +138 -0
- data/lib/pcrd/demo/generator.rb +214 -0
- data/lib/pcrd/demo/schema.rb +154 -0
- data/lib/pcrd/error.rb +12 -0
- data/lib/pcrd/migration/orchestrator.rb +272 -0
- data/lib/pcrd/monitor/lag.rb +107 -0
- data/lib/pcrd/options.rb +15 -0
- data/lib/pcrd/output/analyze_printer.rb +173 -0
- data/lib/pcrd/output/cutover_printer.rb +128 -0
- data/lib/pcrd/output/preflight_printer.rb +119 -0
- data/lib/pcrd/output/readiness_printer.rb +72 -0
- data/lib/pcrd/preflight.rb +331 -0
- data/lib/pcrd/readiness/manifest.rb +201 -0
- data/lib/pcrd/replication/consumer.rb +235 -0
- data/lib/pcrd/replication/error.rb +10 -0
- data/lib/pcrd/replication/pgoutput/messages.rb +68 -0
- data/lib/pcrd/replication/pgoutput/parser.rb +316 -0
- data/lib/pcrd/reporter/console.rb +46 -0
- data/lib/pcrd/reporter/null.rb +14 -0
- data/lib/pcrd/schema/column.rb +59 -0
- data/lib/pcrd/schema/ddl.rb +71 -0
- data/lib/pcrd/schema/diff_entry.rb +36 -0
- data/lib/pcrd/schema/differ.rb +175 -0
- data/lib/pcrd/schema/object_reader.rb +187 -0
- data/lib/pcrd/schema/packer.rb +90 -0
- data/lib/pcrd/schema/reader.rb +118 -0
- data/lib/pcrd/schema/setup.rb +143 -0
- data/lib/pcrd/schema/setup_error.rb +9 -0
- data/lib/pcrd/schema/table_not_found.rb +8 -0
- data/lib/pcrd/schema/type_registry.rb +116 -0
- data/lib/pcrd/sql.rb +55 -0
- data/lib/pcrd/transform/row_transformer.rb +69 -0
- data/lib/pcrd/transform/type_map.rb +209 -0
- data/lib/pcrd/transform/validator.rb +106 -0
- data/lib/pcrd/version.rb +5 -0
- data/lib/pcrd.rb +11 -0
- metadata +231 -0
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pcrd
|
|
4
|
+
module Migration
|
|
5
|
+
# Drives the full migrate flow: setup, concurrent backfill + WAL apply, and
|
|
6
|
+
# the streaming monitor, with cleanup guaranteed. Extracted from the CLI so
|
|
7
|
+
# the orchestration is testable on its own and free of Thor.
|
|
8
|
+
#
|
|
9
|
+
# The CLI stays a thin adapter: it runs preflight, confirms with the
|
|
10
|
+
# operator, installs signal traps that call #request_stop, and renders the
|
|
11
|
+
# result. All progress output goes through an injected Reporter.
|
|
12
|
+
#
|
|
13
|
+
# #run assumes preflight has passed and the operator has confirmed; it
|
|
14
|
+
# returns an outcome symbol (:completed | :interrupted | :backfill_only) and
|
|
15
|
+
# raises Pcrd::Error subclasses (Replication::Error, Connection::Error, ...)
|
|
16
|
+
# for the CLI to translate.
|
|
17
|
+
class Orchestrator
|
|
18
|
+
LAG_CHECK_INTERVAL = 2 # seconds between lag readings in streaming mode
|
|
19
|
+
|
|
20
|
+
def initialize(config:, options: {}, reporter: Reporter::Console.new)
|
|
21
|
+
@config = config
|
|
22
|
+
@options = Options.normalize(options)
|
|
23
|
+
@reporter = reporter
|
|
24
|
+
@mutex = Mutex.new
|
|
25
|
+
@stop = false
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Safe to call from a signal handler / another thread.
|
|
29
|
+
def request_stop
|
|
30
|
+
@mutex.synchronize { @stop = true }
|
|
31
|
+
@backfill_engine&.stop!
|
|
32
|
+
@reporter.info("\nStopping...")
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def run
|
|
36
|
+
@source_pool = Connection::Client.new(@config.source)
|
|
37
|
+
@target_pool = Connection::Client.new(@config.target)
|
|
38
|
+
@checkpoint = Checkpoint::Store.new(@config.migrate.checkpoint_db)
|
|
39
|
+
setup = Schema::Setup.new(source_pool: @source_pool, target_pool: @target_pool, config: @config)
|
|
40
|
+
|
|
41
|
+
report_session_settings
|
|
42
|
+
acquire_lock!
|
|
43
|
+
|
|
44
|
+
start_lsn = prepare_replication(setup)
|
|
45
|
+
start_streaming(start_lsn) unless backfill_only?
|
|
46
|
+
|
|
47
|
+
return :interrupted if run_backfill == :interrupted
|
|
48
|
+
|
|
49
|
+
if backfill_only?
|
|
50
|
+
@reporter.info("Run `pcrd verify` to check row counts, then `pcrd cutover` when ready.")
|
|
51
|
+
return :backfill_only
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
stream_until_stopped
|
|
55
|
+
ensure
|
|
56
|
+
cleanup
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
def stopped?
|
|
62
|
+
@mutex.synchronize { @stop }
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def backfill_only?
|
|
66
|
+
@options[:"backfill-only"]
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def report_session_settings
|
|
70
|
+
s = @source_pool.session_settings
|
|
71
|
+
@reporter.info(
|
|
72
|
+
"Session: application_name=#{s['application_name']}, lock_timeout=#{s['lock_timeout']}, " \
|
|
73
|
+
"idle_in_transaction=#{s['idle_in_transaction_session_timeout']}, " \
|
|
74
|
+
"statement_timeout=#{s['statement_timeout']}"
|
|
75
|
+
)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def acquire_lock!
|
|
79
|
+
@migration_lock = AdvisoryLock.new(pool: @source_pool, name: @config.migrate.replication_slot)
|
|
80
|
+
return if @migration_lock.try_acquire
|
|
81
|
+
|
|
82
|
+
raise Pcrd::Error,
|
|
83
|
+
"Another pcrd migration is already running against slot " \
|
|
84
|
+
"'#{@config.migrate.replication_slot}'. Wait for it to finish, or stop it before retrying."
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Ensures (or, on resume, validates) the slot/publication and target
|
|
88
|
+
# tables, returning the LSN streaming should start from.
|
|
89
|
+
def prepare_replication(setup)
|
|
90
|
+
if @options[:resume]
|
|
91
|
+
unless backfill_only?
|
|
92
|
+
setup.validate_resumable!(
|
|
93
|
+
pub_name: @config.migrate.publication, slot_name: @config.migrate.replication_slot
|
|
94
|
+
)
|
|
95
|
+
end
|
|
96
|
+
start_lsn = @checkpoint.backfill_start_lsn || "0/0"
|
|
97
|
+
@reporter.info("\nResuming migration from LSN #{start_lsn}...")
|
|
98
|
+
return start_lsn
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
start_lsn = "0/0"
|
|
102
|
+
unless backfill_only?
|
|
103
|
+
@reporter.info("\nCreating publication and replication slot...")
|
|
104
|
+
start_lsn = setup.create_publication_and_slot(
|
|
105
|
+
pub_name: @config.migrate.publication, slot_name: @config.migrate.replication_slot
|
|
106
|
+
)
|
|
107
|
+
@checkpoint.set_backfill_start_lsn(start_lsn)
|
|
108
|
+
@checkpoint.set_publication(@config.migrate.publication)
|
|
109
|
+
@checkpoint.set_replication_slot(@config.migrate.replication_slot)
|
|
110
|
+
@reporter.success(" Slot created at LSN #{start_lsn}.")
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
@reporter.info("\nCreating target tables...")
|
|
114
|
+
setup.create_target_tables(force_overwrite: @options[:"force-overwrite"])
|
|
115
|
+
@reporter.success(" Target tables created.")
|
|
116
|
+
start_lsn
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Starts the WAL consumer and the apply worker on its own target
|
|
120
|
+
# connection (Connection::Client is single-connection and unsafe to share
|
|
121
|
+
# with backfill's writes).
|
|
122
|
+
def start_streaming(start_lsn)
|
|
123
|
+
repl_conn = Connection::Replication.new(@config.source)
|
|
124
|
+
@parser = Replication::Pgoutput::Parser.new
|
|
125
|
+
@consumer = Replication::Consumer.new(
|
|
126
|
+
repl_conn: repl_conn, parser: @parser,
|
|
127
|
+
slot_name: @config.migrate.replication_slot,
|
|
128
|
+
pub_name: @config.migrate.publication, start_lsn: start_lsn
|
|
129
|
+
)
|
|
130
|
+
@reporter.info("\nStarting WAL consumer...")
|
|
131
|
+
@consumer.start
|
|
132
|
+
@reporter.success(" Streaming from #{start_lsn}.")
|
|
133
|
+
|
|
134
|
+
@apply_pool = Connection::Client.new(@config.target)
|
|
135
|
+
apply_engine = Apply::Engine.new(
|
|
136
|
+
target_pool: @apply_pool, config: @config, parser: @parser,
|
|
137
|
+
source_schema: read_source_schema
|
|
138
|
+
)
|
|
139
|
+
@apply_worker = Apply::Worker.new(
|
|
140
|
+
engine: apply_engine, queue: @consumer.queue,
|
|
141
|
+
# Acknowledge to the source only after the txn is durably applied and
|
|
142
|
+
# checkpointed, so WAL is not released prematurely.
|
|
143
|
+
on_committed: lambda { |lsn|
|
|
144
|
+
@checkpoint.set_lsn(lsn)
|
|
145
|
+
@consumer.advance_lsn(lsn)
|
|
146
|
+
}
|
|
147
|
+
)
|
|
148
|
+
@apply_worker.start
|
|
149
|
+
@reporter.success(" Applying WAL concurrently with backfill.")
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Runs backfill concurrently with the apply worker. Returns :interrupted
|
|
153
|
+
# if it stopped early, otherwise nil.
|
|
154
|
+
def run_backfill
|
|
155
|
+
@backfill_engine = Backfill::Engine.new(
|
|
156
|
+
source_pool: @source_pool, target_pool: @target_pool,
|
|
157
|
+
config: @config, checkpoint: @checkpoint
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
if (rps = @config.migrate.max_rows_per_second)
|
|
161
|
+
@reporter.info("\nStarting backfill (throttled to #{format_count(rps)} rows/s)...")
|
|
162
|
+
else
|
|
163
|
+
@reporter.info("\nStarting backfill...")
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
results = @backfill_engine.run(on_batch: method(:report_batch))
|
|
167
|
+
@reporter.info("")
|
|
168
|
+
|
|
169
|
+
results.each do |r|
|
|
170
|
+
status = r.stopped_early ? " (interrupted)" : ""
|
|
171
|
+
@reporter.success(" #{r.table_name}: #{format_count(r.rows_copied)} rows " \
|
|
172
|
+
"in #{r.batch_count} batches#{status}")
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
raise_if_streaming_failed
|
|
176
|
+
|
|
177
|
+
if results.any?(&:stopped_early) || stopped?
|
|
178
|
+
@reporter.warn("\nInterrupted. Resume with --resume.")
|
|
179
|
+
return :interrupted
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
@reporter.success("\nBackfill complete.")
|
|
183
|
+
nil
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Surfaces a dead apply worker or WAL consumer as a Replication::Error so
|
|
187
|
+
# a failure during backfill (e.g. a TRUNCATE or a dropped connection)
|
|
188
|
+
# aborts promptly instead of being noticed only once streaming begins.
|
|
189
|
+
def raise_if_streaming_failed
|
|
190
|
+
if @apply_worker&.failed?
|
|
191
|
+
raise Replication::Error, "Apply worker stopped: #{@apply_worker.error.message}"
|
|
192
|
+
end
|
|
193
|
+
return unless @consumer&.failed?
|
|
194
|
+
|
|
195
|
+
raise Replication::Error, "WAL consumer stopped: #{@consumer.last_error.message}"
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def stream_until_stopped
|
|
199
|
+
@reporter.info("Entering streaming mode. Press Ctrl-C to stop.\n")
|
|
200
|
+
lag_monitor = Monitor::Lag.new(source_pool: @source_pool, slot_name: @config.migrate.replication_slot)
|
|
201
|
+
last_lag_check = 0.0
|
|
202
|
+
|
|
203
|
+
loop do
|
|
204
|
+
break if stopped?
|
|
205
|
+
|
|
206
|
+
if @apply_worker.failed?
|
|
207
|
+
raise Replication::Error, "Apply worker stopped: #{@apply_worker.error.message}"
|
|
208
|
+
end
|
|
209
|
+
if @consumer.failed? && @consumer.queue.empty?
|
|
210
|
+
raise Replication::Error, "WAL consumer stopped: #{@consumer.last_error.message}"
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
214
|
+
if now - last_lag_check >= LAG_CHECK_INTERVAL
|
|
215
|
+
render_lag(lag_monitor)
|
|
216
|
+
last_lag_check = now
|
|
217
|
+
end
|
|
218
|
+
sleep 0.1
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
@reporter.info("")
|
|
222
|
+
if stopped?
|
|
223
|
+
@reporter.warn("Migration interrupted. Resume with:")
|
|
224
|
+
@reporter.warn(" pcrd migrate --config #{@options[:config] || Config::DEFAULT_CONFIG_FILE} --resume")
|
|
225
|
+
end
|
|
226
|
+
:completed
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def render_lag(lag_monitor)
|
|
230
|
+
lag = lag_monitor.lag_bytes
|
|
231
|
+
threshold = @config.migrate.lag_threshold_bytes
|
|
232
|
+
metrics = "queue: #{@consumer.queue_depth} applied: #{@apply_worker.last_applied_lsn || '—'}"
|
|
233
|
+
ready = lag && lag <= threshold ? " #{@reporter.green('✓ Ready for cutover')}" : ""
|
|
234
|
+
@reporter.status(" Lag: #{lag_monitor.summary} | #{metrics}#{ready} ")
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def read_source_schema
|
|
238
|
+
reader = Schema::Reader.new(@source_pool)
|
|
239
|
+
(@config.migrate&.tables || []).each_with_object({}) do |table, h|
|
|
240
|
+
h[table.name] = {
|
|
241
|
+
columns: reader.read(table.name),
|
|
242
|
+
pk_columns: reader.primary_key_columns(table.name)
|
|
243
|
+
}
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def report_batch(stats)
|
|
248
|
+
rps = stats[:duration_ms] > 0 ? (stats[:row_count] * 1000.0 / stats[:duration_ms]).round : 0
|
|
249
|
+
@reporter.status(
|
|
250
|
+
" #{stats[:table]} batch #{stats[:batch_num]} " \
|
|
251
|
+
"#{format_count(stats[:rows_so_far])} rows #{format_count(rps)} rows/s "
|
|
252
|
+
)
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Stop the producer first so the queue is finite, then let the worker
|
|
256
|
+
# drain what's left before closing its connection.
|
|
257
|
+
def cleanup
|
|
258
|
+
@consumer&.stop rescue nil
|
|
259
|
+
@apply_worker&.stop rescue nil
|
|
260
|
+
@apply_pool&.close rescue nil
|
|
261
|
+
@checkpoint&.close rescue nil
|
|
262
|
+
@migration_lock&.release rescue nil
|
|
263
|
+
@source_pool&.close rescue nil
|
|
264
|
+
@target_pool&.close rescue nil
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def format_count(n)
|
|
268
|
+
n.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
|
|
269
|
+
end
|
|
270
|
+
end
|
|
271
|
+
end
|
|
272
|
+
end
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pcrd
|
|
4
|
+
module Monitor
|
|
5
|
+
# Queries the source database for replication lag on a named slot.
|
|
6
|
+
#
|
|
7
|
+
# Lag is reported in bytes (WAL bytes the slot has not yet confirmed).
|
|
8
|
+
# A rolling window of recent readings is maintained so callers can
|
|
9
|
+
# compute rate-of-change and estimated time to zero.
|
|
10
|
+
class Lag
|
|
11
|
+
WINDOW_SIZE = 10 # readings to keep for trend analysis
|
|
12
|
+
|
|
13
|
+
Reading = Data.define(:bytes, :taken_at)
|
|
14
|
+
|
|
15
|
+
def initialize(source_pool:, slot_name:)
|
|
16
|
+
@pool = source_pool
|
|
17
|
+
@slot_name = slot_name
|
|
18
|
+
@history = []
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Queries the current lag in bytes. Returns nil if the slot is not found.
|
|
22
|
+
def lag_bytes
|
|
23
|
+
result = @pool.exec(<<~SQL, [@slot_name])
|
|
24
|
+
SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn)::bigint AS lag
|
|
25
|
+
FROM pg_replication_slots
|
|
26
|
+
WHERE slot_name = $1
|
|
27
|
+
SQL
|
|
28
|
+
|
|
29
|
+
return nil if result.ntuples.zero?
|
|
30
|
+
|
|
31
|
+
bytes = result[0]["lag"].to_i
|
|
32
|
+
record(bytes)
|
|
33
|
+
bytes
|
|
34
|
+
rescue Connection::Error
|
|
35
|
+
nil
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Returns the confirmed_flush_lsn for the slot as a "X/Y" string.
|
|
39
|
+
def confirmed_lsn
|
|
40
|
+
result = @pool.exec(<<~SQL, [@slot_name])
|
|
41
|
+
SELECT confirmed_flush_lsn FROM pg_replication_slots WHERE slot_name = $1
|
|
42
|
+
SQL
|
|
43
|
+
result.ntuples > 0 ? result[0]["confirmed_flush_lsn"] : nil
|
|
44
|
+
rescue Connection::Error
|
|
45
|
+
nil
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Returns bytes/second rate of lag change (negative = lag is shrinking).
|
|
49
|
+
# Returns nil if fewer than 2 readings available.
|
|
50
|
+
def trend_bytes_per_sec
|
|
51
|
+
return nil if @history.size < 2
|
|
52
|
+
|
|
53
|
+
oldest = @history.first
|
|
54
|
+
newest = @history.last
|
|
55
|
+
elapsed = newest.taken_at - oldest.taken_at
|
|
56
|
+
return nil if elapsed <= 0
|
|
57
|
+
|
|
58
|
+
(newest.bytes - oldest.bytes) / elapsed
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Estimated seconds until lag reaches zero at current trend.
|
|
62
|
+
# Returns nil if trend is not converging (positive or unknown).
|
|
63
|
+
def eta_seconds
|
|
64
|
+
trend = trend_bytes_per_sec
|
|
65
|
+
return nil unless trend&.negative?
|
|
66
|
+
|
|
67
|
+
current = @history.last&.bytes
|
|
68
|
+
return nil unless current
|
|
69
|
+
|
|
70
|
+
-(current / trend).ceil
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Human-readable lag summary string.
|
|
74
|
+
def summary
|
|
75
|
+
bytes = lag_bytes
|
|
76
|
+
return "unknown (slot not found)" if bytes.nil?
|
|
77
|
+
|
|
78
|
+
parts = ["#{format_bytes(bytes)} behind"]
|
|
79
|
+
eta = eta_seconds
|
|
80
|
+
parts << "ETA ~#{format_duration(eta)}" if eta
|
|
81
|
+
parts.join(" ")
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
private
|
|
85
|
+
|
|
86
|
+
def record(bytes)
|
|
87
|
+
@history << Reading.new(bytes: bytes, taken_at: Time.now.to_f)
|
|
88
|
+
@history = @history.last(WINDOW_SIZE)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def format_bytes(n)
|
|
92
|
+
return "#{n} B" if n < 1024
|
|
93
|
+
return "#{(n / 1024.0).round(1)} KB" if n < 1_048_576
|
|
94
|
+
return "#{(n / 1_048_576.0).round(1)} MB" if n < 1_073_741_824
|
|
95
|
+
|
|
96
|
+
"#{(n / 1_073_741_824.0).round(2)} GB"
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def format_duration(secs)
|
|
100
|
+
return "#{secs}s" if secs < 60
|
|
101
|
+
return "#{(secs / 60.0).ceil}m" if secs < 3600
|
|
102
|
+
|
|
103
|
+
"#{(secs / 3600.0).ceil}h"
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
data/lib/pcrd/options.rb
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pcrd
|
|
4
|
+
# Normalizes the options hash once at the boundary so commands can use symbol
|
|
5
|
+
# keys consistently, instead of guarding every read with
|
|
6
|
+
# `options["x"] || options[:"x"]`. Accepts Thor's string-keyed options, a
|
|
7
|
+
# plain symbol hash, or nil.
|
|
8
|
+
module Options
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
def normalize(opts)
|
|
12
|
+
(opts || {}).to_h.transform_keys(&:to_sym)
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tty-table"
|
|
4
|
+
require "tty-screen"
|
|
5
|
+
require "pastel"
|
|
6
|
+
|
|
7
|
+
module Pcrd
|
|
8
|
+
module Output
|
|
9
|
+
class AnalyzePrinter
|
|
10
|
+
PASTEL = Pastel.new
|
|
11
|
+
|
|
12
|
+
STATUS_COLORS = {
|
|
13
|
+
unchanged: ->(s) { s },
|
|
14
|
+
type_changed: ->(s) { PASTEL.yellow(s) },
|
|
15
|
+
renamed: ->(s) { PASTEL.cyan(s) },
|
|
16
|
+
type_and_renamed: ->(s) { PASTEL.yellow(s) },
|
|
17
|
+
dropped: ->(s) { PASTEL.red(s) },
|
|
18
|
+
added: ->(s) { PASTEL.green(s) }
|
|
19
|
+
}.freeze
|
|
20
|
+
|
|
21
|
+
def initialize(output: $stdout)
|
|
22
|
+
@out = output
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Single-cluster padding analysis: current layout vs. optimal reordering.
|
|
26
|
+
def print_table_report(table_name:, schema_name: "public", row_count:, report:)
|
|
27
|
+
heading = "Table: #{schema_name}.#{table_name}"
|
|
28
|
+
heading += " (#{format_count(row_count)} rows)" if row_count > 0
|
|
29
|
+
@out.puts
|
|
30
|
+
@out.puts PASTEL.bold(heading)
|
|
31
|
+
@out.puts
|
|
32
|
+
|
|
33
|
+
if report[:already_optimal]
|
|
34
|
+
@out.puts PASTEL.green(" ✓ Column order is already optimal. No padding waste detected.")
|
|
35
|
+
@out.puts
|
|
36
|
+
print_layout_table(report[:current_layout], title: "Current layout")
|
|
37
|
+
return
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
print_layout_table(report[:current_layout], title: "Current layout",
|
|
41
|
+
highlight_padding: true)
|
|
42
|
+
@out.puts
|
|
43
|
+
print_savings_summary(report, row_count)
|
|
44
|
+
@out.puts
|
|
45
|
+
print_layout_table(report[:optimal_layout], title: "Suggested layout (optimal packing)")
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Cross-cluster diff: source schema vs. target (live or synthesized from spec).
|
|
49
|
+
def print_diff_report(table_name:, schema_name: "public", row_count:,
|
|
50
|
+
diff_entries:, packer:, target_is_live: false)
|
|
51
|
+
heading = "Table: #{schema_name}.#{table_name}"
|
|
52
|
+
heading += " (#{format_count(row_count)} rows)" if row_count > 0
|
|
53
|
+
heading += target_is_live ? " — live source vs. live target" \
|
|
54
|
+
: " — source vs. proposed target (synthesized from spec)"
|
|
55
|
+
@out.puts
|
|
56
|
+
@out.puts PASTEL.bold(heading)
|
|
57
|
+
@out.puts
|
|
58
|
+
|
|
59
|
+
print_diff_table(diff_entries)
|
|
60
|
+
@out.puts
|
|
61
|
+
print_diff_padding_summary(diff_entries, packer, row_count)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
def print_layout_table(layout_entries, title:, highlight_padding: false)
|
|
67
|
+
@out.puts " #{PASTEL.bold(title)}:"
|
|
68
|
+
@out.puts
|
|
69
|
+
|
|
70
|
+
rows = layout_entries.map do |entry|
|
|
71
|
+
col = entry.column
|
|
72
|
+
padding = entry.padding_before
|
|
73
|
+
|
|
74
|
+
padding_cell = if padding > 0 && highlight_padding
|
|
75
|
+
PASTEL.yellow("← #{padding} wasted")
|
|
76
|
+
elsif padding > 0
|
|
77
|
+
"#{padding} bytes"
|
|
78
|
+
else
|
|
79
|
+
"—"
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
[col.name, col.display_type, col.display_alignment, col.display_size, padding_cell]
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
render_table(["Column", "Type", "Align", "Size", "Padding before"], rows)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def print_savings_summary(report, row_count)
|
|
89
|
+
saved = report[:saved_bytes]
|
|
90
|
+
pct = report[:savings_pct]
|
|
91
|
+
curr = report[:current_size]
|
|
92
|
+
opt = report[:optimal_size]
|
|
93
|
+
|
|
94
|
+
@out.puts " #{PASTEL.bold("Padding analysis:")}"
|
|
95
|
+
@out.puts " Current row overhead (fixed cols + padding): #{curr} bytes"
|
|
96
|
+
@out.puts " Optimal row overhead (fixed cols only): #{opt} bytes"
|
|
97
|
+
@out.puts " #{PASTEL.yellow("Wasted padding:")} #{PASTEL.bold("#{saved} bytes/row")} (#{pct}%)"
|
|
98
|
+
|
|
99
|
+
return unless row_count > 0
|
|
100
|
+
|
|
101
|
+
total_mb = (saved * row_count) / (1024.0 * 1024.0)
|
|
102
|
+
scale = scale_label(total_mb)
|
|
103
|
+
@out.puts " At #{format_count(row_count)} rows: #{PASTEL.bold("~#{scale} reclaimed")} by reordering columns"
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def print_diff_table(diff_entries)
|
|
107
|
+
@out.puts " #{PASTEL.bold("Schema diff")} " \
|
|
108
|
+
"(#{PASTEL.yellow("■")} type changed " \
|
|
109
|
+
"#{PASTEL.cyan("■")} renamed " \
|
|
110
|
+
"#{PASTEL.red("■")} dropped " \
|
|
111
|
+
"#{PASTEL.green("■")} added)"
|
|
112
|
+
@out.puts
|
|
113
|
+
|
|
114
|
+
rows = diff_entries.map do |entry|
|
|
115
|
+
colorize = STATUS_COLORS[entry.status]
|
|
116
|
+
src_name = entry.source_column ? colorize.call(entry.source_column.name) : PASTEL.dim("—")
|
|
117
|
+
src_type = entry.source_column ? colorize.call(entry.source_column.display_type) : PASTEL.dim("—")
|
|
118
|
+
tgt_name = entry.target_column ? colorize.call(entry.target_column.name) : PASTEL.dim("—")
|
|
119
|
+
tgt_type = entry.target_column ? colorize.call(entry.target_column.display_type) : PASTEL.dim("—")
|
|
120
|
+
[src_name, src_type, tgt_name, tgt_type, colorize.call(entry.status_label)]
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
render_table(["Source column", "Source type", "Target column", "Target type", "Change"], rows)
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def print_diff_padding_summary(diff_entries, packer, row_count)
|
|
127
|
+
source_cols = diff_entries.reject(&:added?).map(&:source_column).compact
|
|
128
|
+
target_cols = diff_entries.reject(&:dropped?).map(&:target_column).compact
|
|
129
|
+
|
|
130
|
+
src_size = packer.estimated_row_size(source_cols)
|
|
131
|
+
tgt_size = packer.estimated_row_size(target_cols)
|
|
132
|
+
src_padding = packer.total_padding(source_cols)
|
|
133
|
+
tgt_padding = packer.total_padding(target_cols)
|
|
134
|
+
delta = src_size - tgt_size
|
|
135
|
+
delta_pct = src_size > 0 ? (delta.to_f / src_size * 100).abs.round(1) : 0.0
|
|
136
|
+
|
|
137
|
+
@out.puts " #{PASTEL.bold("Row size comparison (fixed columns + padding):")}"
|
|
138
|
+
@out.puts " Source: #{src_size} bytes (#{src_padding} bytes padding)"
|
|
139
|
+
|
|
140
|
+
size_line = " Target: #{tgt_size} bytes (#{tgt_padding} bytes padding)"
|
|
141
|
+
if delta > 0
|
|
142
|
+
@out.puts size_line + " #{PASTEL.green("▼ #{delta} bytes smaller (#{delta_pct}%)")}"
|
|
143
|
+
elsif delta < 0
|
|
144
|
+
@out.puts size_line + " #{PASTEL.yellow("▲ #{delta.abs} bytes larger (#{delta_pct}%)")}"
|
|
145
|
+
else
|
|
146
|
+
@out.puts size_line + " (no change)"
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
return unless row_count > 0 && delta != 0
|
|
150
|
+
|
|
151
|
+
total_mb = (delta.abs * row_count) / (1024.0 * 1024.0)
|
|
152
|
+
verb = delta > 0 ? "saved" : "added"
|
|
153
|
+
@out.puts " At #{format_count(row_count)} rows: #{PASTEL.bold("~#{scale_label(total_mb)} #{verb}")}"
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def render_table(headers, rows)
|
|
157
|
+
table = TTY::Table.new(header: headers, rows: rows)
|
|
158
|
+
rendered = table.render(:unicode, padding: [0, 1]) do |r|
|
|
159
|
+
r.border.separator = :each_row
|
|
160
|
+
end
|
|
161
|
+
rendered.each_line { @out.puts " #{_1.chomp}" }
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def scale_label(mb)
|
|
165
|
+
mb >= 1024 ? "#{(mb / 1024).round(1)} GB" : "#{mb.round(1)} MB"
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def format_count(n)
|
|
169
|
+
n.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pastel"
|
|
4
|
+
|
|
5
|
+
module Pcrd
|
|
6
|
+
module Output
|
|
7
|
+
class CutoverPrinter
|
|
8
|
+
PASTEL = Pastel.new
|
|
9
|
+
|
|
10
|
+
def initialize(output: $stdout)
|
|
11
|
+
@out = output
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def print(result)
|
|
15
|
+
@out.puts
|
|
16
|
+
@out.puts PASTEL.bold("Cutover report")
|
|
17
|
+
@out.puts PASTEL.dim("─" * 70)
|
|
18
|
+
@out.puts
|
|
19
|
+
|
|
20
|
+
print_row_counts(result.row_counts)
|
|
21
|
+
print_sequences(result.sequence_results)
|
|
22
|
+
print_warnings(result.warnings)
|
|
23
|
+
print_summary(result)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def print_verify(result)
|
|
27
|
+
@out.puts
|
|
28
|
+
@out.puts PASTEL.bold("Verify results")
|
|
29
|
+
@out.puts PASTEL.dim("─" * 70)
|
|
30
|
+
@out.puts
|
|
31
|
+
|
|
32
|
+
result.tables.each do |t|
|
|
33
|
+
src = t.source_count
|
|
34
|
+
tgt = t.target_count
|
|
35
|
+
|
|
36
|
+
if src.nil?
|
|
37
|
+
@out.puts " #{PASTEL.red("✗")} #{t.table_name} #{PASTEL.red(t.mismatches.first)}"
|
|
38
|
+
next
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
if src == tgt
|
|
42
|
+
suffix = t.mismatches.empty? ? "" : PASTEL.yellow(" (#{t.mismatches.length} spot-check mismatch(es))")
|
|
43
|
+
@out.puts " #{PASTEL.green("✓")} #{t.table_name} " \
|
|
44
|
+
"#{PASTEL.dim("#{format_count(src)} rows match")}#{suffix}"
|
|
45
|
+
else
|
|
46
|
+
@out.puts " #{PASTEL.red("✗")} #{t.table_name} " \
|
|
47
|
+
"source=#{format_count(src)} target=#{format_count(tgt)} " \
|
|
48
|
+
"#{PASTEL.red("count mismatch")}"
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
@out.puts
|
|
53
|
+
if result.passed
|
|
54
|
+
@out.puts " #{PASTEL.green("✓")} #{PASTEL.bold("All tables verified.")}"
|
|
55
|
+
else
|
|
56
|
+
@out.puts " #{PASTEL.red("✗")} #{PASTEL.bold(PASTEL.red("Verification failed."))}"
|
|
57
|
+
end
|
|
58
|
+
@out.puts
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def print_row_counts(row_counts)
|
|
64
|
+
@out.puts " #{PASTEL.bold("Row counts:")}"
|
|
65
|
+
row_counts.each do |table, counts|
|
|
66
|
+
src = counts[:source]
|
|
67
|
+
tgt = counts[:target]
|
|
68
|
+
|
|
69
|
+
if src == tgt
|
|
70
|
+
@out.puts " #{PASTEL.green("✓")} #{table} #{format_count(src)} rows"
|
|
71
|
+
else
|
|
72
|
+
@out.puts " #{PASTEL.red("✗")} #{table} " \
|
|
73
|
+
"source=#{format_count(src)} target=#{format_count(tgt)} " \
|
|
74
|
+
"#{PASTEL.red("MISMATCH")}"
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
@out.puts
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def print_sequences(seq_results)
|
|
81
|
+
return if seq_results.empty?
|
|
82
|
+
|
|
83
|
+
@out.puts " #{PASTEL.bold("Sequence advancement:")}"
|
|
84
|
+
seq_results.each do |r|
|
|
85
|
+
@out.puts " #{PASTEL.green("✓")} #{r.table_name}.#{r.column_name} " \
|
|
86
|
+
"#{PASTEL.dim("setval(#{r.target_seq_name}, #{r.target_value})")}"
|
|
87
|
+
@out.puts " #{PASTEL.dim("source last_value=#{r.source_last_value} " \
|
|
88
|
+
"source max=#{r.source_max_id} buffer=+#{r.safety_buffer}")}"
|
|
89
|
+
end
|
|
90
|
+
@out.puts
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def print_warnings(warnings)
|
|
94
|
+
return if warnings.empty?
|
|
95
|
+
|
|
96
|
+
warnings.each do |w|
|
|
97
|
+
@out.puts " #{PASTEL.yellow("⚠")} #{PASTEL.yellow(w)}"
|
|
98
|
+
end
|
|
99
|
+
@out.puts
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def print_summary(result)
|
|
103
|
+
@out.puts PASTEL.dim("─" * 70)
|
|
104
|
+
@out.puts
|
|
105
|
+
|
|
106
|
+
if result.passed
|
|
107
|
+
@out.puts " #{PASTEL.green("✓")} #{PASTEL.bold("Cutover complete.")}"
|
|
108
|
+
@out.puts
|
|
109
|
+
@out.puts " #{PASTEL.bold("Next steps:")}"
|
|
110
|
+
@out.puts " 1. Update DATABASE_URL to point at the target cluster"
|
|
111
|
+
@out.puts " 2. Restart the application"
|
|
112
|
+
@out.puts " 3. Run `pcrd verify` to confirm row counts"
|
|
113
|
+
@out.puts " 4. End maintenance mode"
|
|
114
|
+
@out.puts " 5. Run `pcrd cleanup` (days later, when confident)"
|
|
115
|
+
else
|
|
116
|
+
@out.puts " #{PASTEL.red("✗")} #{PASTEL.bold("Cutover check failed.")} " \
|
|
117
|
+
"Review the issues above before switching connection strings."
|
|
118
|
+
end
|
|
119
|
+
@out.puts
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def format_count(n)
|
|
123
|
+
return "?" if n.nil?
|
|
124
|
+
n.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|