pgsync 0.5.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pgsync might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +19 -3
- data/LICENSE.txt +1 -1
- data/README.md +28 -26
- data/config.yml +4 -0
- data/lib/pgsync.rb +5 -1
- data/lib/pgsync/client.rb +52 -57
- data/lib/pgsync/data_source.rb +47 -83
- data/lib/pgsync/init.rb +19 -4
- data/lib/pgsync/schema_sync.rb +83 -0
- data/lib/pgsync/sync.rb +77 -195
- data/lib/pgsync/table.rb +28 -0
- data/lib/pgsync/table_sync.rb +150 -220
- data/lib/pgsync/task.rb +315 -0
- data/lib/pgsync/task_resolver.rb +235 -0
- data/lib/pgsync/utils.rb +53 -13
- data/lib/pgsync/version.rb +1 -1
- metadata +6 -3
- data/lib/pgsync/table_list.rb +0 -141
data/lib/pgsync/init.rb
CHANGED
@@ -2,11 +2,26 @@ module PgSync
|
|
2
2
|
class Init
|
3
3
|
include Utils
|
4
4
|
|
5
|
-
def
|
6
|
-
|
7
|
-
@options =
|
5
|
+
def initialize(arguments, options)
|
6
|
+
@arguments = arguments
|
7
|
+
@options = options
|
8
|
+
end
|
9
|
+
|
10
|
+
def perform
|
11
|
+
if @arguments.size > 1
|
12
|
+
raise Error, "Usage:\n pgsync --init [db]"
|
13
|
+
end
|
8
14
|
|
9
|
-
file =
|
15
|
+
file =
|
16
|
+
if @options[:config]
|
17
|
+
@options[:config]
|
18
|
+
elsif @arguments.any?
|
19
|
+
db_config_file(@arguments.first)
|
20
|
+
elsif @options[:db]
|
21
|
+
db_config_file(@options[:db])
|
22
|
+
else
|
23
|
+
".pgsync.yml"
|
24
|
+
end
|
10
25
|
|
11
26
|
if File.exist?(file)
|
12
27
|
raise Error, "#{file} exists."
|
@@ -0,0 +1,83 @@
|
|
1
|
+
module PgSync
|
2
|
+
class SchemaSync
|
3
|
+
include Utils
|
4
|
+
|
5
|
+
attr_reader :args, :opts
|
6
|
+
|
7
|
+
def initialize(source:, destination:, tasks:, args:, opts:)
|
8
|
+
@source = source
|
9
|
+
@destination = destination
|
10
|
+
@tasks = tasks
|
11
|
+
@args = args
|
12
|
+
@opts = opts
|
13
|
+
end
|
14
|
+
|
15
|
+
def perform
|
16
|
+
if opts[:preserve]
|
17
|
+
raise Error, "Cannot use --preserve with --schema-first or --schema-only"
|
18
|
+
end
|
19
|
+
|
20
|
+
show_spinner = output.tty? && !opts[:debug]
|
21
|
+
|
22
|
+
if show_spinner
|
23
|
+
spinner = TTY::Spinner.new(":spinner Syncing schema", format: :dots)
|
24
|
+
spinner.auto_spin
|
25
|
+
end
|
26
|
+
|
27
|
+
# if spinner, capture lines to show on error
|
28
|
+
lines = []
|
29
|
+
success =
|
30
|
+
run_command("#{dump_command} | #{restore_command}") do |line|
|
31
|
+
if show_spinner
|
32
|
+
lines << line
|
33
|
+
else
|
34
|
+
log line
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
if show_spinner
|
39
|
+
if success
|
40
|
+
spinner.success
|
41
|
+
else
|
42
|
+
spinner.error
|
43
|
+
log lines.join
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
raise Error, "Schema sync returned non-zero exit code" unless success
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def run_command(command)
|
53
|
+
Open3.popen2e(command) do |stdin, stdout, wait_thr|
|
54
|
+
stdout.each do |line|
|
55
|
+
yield line
|
56
|
+
end
|
57
|
+
wait_thr.value.success?
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def pg_restore_version
|
62
|
+
`pg_restore --version`.lines[0].chomp.split(" ")[-1].split(/[^\d.]/)[0]
|
63
|
+
rescue Errno::ENOENT
|
64
|
+
raise Error, "pg_restore not found"
|
65
|
+
end
|
66
|
+
|
67
|
+
def dump_command
|
68
|
+
tables =
|
69
|
+
if !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude] || opts[:schemas]
|
70
|
+
@tasks.map { |task| "-t #{Shellwords.escape(task.quoted_table)}" }
|
71
|
+
else
|
72
|
+
[]
|
73
|
+
end
|
74
|
+
|
75
|
+
"pg_dump -Fc --verbose --schema-only --no-owner --no-acl #{tables.join(" ")} -d #{@source.url}"
|
76
|
+
end
|
77
|
+
|
78
|
+
def restore_command
|
79
|
+
if_exists = Gem::Version.new(pg_restore_version) >= Gem::Version.new("9.4.0")
|
80
|
+
"pg_restore --verbose --no-owner --no-acl --clean #{if_exists ? "--if-exists" : nil} -d #{@destination.url}"
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
data/lib/pgsync/sync.rb
CHANGED
@@ -2,21 +2,26 @@ module PgSync
|
|
2
2
|
class Sync
|
3
3
|
include Utils
|
4
4
|
|
5
|
-
def
|
6
|
-
|
7
|
-
|
8
|
-
|
5
|
+
def initialize(arguments, options)
|
6
|
+
@arguments = arguments
|
7
|
+
@options = options
|
8
|
+
end
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
opts[opt] ||= config[opt.to_s]
|
13
|
-
end
|
10
|
+
def perform
|
11
|
+
started_at = Time.now
|
14
12
|
|
15
|
-
|
16
|
-
|
13
|
+
args = @arguments
|
14
|
+
opts = @options
|
17
15
|
|
18
|
-
#
|
19
|
-
|
16
|
+
# only resolve commands from config, not CLI arguments
|
17
|
+
[:to, :from].each do |opt|
|
18
|
+
opts[opt] ||= resolve_source(config[opt.to_s])
|
19
|
+
end
|
20
|
+
|
21
|
+
# merge other config
|
22
|
+
[:to_safe, :exclude, :schemas].each do |opt|
|
23
|
+
opts[opt] ||= config[opt.to_s]
|
24
|
+
end
|
20
25
|
|
21
26
|
if args.size > 2
|
22
27
|
raise Error, "Usage:\n pgsync [options]"
|
@@ -29,92 +34,44 @@ module PgSync
|
|
29
34
|
raise Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
|
30
35
|
end
|
31
36
|
|
37
|
+
if (opts[:preserve] || opts[:overwrite]) && destination.server_version_num < 90500
|
38
|
+
raise Error, "Postgres 9.5+ is required for --preserve and --overwrite"
|
39
|
+
end
|
40
|
+
|
32
41
|
print_description("From", source)
|
33
42
|
print_description("To", destination)
|
34
43
|
|
35
|
-
|
36
|
-
|
44
|
+
resolver = TaskResolver.new(args: args, opts: opts, source: source, destination: destination, config: config, first_schema: first_schema)
|
45
|
+
tasks =
|
46
|
+
resolver.tasks.map do |task|
|
47
|
+
Task.new(source: source, destination: destination, config: config, table: task[:table], opts: opts.merge(sql: task[:sql]))
|
48
|
+
end
|
37
49
|
|
38
|
-
if opts[:
|
39
|
-
|
50
|
+
if opts[:in_batches] && tasks.size > 1
|
51
|
+
raise Error, "Cannot use --in-batches with multiple tables"
|
52
|
+
end
|
40
53
|
|
41
|
-
|
42
|
-
if args[0] == "groups"
|
43
|
-
(config["groups"] || {}).keys
|
44
|
-
else
|
45
|
-
tables.keys
|
46
|
-
end
|
54
|
+
confirm_tables_exist(source, tasks, "source")
|
47
55
|
|
48
|
-
|
56
|
+
if opts[:list]
|
57
|
+
confirm_tables_exist(destination, tasks, "destination")
|
58
|
+
tasks.each do |task|
|
59
|
+
log task_name(task)
|
60
|
+
end
|
49
61
|
else
|
50
62
|
if opts[:schema_first] || opts[:schema_only]
|
51
|
-
|
52
|
-
raise Error, "Cannot use --preserve with --schema-first or --schema-only"
|
53
|
-
end
|
54
|
-
|
55
|
-
log "* Dumping schema"
|
56
|
-
schema_tables = tables if !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude]
|
57
|
-
sync_schema(source, destination, schema_tables)
|
63
|
+
SchemaSync.new(source: source, destination: destination, tasks: tasks, args: args, opts: opts).perform
|
58
64
|
end
|
59
65
|
|
60
66
|
unless opts[:schema_only]
|
61
|
-
|
62
|
-
|
63
|
-
in_parallel(tables, first_schema: source.search_path.find { |sp| sp != "pg_catalog" }) do |table, table_opts, source, destination|
|
64
|
-
TableSync.new(source: source, destination: destination).sync(config, table, opts.merge(table_opts))
|
65
|
-
end
|
67
|
+
TableSync.new(source: source, destination: destination, tasks: tasks, opts: opts, resolver: resolver).perform
|
66
68
|
end
|
67
69
|
|
68
|
-
log_completed(
|
70
|
+
log_completed(started_at)
|
69
71
|
end
|
70
72
|
end
|
71
73
|
|
72
|
-
|
73
|
-
tables.keys.each do |table|
|
74
|
-
unless data_source.table_exists?(table)
|
75
|
-
raise Error, "Table does not exist in #{description}: #{table}"
|
76
|
-
end
|
77
|
-
end
|
78
|
-
end
|
79
|
-
|
80
|
-
def map_deprecations(args, opts)
|
81
|
-
command = args[0]
|
82
|
-
|
83
|
-
case command
|
84
|
-
when "schema"
|
85
|
-
args.shift
|
86
|
-
opts[:schema_only] = true
|
87
|
-
deprecated "Use `psync --schema-only` instead"
|
88
|
-
when "tables"
|
89
|
-
args.shift
|
90
|
-
opts[:tables] = args.shift
|
91
|
-
deprecated "Use `pgsync #{opts[:tables]}` instead"
|
92
|
-
when "groups"
|
93
|
-
args.shift
|
94
|
-
opts[:groups] = args.shift
|
95
|
-
deprecated "Use `pgsync #{opts[:groups]}` instead"
|
96
|
-
end
|
97
|
-
|
98
|
-
if opts[:where]
|
99
|
-
opts[:sql] ||= String.new
|
100
|
-
opts[:sql] << " WHERE #{opts[:where]}"
|
101
|
-
deprecated "Use `\"WHERE #{opts[:where]}\"` instead"
|
102
|
-
end
|
103
|
-
|
104
|
-
if opts[:limit]
|
105
|
-
opts[:sql] ||= String.new
|
106
|
-
opts[:sql] << " LIMIT #{opts[:limit]}"
|
107
|
-
deprecated "Use `\"LIMIT #{opts[:limit]}\"` instead"
|
108
|
-
end
|
109
|
-
end
|
110
|
-
|
111
|
-
def sync_schema(source, destination, tables = nil)
|
112
|
-
dump_command = source.dump_command(tables)
|
113
|
-
restore_command = destination.restore_command
|
114
|
-
unless system("#{dump_command} | #{restore_command}")
|
115
|
-
raise Error, "Schema sync returned non-zero exit code"
|
116
|
-
end
|
117
|
-
end
|
74
|
+
private
|
118
75
|
|
119
76
|
def config
|
120
77
|
@config ||= begin
|
@@ -124,6 +81,8 @@ module PgSync
|
|
124
81
|
YAML.load_file(file) || {}
|
125
82
|
rescue Psych::SyntaxError => e
|
126
83
|
raise Error, e.message
|
84
|
+
rescue Errno::ENOENT
|
85
|
+
raise Error, "Config file not found: #{file}"
|
127
86
|
end
|
128
87
|
else
|
129
88
|
{}
|
@@ -131,133 +90,41 @@ module PgSync
|
|
131
90
|
end
|
132
91
|
end
|
133
92
|
|
134
|
-
def
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
spinners = TTY::Spinner::Multi.new(format: :dots, output: output)
|
141
|
-
item_spinners = {}
|
142
|
-
|
143
|
-
start = lambda do |item, i|
|
144
|
-
table, opts = item
|
145
|
-
message = String.new(":spinner ")
|
146
|
-
message << table.sub("#{first_schema}.", "")
|
147
|
-
message << " #{opts[:sql]}" if opts[:sql]
|
148
|
-
spinner = spinners.register(message)
|
149
|
-
if @options[:in_batches]
|
150
|
-
# log instead of spin for non-tty
|
151
|
-
log message.sub(":spinner", "⠋")
|
152
|
-
else
|
153
|
-
spinner.auto_spin
|
154
|
-
end
|
155
|
-
item_spinners[item] = spinner
|
156
|
-
end
|
157
|
-
|
158
|
-
failed_tables = []
|
159
|
-
|
160
|
-
finish = lambda do |item, i, result|
|
161
|
-
spinner = item_spinners[item]
|
162
|
-
table_name = item.first.sub("#{first_schema}.", "")
|
163
|
-
|
164
|
-
if result[:status] == "success"
|
165
|
-
spinner.success(display_message(result))
|
166
|
-
else
|
167
|
-
# TODO add option to fail fast
|
168
|
-
spinner.error(display_message(result))
|
169
|
-
failed_tables << table_name
|
170
|
-
fail_sync(failed_tables) if @options[:fail_fast]
|
171
|
-
end
|
172
|
-
|
173
|
-
unless spinner.send(:tty?)
|
174
|
-
status = result[:status] == "success" ? "✔" : "✖"
|
175
|
-
log [status, table_name, item.last[:sql], display_message(result)].compact.join(" ")
|
176
|
-
end
|
177
|
-
end
|
178
|
-
|
179
|
-
options = {start: start, finish: finish}
|
180
|
-
|
181
|
-
jobs = @options[:jobs]
|
182
|
-
if @options[:debug] || @options[:in_batches] || @options[:defer_constraints]
|
183
|
-
warn "--jobs ignored" if jobs
|
184
|
-
jobs = 0
|
185
|
-
end
|
186
|
-
|
187
|
-
if windows?
|
188
|
-
options[:in_threads] = jobs || 4
|
189
|
-
else
|
190
|
-
options[:in_processes] = jobs if jobs
|
191
|
-
end
|
192
|
-
|
193
|
-
maybe_defer_constraints do
|
194
|
-
# could try to use `raise Parallel::Kill` to fail faster with --fail-fast
|
195
|
-
# see `fast_faster` branch
|
196
|
-
# however, need to make sure connections are cleaned up properly
|
197
|
-
Parallel.each(tables, **options) do |table, table_opts|
|
198
|
-
# must reconnect for new thread or process
|
199
|
-
# TODO only reconnect first time
|
200
|
-
unless options[:in_processes] == 0
|
201
|
-
source.reconnect
|
202
|
-
destination.reconnect
|
203
|
-
end
|
204
|
-
|
205
|
-
# TODO warn if there are non-deferrable constraints on the table
|
206
|
-
|
207
|
-
yield table, table_opts, source, destination
|
208
|
-
end
|
209
|
-
end
|
210
|
-
|
211
|
-
fail_sync(failed_tables) if failed_tables.any?
|
212
|
-
end
|
213
|
-
|
214
|
-
def maybe_defer_constraints
|
215
|
-
if @options[:defer_constraints]
|
216
|
-
destination.transaction do
|
217
|
-
destination.execute("SET CONSTRAINTS ALL DEFERRED")
|
218
|
-
|
219
|
-
# create a transaction on the source
|
220
|
-
# to ensure we get a consistent snapshot
|
221
|
-
source.transaction do
|
222
|
-
yield
|
223
|
-
end
|
224
|
-
end
|
93
|
+
def config_file
|
94
|
+
if @options[:config]
|
95
|
+
@options[:config]
|
96
|
+
elsif @options[:db]
|
97
|
+
file = db_config_file(@options[:db])
|
98
|
+
search_tree(file) || file
|
225
99
|
else
|
226
|
-
|
100
|
+
search_tree(".pgsync.yml")
|
227
101
|
end
|
228
102
|
end
|
229
103
|
|
230
|
-
def
|
231
|
-
|
232
|
-
end
|
233
|
-
|
234
|
-
def display_message(result)
|
235
|
-
messages = []
|
236
|
-
messages << "- #{result[:time]}s" if result[:time]
|
237
|
-
messages << "(#{result[:message].lines.first.to_s.strip})" if result[:message]
|
238
|
-
messages.join(" ")
|
239
|
-
end
|
104
|
+
def search_tree(file)
|
105
|
+
return file if File.exist?(file)
|
240
106
|
|
241
|
-
|
242
|
-
|
243
|
-
|
107
|
+
path = Dir.pwd
|
108
|
+
# prevent infinite loop
|
109
|
+
20.times do
|
110
|
+
absolute_file = File.join(path, file)
|
111
|
+
break absolute_file if File.exist?(absolute_file)
|
112
|
+
path = File.dirname(path)
|
113
|
+
break if path == "/"
|
244
114
|
end
|
245
115
|
end
|
246
116
|
|
247
|
-
def
|
248
|
-
|
117
|
+
def print_description(prefix, source)
|
118
|
+
location = " on #{source.host}:#{source.port}" if source.host
|
119
|
+
log "#{prefix}: #{source.dbname}#{location}"
|
249
120
|
end
|
250
121
|
|
251
|
-
def log_completed(
|
252
|
-
time = Time.now -
|
122
|
+
def log_completed(started_at)
|
123
|
+
time = Time.now - started_at
|
253
124
|
message = "Completed in #{time.round(1)}s"
|
254
125
|
log colorize(message, :green)
|
255
126
|
end
|
256
127
|
|
257
|
-
def windows?
|
258
|
-
Gem.win_platform?
|
259
|
-
end
|
260
|
-
|
261
128
|
def source
|
262
129
|
@source ||= data_source(@options[:from])
|
263
130
|
end
|
@@ -272,6 +139,21 @@ module PgSync
|
|
272
139
|
ds
|
273
140
|
end
|
274
141
|
|
142
|
+
def resolve_source(source)
|
143
|
+
if source
|
144
|
+
source = source.dup
|
145
|
+
source.gsub!(/\$\([^)]+\)/) do |m|
|
146
|
+
command = m[2..-2]
|
147
|
+
result = `#{command}`.chomp
|
148
|
+
unless $?.success?
|
149
|
+
raise Error, "Command exited with non-zero status:\n#{command}"
|
150
|
+
end
|
151
|
+
result
|
152
|
+
end
|
153
|
+
end
|
154
|
+
source
|
155
|
+
end
|
156
|
+
|
275
157
|
def self.finalize(ds)
|
276
158
|
# must use proc instead of stabby lambda
|
277
159
|
proc { ds.close }
|