pgsync 0.5.5 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pgsync might be problematic. Click here for more details.

@@ -2,11 +2,26 @@ module PgSync
2
2
  class Init
3
3
  include Utils
4
4
 
5
- def perform(opts)
6
- # needed for config_file method
7
- @options = opts.to_hash
5
+ def initialize(arguments, options)
6
+ @arguments = arguments
7
+ @options = options
8
+ end
9
+
10
+ def perform
11
+ if @arguments.size > 1
12
+ raise Error, "Usage:\n pgsync --init [db]"
13
+ end
8
14
 
9
- file = db_config_file(opts.arguments[0]) || config_file || ".pgsync.yml"
15
+ file =
16
+ if @options[:config]
17
+ @options[:config]
18
+ elsif @arguments.any?
19
+ db_config_file(@arguments.first)
20
+ elsif @options[:db]
21
+ db_config_file(@options[:db])
22
+ else
23
+ ".pgsync.yml"
24
+ end
10
25
 
11
26
  if File.exist?(file)
12
27
  raise Error, "#{file} exists."
@@ -0,0 +1,83 @@
1
+ module PgSync
2
+ class SchemaSync
3
+ include Utils
4
+
5
+ attr_reader :args, :opts
6
+
7
+ def initialize(source:, destination:, tasks:, args:, opts:)
8
+ @source = source
9
+ @destination = destination
10
+ @tasks = tasks
11
+ @args = args
12
+ @opts = opts
13
+ end
14
+
15
+ def perform
16
+ if opts[:preserve]
17
+ raise Error, "Cannot use --preserve with --schema-first or --schema-only"
18
+ end
19
+
20
+ show_spinner = output.tty? && !opts[:debug]
21
+
22
+ if show_spinner
23
+ spinner = TTY::Spinner.new(":spinner Syncing schema", format: :dots)
24
+ spinner.auto_spin
25
+ end
26
+
27
+ # if spinner, capture lines to show on error
28
+ lines = []
29
+ success =
30
+ run_command("#{dump_command} | #{restore_command}") do |line|
31
+ if show_spinner
32
+ lines << line
33
+ else
34
+ log line
35
+ end
36
+ end
37
+
38
+ if show_spinner
39
+ if success
40
+ spinner.success
41
+ else
42
+ spinner.error
43
+ log lines.join
44
+ end
45
+ end
46
+
47
+ raise Error, "Schema sync returned non-zero exit code" unless success
48
+ end
49
+
50
+ private
51
+
52
+ def run_command(command)
53
+ Open3.popen2e(command) do |stdin, stdout, wait_thr|
54
+ stdout.each do |line|
55
+ yield line
56
+ end
57
+ wait_thr.value.success?
58
+ end
59
+ end
60
+
61
+ def pg_restore_version
62
+ `pg_restore --version`.lines[0].chomp.split(" ")[-1].split(/[^\d.]/)[0]
63
+ rescue Errno::ENOENT
64
+ raise Error, "pg_restore not found"
65
+ end
66
+
67
+ def dump_command
68
+ tables =
69
+ if !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude] || opts[:schemas]
70
+ @tasks.map { |task| "-t #{Shellwords.escape(task.quoted_table)}" }
71
+ else
72
+ []
73
+ end
74
+
75
+ "pg_dump -Fc --verbose --schema-only --no-owner --no-acl #{tables.join(" ")} -d #{@source.url}"
76
+ end
77
+
78
+ def restore_command
79
+ if_exists = Gem::Version.new(pg_restore_version) >= Gem::Version.new("9.4.0")
80
+ "pg_restore --verbose --no-owner --no-acl --clean #{if_exists ? "--if-exists" : nil} -d #{@destination.url}"
81
+ end
82
+ end
83
+ end
@@ -2,21 +2,26 @@ module PgSync
2
2
  class Sync
3
3
  include Utils
4
4
 
5
- def perform(options)
6
- args = options.arguments
7
- opts = options.to_hash
8
- @options = opts
5
+ def initialize(arguments, options)
6
+ @arguments = arguments
7
+ @options = options
8
+ end
9
9
 
10
- # merge config
11
- [:to, :from, :to_safe, :exclude, :schemas].each do |opt|
12
- opts[opt] ||= config[opt.to_s]
13
- end
10
+ def perform
11
+ started_at = Time.now
14
12
 
15
- # TODO remove deprecations in 0.6.0
16
- map_deprecations(args, opts)
13
+ args = @arguments
14
+ opts = @options
17
15
 
18
- # start
19
- start_time = Time.now
16
+ # only resolve commands from config, not CLI arguments
17
+ [:to, :from].each do |opt|
18
+ opts[opt] ||= resolve_source(config[opt.to_s])
19
+ end
20
+
21
+ # merge other config
22
+ [:to_safe, :exclude, :schemas].each do |opt|
23
+ opts[opt] ||= config[opt.to_s]
24
+ end
20
25
 
21
26
  if args.size > 2
22
27
  raise Error, "Usage:\n pgsync [options]"
@@ -29,92 +34,44 @@ module PgSync
29
34
  raise Error, "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1"
30
35
  end
31
36
 
37
+ if (opts[:preserve] || opts[:overwrite]) && destination.server_version_num < 90500
38
+ raise Error, "Postgres 9.5+ is required for --preserve and --overwrite"
39
+ end
40
+
32
41
  print_description("From", source)
33
42
  print_description("To", destination)
34
43
 
35
- tables = TableList.new(args, opts, source, config).tables
36
- confirm_tables_exist(source, tables, "source")
44
+ resolver = TaskResolver.new(args: args, opts: opts, source: source, destination: destination, config: config, first_schema: first_schema)
45
+ tasks =
46
+ resolver.tasks.map do |task|
47
+ Task.new(source: source, destination: destination, config: config, table: task[:table], opts: opts.merge(sql: task[:sql]))
48
+ end
37
49
 
38
- if opts[:list]
39
- confirm_tables_exist(destination, tables, "destination")
50
+ if opts[:in_batches] && tasks.size > 1
51
+ raise Error, "Cannot use --in-batches with multiple tables"
52
+ end
40
53
 
41
- list_items =
42
- if args[0] == "groups"
43
- (config["groups"] || {}).keys
44
- else
45
- tables.keys
46
- end
54
+ confirm_tables_exist(source, tasks, "source")
47
55
 
48
- pretty_list list_items
56
+ if opts[:list]
57
+ confirm_tables_exist(destination, tasks, "destination")
58
+ tasks.each do |task|
59
+ log task_name(task)
60
+ end
49
61
  else
50
62
  if opts[:schema_first] || opts[:schema_only]
51
- if opts[:preserve]
52
- raise Error, "Cannot use --preserve with --schema-first or --schema-only"
53
- end
54
-
55
- log "* Dumping schema"
56
- schema_tables = tables if !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude]
57
- sync_schema(source, destination, schema_tables)
63
+ SchemaSync.new(source: source, destination: destination, tasks: tasks, args: args, opts: opts).perform
58
64
  end
59
65
 
60
66
  unless opts[:schema_only]
61
- confirm_tables_exist(destination, tables, "destination")
62
-
63
- in_parallel(tables, first_schema: source.search_path.find { |sp| sp != "pg_catalog" }) do |table, table_opts, source, destination|
64
- TableSync.new(source: source, destination: destination).sync(config, table, opts.merge(table_opts))
65
- end
67
+ TableSync.new(source: source, destination: destination, tasks: tasks, opts: opts, resolver: resolver).perform
66
68
  end
67
69
 
68
- log_completed(start_time)
70
+ log_completed(started_at)
69
71
  end
70
72
  end
71
73
 
72
- def confirm_tables_exist(data_source, tables, description)
73
- tables.keys.each do |table|
74
- unless data_source.table_exists?(table)
75
- raise Error, "Table does not exist in #{description}: #{table}"
76
- end
77
- end
78
- end
79
-
80
- def map_deprecations(args, opts)
81
- command = args[0]
82
-
83
- case command
84
- when "schema"
85
- args.shift
86
- opts[:schema_only] = true
87
- deprecated "Use `psync --schema-only` instead"
88
- when "tables"
89
- args.shift
90
- opts[:tables] = args.shift
91
- deprecated "Use `pgsync #{opts[:tables]}` instead"
92
- when "groups"
93
- args.shift
94
- opts[:groups] = args.shift
95
- deprecated "Use `pgsync #{opts[:groups]}` instead"
96
- end
97
-
98
- if opts[:where]
99
- opts[:sql] ||= String.new
100
- opts[:sql] << " WHERE #{opts[:where]}"
101
- deprecated "Use `\"WHERE #{opts[:where]}\"` instead"
102
- end
103
-
104
- if opts[:limit]
105
- opts[:sql] ||= String.new
106
- opts[:sql] << " LIMIT #{opts[:limit]}"
107
- deprecated "Use `\"LIMIT #{opts[:limit]}\"` instead"
108
- end
109
- end
110
-
111
- def sync_schema(source, destination, tables = nil)
112
- dump_command = source.dump_command(tables)
113
- restore_command = destination.restore_command
114
- unless system("#{dump_command} | #{restore_command}")
115
- raise Error, "Schema sync returned non-zero exit code"
116
- end
117
- end
74
+ private
118
75
 
119
76
  def config
120
77
  @config ||= begin
@@ -124,6 +81,8 @@ module PgSync
124
81
  YAML.load_file(file) || {}
125
82
  rescue Psych::SyntaxError => e
126
83
  raise Error, e.message
84
+ rescue Errno::ENOENT
85
+ raise Error, "Config file not found: #{file}"
127
86
  end
128
87
  else
129
88
  {}
@@ -131,133 +90,41 @@ module PgSync
131
90
  end
132
91
  end
133
92
 
134
- def print_description(prefix, source)
135
- location = " on #{source.host}:#{source.port}" if source.host
136
- log "#{prefix}: #{source.dbname}#{location}"
137
- end
138
-
139
- def in_parallel(tables, first_schema:, &block)
140
- spinners = TTY::Spinner::Multi.new(format: :dots, output: output)
141
- item_spinners = {}
142
-
143
- start = lambda do |item, i|
144
- table, opts = item
145
- message = String.new(":spinner ")
146
- message << table.sub("#{first_schema}.", "")
147
- message << " #{opts[:sql]}" if opts[:sql]
148
- spinner = spinners.register(message)
149
- if @options[:in_batches]
150
- # log instead of spin for non-tty
151
- log message.sub(":spinner", "⠋")
152
- else
153
- spinner.auto_spin
154
- end
155
- item_spinners[item] = spinner
156
- end
157
-
158
- failed_tables = []
159
-
160
- finish = lambda do |item, i, result|
161
- spinner = item_spinners[item]
162
- table_name = item.first.sub("#{first_schema}.", "")
163
-
164
- if result[:status] == "success"
165
- spinner.success(display_message(result))
166
- else
167
- # TODO add option to fail fast
168
- spinner.error(display_message(result))
169
- failed_tables << table_name
170
- fail_sync(failed_tables) if @options[:fail_fast]
171
- end
172
-
173
- unless spinner.send(:tty?)
174
- status = result[:status] == "success" ? "✔" : "✖"
175
- log [status, table_name, item.last[:sql], display_message(result)].compact.join(" ")
176
- end
177
- end
178
-
179
- options = {start: start, finish: finish}
180
-
181
- jobs = @options[:jobs]
182
- if @options[:debug] || @options[:in_batches] || @options[:defer_constraints]
183
- warn "--jobs ignored" if jobs
184
- jobs = 0
185
- end
186
-
187
- if windows?
188
- options[:in_threads] = jobs || 4
189
- else
190
- options[:in_processes] = jobs if jobs
191
- end
192
-
193
- maybe_defer_constraints do
194
- # could try to use `raise Parallel::Kill` to fail faster with --fail-fast
195
- # see `fast_faster` branch
196
- # however, need to make sure connections are cleaned up properly
197
- Parallel.each(tables, **options) do |table, table_opts|
198
- # must reconnect for new thread or process
199
- # TODO only reconnect first time
200
- unless options[:in_processes] == 0
201
- source.reconnect
202
- destination.reconnect
203
- end
204
-
205
- # TODO warn if there are non-deferrable constraints on the table
206
-
207
- yield table, table_opts, source, destination
208
- end
209
- end
210
-
211
- fail_sync(failed_tables) if failed_tables.any?
212
- end
213
-
214
- def maybe_defer_constraints
215
- if @options[:defer_constraints]
216
- destination.transaction do
217
- destination.execute("SET CONSTRAINTS ALL DEFERRED")
218
-
219
- # create a transaction on the source
220
- # to ensure we get a consistent snapshot
221
- source.transaction do
222
- yield
223
- end
224
- end
93
+ def config_file
94
+ if @options[:config]
95
+ @options[:config]
96
+ elsif @options[:db]
97
+ file = db_config_file(@options[:db])
98
+ search_tree(file) || file
225
99
  else
226
- yield
100
+ search_tree(".pgsync.yml")
227
101
  end
228
102
  end
229
103
 
230
- def fail_sync(failed_tables)
231
- raise Error, "Sync failed for #{failed_tables.size} table#{failed_tables.size == 1 ? nil : "s"}: #{failed_tables.join(", ")}"
232
- end
233
-
234
- def display_message(result)
235
- messages = []
236
- messages << "- #{result[:time]}s" if result[:time]
237
- messages << "(#{result[:message].lines.first.to_s.strip})" if result[:message]
238
- messages.join(" ")
239
- end
104
+ def search_tree(file)
105
+ return file if File.exist?(file)
240
106
 
241
- def pretty_list(items)
242
- items.each do |item|
243
- log item
107
+ path = Dir.pwd
108
+ # prevent infinite loop
109
+ 20.times do
110
+ absolute_file = File.join(path, file)
111
+ break absolute_file if File.exist?(absolute_file)
112
+ path = File.dirname(path)
113
+ break if path == "/"
244
114
  end
245
115
  end
246
116
 
247
- def deprecated(message)
248
- log colorize("[DEPRECATED] #{message}", :yellow)
117
+ def print_description(prefix, source)
118
+ location = " on #{source.host}:#{source.port}" if source.host
119
+ log "#{prefix}: #{source.dbname}#{location}"
249
120
  end
250
121
 
251
- def log_completed(start_time)
252
- time = Time.now - start_time
122
+ def log_completed(started_at)
123
+ time = Time.now - started_at
253
124
  message = "Completed in #{time.round(1)}s"
254
125
  log colorize(message, :green)
255
126
  end
256
127
 
257
- def windows?
258
- Gem.win_platform?
259
- end
260
-
261
128
  def source
262
129
  @source ||= data_source(@options[:from])
263
130
  end
@@ -272,6 +139,21 @@ module PgSync
272
139
  ds
273
140
  end
274
141
 
142
+ def resolve_source(source)
143
+ if source
144
+ source = source.dup
145
+ source.gsub!(/\$\([^)]+\)/) do |m|
146
+ command = m[2..-2]
147
+ result = `#{command}`.chomp
148
+ unless $?.success?
149
+ raise Error, "Command exited with non-zero status:\n#{command}"
150
+ end
151
+ result
152
+ end
153
+ end
154
+ source
155
+ end
156
+
275
157
  def self.finalize(ds)
276
158
  # must use proc instead of stabby lambda
277
159
  proc { ds.close }