pgsync 0.5.2 → 0.5.3

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of pgsync might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: af75f9e9eb115a889d53b10ed89d8355e3ef111868b69eec31a025a8f22db27e
4
- data.tar.gz: 5da0d56267ac89f2ffc282e92275a91d3fb83d80fe969207329ec8ce1876472b
3
+ metadata.gz: f1c278b92d9c3c7848f655be4148555df1739194e959b8aff56ef39228c2656d
4
+ data.tar.gz: e2620166fef4fd1a785863d28796390ee4c29a7bc301a7b32800116306d9a15b
5
5
  SHA512:
6
- metadata.gz: 9f9d78aebf02f75a5fee9413b56bba68ed97334b3134cef5c28f4b5a8a15cbab45e752cb73dcd88028e2b3b2b0fefb20ef9bd4abe0c92d35998a520b0029e961
7
- data.tar.gz: a2fdaee8ba32df378ed2538119817b7649fe134a33c0f6013d55d8e1cb76ae6b0a75d3dd10dff4872d0452bc2e41bb0024c5f276f16ed8f0d1c81b114574f757
6
+ metadata.gz: bcd912263fce83a965d790eed4af30f59227b2b55e82cc46e90c109df8c1e3a6e0d7d25f29d14c946bbdc578a3817373a4eac938fe75282586d939a2fba522ea
7
+ data.tar.gz: 4606494cd7608080cf7863432eaeee6ab1a49fba221ef82da906ccce264e07003eb6458600134f398bb717e556b83339b0b2362d55d7bb62a21ee645678c653f
data/CHANGELOG.md CHANGED
@@ -1,3 +1,8 @@
1
+ ## 0.5.3 (2020-04-03)
2
+
3
+ - Improved Postgres error messages
4
+ - Fixed behavior of wildcard without schema
5
+
1
6
  ## 0.5.2 (2020-03-27)
2
7
 
3
8
  - Added `--fail-fast` option
data/exe/pgsync CHANGED
@@ -1,9 +1,5 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- # disable Slop warning with Ruby 2.7
4
- # TODO remove when Slop > 4.7.0 is released
5
- $VERBOSE = nil
6
-
7
3
  # handle interrupts
8
4
  trap("SIGINT") { abort }
9
5
 
data/lib/pgsync/client.rb CHANGED
@@ -4,12 +4,15 @@ module PgSync
4
4
 
5
5
  def initialize(args)
6
6
  @args = args
7
- $stderr.sync = true
7
+ output.sync = true
8
8
  end
9
9
 
10
10
  def perform(testing: true)
11
11
  opts = parse_args
12
12
 
13
+ # TODO throw error in 0.6.0
14
+ warn "Specify either --db or --config, not both" if opts[:db] && opts[:config]
15
+
13
16
  if opts.version?
14
17
  log VERSION
15
18
  elsif opts.help?
@@ -22,7 +25,7 @@ module PgSync
22
25
  end
23
26
  rescue Error, PG::ConnectionBad => e
24
27
  raise e if testing
25
- abort colorize(e.message, 31) # red
28
+ abort colorize(e.message, :red)
26
29
  end
27
30
 
28
31
  def self.start
data/lib/pgsync/init.rb CHANGED
@@ -12,7 +12,7 @@ module PgSync
12
12
  raise Error, "#{file} exists."
13
13
  else
14
14
  exclude =
15
- if rails_app?
15
+ if rails?
16
16
  <<~EOS
17
17
  exclude:
18
18
  - schema_migrations
@@ -21,21 +21,25 @@ module PgSync
21
21
  else
22
22
  <<~EOS
23
23
  # exclude:
24
- # - schema_migrations
25
- # - ar_internal_metadata
24
+ # - table1
25
+ # - table2
26
26
  EOS
27
27
  end
28
28
 
29
29
  # create file
30
30
  contents = File.read(__dir__ + "/../../config.yml")
31
+ contents.sub!("$(some_command)", "$(heroku config:get DATABASE_URL)") if heroku?
31
32
  File.write(file, contents % {exclude: exclude})
32
33
 
33
34
  log "#{file} created. Add your database credentials."
34
35
  end
35
36
  end
36
37
 
37
- # TODO maybe check parent directories
38
- def rails_app?
38
+ def heroku?
39
+ `git remote -v 2>&1`.include?("git.heroku.com") rescue false
40
+ end
41
+
42
+ def rails?
39
43
  File.exist?("bin/rails")
40
44
  end
41
45
  end
data/lib/pgsync/sync.rb CHANGED
@@ -138,9 +138,10 @@ module PgSync
138
138
 
139
139
  def config
140
140
  @config ||= begin
141
- if config_file
141
+ file = config_file
142
+ if file
142
143
  begin
143
- YAML.load_file(config_file) || {}
144
+ YAML.load_file(file) || {}
144
145
  rescue Psych::SyntaxError => e
145
146
  raise Error, e.message
146
147
  end
@@ -156,15 +157,14 @@ module PgSync
156
157
  end
157
158
 
158
159
  def in_parallel(tables, first_schema:, &block)
159
- spinners = TTY::Spinner::Multi.new(format: :dots)
160
+ spinners = TTY::Spinner::Multi.new(format: :dots, output: output)
160
161
  item_spinners = {}
161
162
 
162
163
  start = lambda do |item, i|
163
164
  table, opts = item
164
165
  message = String.new(":spinner ")
165
166
  message << table.sub("#{first_schema}.", "")
166
- # maybe output later
167
- # message << " #{opts[:sql]}" if opts[:sql]
167
+ message << " #{opts[:sql]}" if opts[:sql]
168
168
  spinner = spinners.register(message)
169
169
  spinner.auto_spin
170
170
  item_spinners[item] = spinner
@@ -187,7 +187,7 @@ module PgSync
187
187
 
188
188
  unless spinner.send(:tty?)
189
189
  status = result[:status] == "success" ? "✔" : "✖"
190
- log [status, table_name, display_message(result)].compact.join(" ")
190
+ log [status, table_name, item.last[:sql], display_message(result)].compact.join(" ")
191
191
  end
192
192
  end
193
193
 
@@ -213,7 +213,7 @@ module PgSync
213
213
  def display_message(result)
214
214
  messages = []
215
215
  messages << "- #{result[:time]}s" if result[:time]
216
- messages << "(#{result[:message].gsub("\n", " ").strip})" if result[:message]
216
+ messages << "(#{result[:message].lines.first.to_s.strip})" if result[:message]
217
217
  messages.join(" ")
218
218
  end
219
219
 
@@ -224,13 +224,13 @@ module PgSync
224
224
  end
225
225
 
226
226
  def deprecated(message)
227
- log colorize("[DEPRECATED] #{message}", 33) # yellow
227
+ log colorize("[DEPRECATED] #{message}", :yellow)
228
228
  end
229
229
 
230
230
  def log_completed(start_time)
231
231
  time = Time.now - start_time
232
232
  message = "Completed in #{time.round(1)}s"
233
- log colorize(message, 32) # green
233
+ log colorize(message, :green)
234
234
  end
235
235
 
236
236
  def windows?
@@ -9,49 +9,58 @@ module PgSync
9
9
  @opts = options
10
10
  @source = source
11
11
  @config = config
12
+ @groups = config["groups"] || {}
13
+ end
14
+
15
+ def group?(group)
16
+ @groups.key?(group)
12
17
  end
13
18
 
14
19
  def tables
15
- tables = nil
20
+ tables = {}
21
+ sql = args[1]
16
22
 
17
- if opts[:groups]
18
- tables ||= Hash.new { |hash, key| hash[key] = {} }
19
- specified_groups = to_arr(opts[:groups])
20
- specified_groups.map do |tag|
23
+ groups = to_arr(opts[:groups])
24
+ tables2 = to_arr(opts[:tables])
25
+
26
+ if args[0]
27
+ # could be a group, table, or mix
28
+ to_arr(args[0]).each do |tag|
21
29
  group, id = tag.split(":", 2)
22
- if (t = (config["groups"] || {})[group])
23
- add_tables(tables, t, id, args[1])
30
+ if group?(group)
31
+ groups << tag
24
32
  else
25
- raise Error, "Group not found: #{group}"
33
+ tables2 << tag
26
34
  end
27
35
  end
28
36
  end
29
37
 
30
- if opts[:tables]
31
- tables ||= Hash.new { |hash, key| hash[key] = {} }
32
- to_arr(opts[:tables]).each do |tag|
33
- table, id = tag.split(":", 2)
34
- raise Error, "Cannot use parameters with tables" if id
35
- add_table(tables, table, id, args[1])
36
- end
37
- end
38
+ groups.each do |tag|
39
+ group, id = tag.split(":", 2)
40
+ raise Error, "Group not found: #{group}" unless group?(group)
38
41
 
39
- if args[0]
40
- # could be a group, table, or mix
41
- tables ||= Hash.new { |hash, key| hash[key] = {} }
42
- specified_groups = to_arr(args[0])
43
- specified_groups.map do |tag|
44
- group, id = tag.split(":", 2)
45
- if (t = (config["groups"] || {})[group])
46
- add_tables(tables, t, id, args[1])
47
- else
48
- raise Error, "Cannot use parameters with tables" if id
49
- add_table(tables, group, id, args[1])
42
+ # if id
43
+ # # TODO show group name and value
44
+ # log colorize("`pgsync group:value` is deprecated and will have a different function in 0.6.0.", :yellow)
45
+ # log colorize("Use `pgsync group --var 1=value` instead.", :yellow)
46
+ # end
47
+
48
+ @groups[group].each do |table|
49
+ table_sql = nil
50
+ if table.is_a?(Array)
51
+ table, table_sql = table
50
52
  end
53
+ add_table(tables, table, id, sql || table_sql)
51
54
  end
52
55
  end
53
56
 
54
- tables ||= begin
57
+ tables2.each do |tag|
58
+ table, id = tag.split(":", 2)
59
+ raise Error, "Cannot use parameters with tables" if id
60
+ add_table(tables, table, id, sql)
61
+ end
62
+
63
+ if !opts[:groups] && !opts[:tables] && !args[0]
55
64
  exclude = to_arr(opts[:exclude])
56
65
  exclude = source.fully_resolve_tables(exclude).keys if exclude.any?
57
66
 
@@ -61,7 +70,9 @@ module PgSync
61
70
  tabs.select! { |t| schemas.include?(t.split(".", 2)[0]) }
62
71
  end
63
72
 
64
- Hash[(tabs - exclude).map { |k| [k, {}] }]
73
+ (tabs - exclude).each do |k|
74
+ tables[k] = {}
75
+ end
65
76
  end
66
77
 
67
78
  source.fully_resolve_tables(tables)
@@ -79,60 +90,47 @@ module PgSync
79
90
  end
80
91
  end
81
92
 
82
- def add_tables(tables, t, id, boom)
83
- # if id
84
- # # TODO show group name and value
85
- # log colorize("`pgsync group:value` is deprecated and will have a different function in 0.6.0.", 33) # yellow
86
- # log colorize("Use `pgsync group --var 1=value` instead.", 33) # yellow
87
- # end
88
-
89
- t.each do |table|
90
- sql = nil
91
- if table.is_a?(Array)
92
- table, sql = table
93
+ def add_table(tables, table, id, sql)
94
+ tables2 =
95
+ if table.include?("*")
96
+ regex = Regexp.new('\A' + Regexp.escape(table).gsub('\*','[^\.]*') + '\z')
97
+ source.tables.select { |t| regex.match(t) || regex.match(t.split(".", 2).last) }
98
+ else
99
+ [table]
93
100
  end
94
- add_table(tables, table, id, boom || sql)
101
+
102
+ tables2.each do |tab|
103
+ tables[tab] = {}
104
+ tables[tab][:sql] = table_sql(sql, id) if sql
95
105
  end
96
106
  end
97
107
 
98
- def add_table(tables, table, id, boom, wildcard = false)
99
- if table.include?("*") && !wildcard
100
- regex = Regexp.new('\A' + Regexp.escape(table).gsub('\*','[^\.]*') + '\z')
101
- t2 = source.tables.select { |t| regex.match(t) }
102
- t2.each do |tab|
103
- add_table(tables, tab, id, boom, true)
104
- end
105
- else
106
- tables[table] = {}
107
- if boom
108
- sql = boom.dup
109
- # vars must match \w
110
- missing_vars = sql.scan(/{\w+}/).map { |v| v[1..-2] }
111
-
112
- vars = {}
113
-
114
- # legacy
115
- if id
116
- vars["id"] = cast(id)
117
- vars["1"] = cast(id)
118
- end
108
+ def table_sql(sql, id)
109
+ # vars must match \w
110
+ missing_vars = sql.scan(/{\w+}/).map { |v| v[1..-2] }
119
111
 
120
- # opts[:var].each do |value|
121
- # k, v = value.split("=", 2)
122
- # vars[k] = v
123
- # end
112
+ vars = {}
124
113
 
125
- sql = boom.dup
126
- vars.each do |k, v|
127
- # only sub if in var list
128
- sql.gsub!("{#{k}}", cast(v)) if missing_vars.delete(k)
129
- end
114
+ # legacy
115
+ if id
116
+ vars["id"] = cast(id)
117
+ vars["1"] = cast(id)
118
+ end
130
119
 
131
- raise Error, "Missing variables: #{missing_vars.uniq.join(", ")}" if missing_vars.any?
120
+ # opts[:var].each do |value|
121
+ # k, v = value.split("=", 2)
122
+ # vars[k] = v
123
+ # end
132
124
 
133
- tables[table][:sql] = sql
134
- end
125
+ sql = sql.dup
126
+ vars.each do |k, v|
127
+ # only sub if in var list
128
+ sql.gsub!("{#{k}}", cast(v)) if missing_vars.delete(k)
135
129
  end
130
+
131
+ raise Error, "Missing variables: #{missing_vars.uniq.join(", ")}" if missing_vars.any?
132
+
133
+ sql
136
134
  end
137
135
 
138
136
  # TODO quote vars in next major version
@@ -4,173 +4,150 @@ module PgSync
4
4
 
5
5
  def sync(config, table, opts, source_url, destination_url)
6
6
  start_time = Time.now
7
+
7
8
  source = DataSource.new(source_url, timeout: 0)
8
9
  destination = DataSource.new(destination_url, timeout: 0)
9
10
 
10
- begin
11
- from_connection = source.conn
12
- to_connection = destination.conn
13
-
14
- bad_fields = opts[:no_rules] ? [] : config["data_rules"]
15
-
16
- from_fields = source.columns(table)
17
- to_fields = destination.columns(table)
18
- shared_fields = to_fields & from_fields
19
- extra_fields = to_fields - from_fields
20
- missing_fields = from_fields - to_fields
11
+ from_connection = source.conn
12
+ to_connection = destination.conn
21
13
 
22
- if opts[:no_sequences]
23
- from_sequences = []
24
- to_sequences = []
25
- else
26
- from_sequences = source.sequences(table, shared_fields)
27
- to_sequences = destination.sequences(table, shared_fields)
28
- end
14
+ from_fields = source.columns(table)
15
+ to_fields = destination.columns(table)
16
+ shared_fields = to_fields & from_fields
17
+ extra_fields = to_fields - from_fields
18
+ missing_fields = from_fields - to_fields
29
19
 
30
- shared_sequences = to_sequences & from_sequences
31
- extra_sequences = to_sequences - from_sequences
32
- missing_sequences = from_sequences - to_sequences
20
+ if opts[:no_sequences]
21
+ from_sequences = []
22
+ to_sequences = []
23
+ else
24
+ from_sequences = source.sequences(table, shared_fields)
25
+ to_sequences = destination.sequences(table, shared_fields)
26
+ end
33
27
 
34
- sql_clause = String.new
28
+ shared_sequences = to_sequences & from_sequences
29
+ extra_sequences = to_sequences - from_sequences
30
+ missing_sequences = from_sequences - to_sequences
35
31
 
36
- if opts[:sql]
37
- sql_clause << " #{opts[:sql]}"
38
- end
32
+ sql_clause = String.new("")
33
+ sql_clause << " #{opts[:sql]}" if opts[:sql]
39
34
 
40
- notes = []
41
- notes << "Extra columns: #{extra_fields.join(", ")}" if extra_fields.any?
42
- notes << "Missing columns: #{missing_fields.join(", ")}" if missing_fields.any?
43
- notes << "Extra sequences: #{extra_sequences.join(", ")}" if extra_sequences.any?
44
- notes << "Missing sequences: #{missing_sequences.join(", ")}" if missing_sequences.any?
35
+ notes = []
36
+ notes << "Extra columns: #{extra_fields.join(", ")}" if extra_fields.any?
37
+ notes << "Missing columns: #{missing_fields.join(", ")}" if missing_fields.any?
38
+ notes << "Extra sequences: #{extra_sequences.join(", ")}" if extra_sequences.any?
39
+ notes << "Missing sequences: #{missing_sequences.join(", ")}" if missing_sequences.any?
45
40
 
46
- if shared_fields.empty?
47
- return {status: "success", message: "No fields to copy"}
48
- end
41
+ return {status: "success", message: "No fields to copy"} if shared_fields.empty?
49
42
 
50
- if shared_fields.any?
51
- primary_key = destination.primary_key(table)
52
- copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, _| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], table, f, primary_key)} AS #{quote_ident(f)}" : "#{quote_ident_full(table)}.#{quote_ident(f)}" }.join(", ")
53
- fields = shared_fields.map { |f| quote_ident(f) }.join(", ")
43
+ bad_fields = opts[:no_rules] ? [] : config["data_rules"]
44
+ primary_key = destination.primary_key(table)
45
+ copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, _| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], table, f, primary_key)} AS #{quote_ident(f)}" : "#{quote_ident_full(table)}.#{quote_ident(f)}" }.join(", ")
46
+ fields = shared_fields.map { |f| quote_ident(f) }.join(", ")
54
47
 
55
- seq_values = {}
56
- shared_sequences.each do |seq|
57
- seq_values[seq] = source.last_value(seq)
58
- end
48
+ seq_values = {}
49
+ shared_sequences.each do |seq|
50
+ seq_values[seq] = source.last_value(seq)
51
+ end
59
52
 
60
- copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quote_ident_full(table)}#{sql_clause}) TO STDOUT"
61
- if opts[:in_batches]
62
- raise Error, "Cannot use --overwrite with --in-batches" if opts[:overwrite]
63
- raise Error, "No primary key" unless primary_key
53
+ copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quote_ident_full(table)}#{sql_clause}) TO STDOUT"
54
+ if opts[:in_batches]
55
+ raise Error, "Cannot use --overwrite with --in-batches" if opts[:overwrite]
56
+ raise Error, "No primary key" unless primary_key
64
57
 
65
- destination.truncate(table) if opts[:truncate]
58
+ destination.truncate(table) if opts[:truncate]
66
59
 
67
- from_max_id = source.max_id(table, primary_key)
68
- to_max_id = destination.max_id(table, primary_key) + 1
60
+ from_max_id = source.max_id(table, primary_key)
61
+ to_max_id = destination.max_id(table, primary_key) + 1
69
62
 
70
- if to_max_id == 1
71
- from_min_id = source.min_id(table, primary_key)
72
- to_max_id = from_min_id if from_min_id > 0
73
- end
63
+ if to_max_id == 1
64
+ from_min_id = source.min_id(table, primary_key)
65
+ to_max_id = from_min_id if from_min_id > 0
66
+ end
74
67
 
75
- starting_id = to_max_id
76
- batch_size = opts[:batch_size]
68
+ starting_id = to_max_id
69
+ batch_size = opts[:batch_size]
77
70
 
78
- i = 1
79
- batch_count = ((from_max_id - starting_id + 1) / batch_size.to_f).ceil
71
+ i = 1
72
+ batch_count = ((from_max_id - starting_id + 1) / batch_size.to_f).ceil
80
73
 
81
- while starting_id <= from_max_id
82
- where = "#{quote_ident(primary_key)} >= #{starting_id} AND #{quote_ident(primary_key)} < #{starting_id + batch_size}"
83
- log " #{i}/#{batch_count}: #{where}"
74
+ while starting_id <= from_max_id
75
+ where = "#{quote_ident(primary_key)} >= #{starting_id} AND #{quote_ident(primary_key)} < #{starting_id + batch_size}"
76
+ log " #{i}/#{batch_count}: #{where}"
84
77
 
85
- # TODO be smarter for advance sql clauses
86
- batch_sql_clause = " #{sql_clause.length > 0 ? "#{sql_clause} AND" : "WHERE"} #{where}"
78
+ # TODO be smarter for advance sql clauses
79
+ batch_sql_clause = " #{sql_clause.length > 0 ? "#{sql_clause} AND" : "WHERE"} #{where}"
87
80
 
88
- batch_copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quote_ident_full(table)}#{batch_sql_clause}) TO STDOUT"
89
- to_connection.copy_data "COPY #{quote_ident_full(table)} (#{fields}) FROM STDIN" do
90
- from_connection.copy_data batch_copy_to_command do
91
- while (row = from_connection.get_copy_data)
92
- to_connection.put_copy_data(row)
93
- end
94
- end
95
- end
96
-
97
- starting_id += batch_size
98
- i += 1
99
-
100
- if opts[:sleep] && starting_id <= from_max_id
101
- sleep(opts[:sleep])
81
+ batch_copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quote_ident_full(table)}#{batch_sql_clause}) TO STDOUT"
82
+ to_connection.copy_data "COPY #{quote_ident_full(table)} (#{fields}) FROM STDIN" do
83
+ from_connection.copy_data batch_copy_to_command do
84
+ while (row = from_connection.get_copy_data)
85
+ to_connection.put_copy_data(row)
102
86
  end
103
87
  end
88
+ end
104
89
 
105
- log # add extra line for spinner
106
- elsif !opts[:truncate] && (opts[:overwrite] || opts[:preserve] || !sql_clause.empty?)
107
- raise Error, "No primary key" unless primary_key
108
-
109
- temp_table = "pgsync_#{rand(1_000_000_000)}"
110
- file = Tempfile.new(temp_table)
111
- begin
112
- from_connection.copy_data copy_to_command do
113
- while (row = from_connection.get_copy_data)
114
- file.write(row)
115
- end
116
- end
117
- file.rewind
90
+ starting_id += batch_size
91
+ i += 1
92
+
93
+ if opts[:sleep] && starting_id <= from_max_id
94
+ sleep(opts[:sleep])
95
+ end
96
+ end
118
97
 
119
- # create a temp table
120
- to_connection.exec("CREATE TEMPORARY TABLE #{quote_ident_full(temp_table)} AS SELECT * FROM #{quote_ident_full(table)} WITH NO DATA")
98
+ log # add extra line for spinner
99
+ elsif !opts[:truncate] && (opts[:overwrite] || opts[:preserve] || !sql_clause.empty?)
100
+ raise Error, "No primary key" unless primary_key
121
101
 
122
- # load file
123
- to_connection.copy_data "COPY #{quote_ident_full(temp_table)} (#{fields}) FROM STDIN" do
124
- file.each do |row|
125
- to_connection.put_copy_data(row)
126
- end
127
- end
102
+ # create a temp table
103
+ temp_table = "pgsync_#{rand(1_000_000_000)}"
104
+ to_connection.exec("CREATE TEMPORARY TABLE #{quote_ident_full(temp_table)} AS SELECT * FROM #{quote_ident_full(table)} WITH NO DATA")
128
105
 
129
- if opts[:preserve]
130
- # insert into
131
- to_connection.exec("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident_full(temp_table)} WHERE NOT EXISTS (SELECT 1 FROM #{quote_ident_full(table)} WHERE #{quote_ident_full(table)}.#{quote_ident(primary_key)} = #{quote_ident_full(temp_table)}.#{quote_ident(primary_key)}))")
132
- else
133
- to_connection.transaction do
134
- to_connection.exec("DELETE FROM #{quote_ident_full(table)} WHERE #{quote_ident(primary_key)} IN (SELECT #{quote_ident(primary_key)} FROM #{quote_ident_full(temp_table)})")
135
- to_connection.exec("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident(temp_table)})")
136
- end
137
- end
138
- ensure
139
- file.close
140
- file.unlink
141
- end
142
- else
143
- destination.truncate(table)
144
- to_connection.copy_data "COPY #{quote_ident_full(table)} (#{fields}) FROM STDIN" do
145
- from_connection.copy_data copy_to_command do
146
- while (row = from_connection.get_copy_data)
147
- to_connection.put_copy_data(row)
148
- end
149
- end
106
+ # load data
107
+ to_connection.copy_data "COPY #{quote_ident_full(temp_table)} (#{fields}) FROM STDIN" do
108
+ from_connection.copy_data copy_to_command do
109
+ while (row = from_connection.get_copy_data)
110
+ to_connection.put_copy_data(row)
150
111
  end
151
112
  end
152
- seq_values.each do |seq, value|
153
- to_connection.exec("SELECT setval(#{escape(seq)}, #{escape(value)})")
154
- end
155
113
  end
156
114
 
157
- message = nil
158
- if notes.any?
159
- message = notes.join(", ")
115
+ if opts[:preserve]
116
+ # insert into
117
+ to_connection.exec("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident_full(temp_table)} WHERE NOT EXISTS (SELECT 1 FROM #{quote_ident_full(table)} WHERE #{quote_ident_full(table)}.#{quote_ident(primary_key)} = #{quote_ident_full(temp_table)}.#{quote_ident(primary_key)}))")
118
+ else
119
+ to_connection.transaction do
120
+ to_connection.exec("DELETE FROM #{quote_ident_full(table)} WHERE #{quote_ident(primary_key)} IN (SELECT #{quote_ident(primary_key)} FROM #{quote_ident_full(temp_table)})")
121
+ to_connection.exec("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident(temp_table)})")
122
+ end
160
123
  end
161
-
162
- {status: "success", message: message, time: (Time.now - start_time).round(1)}
163
- ensure
164
- source.close
165
- destination.close
124
+ else
125
+ destination.truncate(table)
126
+ to_connection.copy_data "COPY #{quote_ident_full(table)} (#{fields}) FROM STDIN" do
127
+ from_connection.copy_data copy_to_command do
128
+ while (row = from_connection.get_copy_data)
129
+ to_connection.put_copy_data(row)
130
+ end
131
+ end
132
+ end
133
+ end
134
+ seq_values.each do |seq, value|
135
+ to_connection.exec("SELECT setval(#{escape(seq)}, #{escape(value)})")
166
136
  end
137
+
138
+ message = nil
139
+ message = notes.join(", ") if notes.any?
140
+
141
+ {status: "success", message: message, time: (Time.now - start_time).round(1)}
167
142
  rescue => e
168
143
  message =
169
144
  case e
170
- when PG::Error
145
+ when PG::ConnectionBad
171
146
  # likely fine to show simplified message here
172
147
  # the full message will be shown when first trying to connect
173
148
  "Connection failed"
149
+ when PG::Error
150
+ e.message.sub("ERROR: ", "")
174
151
  when Error
175
152
  e.message
176
153
  else
@@ -178,6 +155,9 @@ module PgSync
178
155
  end
179
156
 
180
157
  {status: "error", message: message}
158
+ ensure
159
+ source.close if source
160
+ destination.close if destination
181
161
  end
182
162
 
183
163
  private
data/lib/pgsync/utils.rb CHANGED
@@ -1,43 +1,43 @@
1
1
  module PgSync
2
2
  module Utils
3
+ COLOR_CODES = {
4
+ red: 31,
5
+ green: 32,
6
+ yellow: 33
7
+ }
8
+
3
9
  def log(message = nil)
4
- $stderr.puts message
10
+ output.puts message
5
11
  end
6
12
 
7
- def colorize(message, color_code)
8
- if $stderr.tty?
9
- "\e[#{color_code}m#{message}\e[0m"
13
+ def colorize(message, color)
14
+ if output.tty?
15
+ "\e[#{COLOR_CODES[color]}m#{message}\e[0m"
10
16
  else
11
17
  message
12
18
  end
13
19
  end
14
20
 
15
- def config_file
16
- return @config_file if instance_variable_defined?(:@config_file)
21
+ def output
22
+ $stderr
23
+ end
17
24
 
18
- @config_file =
19
- search_tree(
20
- if @options[:db]
21
- db_config_file(@options[:db])
22
- else
23
- @options[:config] || ".pgsync.yml"
24
- end
25
- )
25
+ def config_file
26
+ search_tree(db_config_file(@options[:db]) || @options[:config] || ".pgsync.yml")
26
27
  end
27
28
 
28
29
  def db_config_file(db)
29
- return unless db
30
- ".pgsync-#{db}.yml"
30
+ ".pgsync-#{db}.yml" if db
31
31
  end
32
32
 
33
33
  def search_tree(file)
34
+ return file if File.exists?(file)
35
+
34
36
  path = Dir.pwd
35
37
  # prevent infinite loop
36
38
  20.times do
37
39
  absolute_file = File.join(path, file)
38
- if File.exist?(absolute_file)
39
- break absolute_file
40
- end
40
+ break absolute_file if File.exist?(absolute_file)
41
41
  path = File.dirname(path)
42
42
  break if path == "/"
43
43
  end
@@ -1,3 +1,3 @@
1
1
  module PgSync
2
- VERSION = "0.5.2"
2
+ VERSION = "0.5.3"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pgsync
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.2
4
+ version: 0.5.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Kane
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-03-28 00:00:00.000000000 Z
11
+ date: 2020-04-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: parallel
@@ -44,14 +44,14 @@ dependencies:
44
44
  requirements:
45
45
  - - ">="
46
46
  - !ruby/object:Gem::Version
47
- version: 4.2.0
47
+ version: 4.8.1
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: 4.2.0
54
+ version: 4.8.1
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: tty-spinner
57
57
  requirement: !ruby/object:Gem::Requirement