pgsync 0.5.2 → 0.5.3
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of pgsync might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/exe/pgsync +0 -4
- data/lib/pgsync/client.rb +5 -2
- data/lib/pgsync/init.rb +9 -5
- data/lib/pgsync/sync.rb +9 -9
- data/lib/pgsync/table_list.rb +72 -74
- data/lib/pgsync/table_sync.rb +109 -129
- data/lib/pgsync/utils.rb +19 -19
- data/lib/pgsync/version.rb +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f1c278b92d9c3c7848f655be4148555df1739194e959b8aff56ef39228c2656d
|
4
|
+
data.tar.gz: e2620166fef4fd1a785863d28796390ee4c29a7bc301a7b32800116306d9a15b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bcd912263fce83a965d790eed4af30f59227b2b55e82cc46e90c109df8c1e3a6e0d7d25f29d14c946bbdc578a3817373a4eac938fe75282586d939a2fba522ea
|
7
|
+
data.tar.gz: 4606494cd7608080cf7863432eaeee6ab1a49fba221ef82da906ccce264e07003eb6458600134f398bb717e556b83339b0b2362d55d7bb62a21ee645678c653f
|
data/CHANGELOG.md
CHANGED
data/exe/pgsync
CHANGED
data/lib/pgsync/client.rb
CHANGED
@@ -4,12 +4,15 @@ module PgSync
|
|
4
4
|
|
5
5
|
def initialize(args)
|
6
6
|
@args = args
|
7
|
-
|
7
|
+
output.sync = true
|
8
8
|
end
|
9
9
|
|
10
10
|
def perform(testing: true)
|
11
11
|
opts = parse_args
|
12
12
|
|
13
|
+
# TODO throw error in 0.6.0
|
14
|
+
warn "Specify either --db or --config, not both" if opts[:db] && opts[:config]
|
15
|
+
|
13
16
|
if opts.version?
|
14
17
|
log VERSION
|
15
18
|
elsif opts.help?
|
@@ -22,7 +25,7 @@ module PgSync
|
|
22
25
|
end
|
23
26
|
rescue Error, PG::ConnectionBad => e
|
24
27
|
raise e if testing
|
25
|
-
abort colorize(e.message,
|
28
|
+
abort colorize(e.message, :red)
|
26
29
|
end
|
27
30
|
|
28
31
|
def self.start
|
data/lib/pgsync/init.rb
CHANGED
@@ -12,7 +12,7 @@ module PgSync
|
|
12
12
|
raise Error, "#{file} exists."
|
13
13
|
else
|
14
14
|
exclude =
|
15
|
-
if
|
15
|
+
if rails?
|
16
16
|
<<~EOS
|
17
17
|
exclude:
|
18
18
|
- schema_migrations
|
@@ -21,21 +21,25 @@ module PgSync
|
|
21
21
|
else
|
22
22
|
<<~EOS
|
23
23
|
# exclude:
|
24
|
-
# -
|
25
|
-
# -
|
24
|
+
# - table1
|
25
|
+
# - table2
|
26
26
|
EOS
|
27
27
|
end
|
28
28
|
|
29
29
|
# create file
|
30
30
|
contents = File.read(__dir__ + "/../../config.yml")
|
31
|
+
contents.sub!("$(some_command)", "$(heroku config:get DATABASE_URL)") if heroku?
|
31
32
|
File.write(file, contents % {exclude: exclude})
|
32
33
|
|
33
34
|
log "#{file} created. Add your database credentials."
|
34
35
|
end
|
35
36
|
end
|
36
37
|
|
37
|
-
|
38
|
-
|
38
|
+
def heroku?
|
39
|
+
`git remote -v 2>&1`.include?("git.heroku.com") rescue false
|
40
|
+
end
|
41
|
+
|
42
|
+
def rails?
|
39
43
|
File.exist?("bin/rails")
|
40
44
|
end
|
41
45
|
end
|
data/lib/pgsync/sync.rb
CHANGED
@@ -138,9 +138,10 @@ module PgSync
|
|
138
138
|
|
139
139
|
def config
|
140
140
|
@config ||= begin
|
141
|
-
|
141
|
+
file = config_file
|
142
|
+
if file
|
142
143
|
begin
|
143
|
-
YAML.load_file(
|
144
|
+
YAML.load_file(file) || {}
|
144
145
|
rescue Psych::SyntaxError => e
|
145
146
|
raise Error, e.message
|
146
147
|
end
|
@@ -156,15 +157,14 @@ module PgSync
|
|
156
157
|
end
|
157
158
|
|
158
159
|
def in_parallel(tables, first_schema:, &block)
|
159
|
-
spinners = TTY::Spinner::Multi.new(format: :dots)
|
160
|
+
spinners = TTY::Spinner::Multi.new(format: :dots, output: output)
|
160
161
|
item_spinners = {}
|
161
162
|
|
162
163
|
start = lambda do |item, i|
|
163
164
|
table, opts = item
|
164
165
|
message = String.new(":spinner ")
|
165
166
|
message << table.sub("#{first_schema}.", "")
|
166
|
-
#
|
167
|
-
# message << " #{opts[:sql]}" if opts[:sql]
|
167
|
+
message << " #{opts[:sql]}" if opts[:sql]
|
168
168
|
spinner = spinners.register(message)
|
169
169
|
spinner.auto_spin
|
170
170
|
item_spinners[item] = spinner
|
@@ -187,7 +187,7 @@ module PgSync
|
|
187
187
|
|
188
188
|
unless spinner.send(:tty?)
|
189
189
|
status = result[:status] == "success" ? "✔" : "✖"
|
190
|
-
log [status, table_name, display_message(result)].compact.join(" ")
|
190
|
+
log [status, table_name, item.last[:sql], display_message(result)].compact.join(" ")
|
191
191
|
end
|
192
192
|
end
|
193
193
|
|
@@ -213,7 +213,7 @@ module PgSync
|
|
213
213
|
def display_message(result)
|
214
214
|
messages = []
|
215
215
|
messages << "- #{result[:time]}s" if result[:time]
|
216
|
-
messages << "(#{result[:message].
|
216
|
+
messages << "(#{result[:message].lines.first.to_s.strip})" if result[:message]
|
217
217
|
messages.join(" ")
|
218
218
|
end
|
219
219
|
|
@@ -224,13 +224,13 @@ module PgSync
|
|
224
224
|
end
|
225
225
|
|
226
226
|
def deprecated(message)
|
227
|
-
log colorize("[DEPRECATED] #{message}",
|
227
|
+
log colorize("[DEPRECATED] #{message}", :yellow)
|
228
228
|
end
|
229
229
|
|
230
230
|
def log_completed(start_time)
|
231
231
|
time = Time.now - start_time
|
232
232
|
message = "Completed in #{time.round(1)}s"
|
233
|
-
log colorize(message,
|
233
|
+
log colorize(message, :green)
|
234
234
|
end
|
235
235
|
|
236
236
|
def windows?
|
data/lib/pgsync/table_list.rb
CHANGED
@@ -9,49 +9,58 @@ module PgSync
|
|
9
9
|
@opts = options
|
10
10
|
@source = source
|
11
11
|
@config = config
|
12
|
+
@groups = config["groups"] || {}
|
13
|
+
end
|
14
|
+
|
15
|
+
def group?(group)
|
16
|
+
@groups.key?(group)
|
12
17
|
end
|
13
18
|
|
14
19
|
def tables
|
15
|
-
tables =
|
20
|
+
tables = {}
|
21
|
+
sql = args[1]
|
16
22
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
23
|
+
groups = to_arr(opts[:groups])
|
24
|
+
tables2 = to_arr(opts[:tables])
|
25
|
+
|
26
|
+
if args[0]
|
27
|
+
# could be a group, table, or mix
|
28
|
+
to_arr(args[0]).each do |tag|
|
21
29
|
group, id = tag.split(":", 2)
|
22
|
-
if (
|
23
|
-
|
30
|
+
if group?(group)
|
31
|
+
groups << tag
|
24
32
|
else
|
25
|
-
|
33
|
+
tables2 << tag
|
26
34
|
end
|
27
35
|
end
|
28
36
|
end
|
29
37
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
table, id = tag.split(":", 2)
|
34
|
-
raise Error, "Cannot use parameters with tables" if id
|
35
|
-
add_table(tables, table, id, args[1])
|
36
|
-
end
|
37
|
-
end
|
38
|
+
groups.each do |tag|
|
39
|
+
group, id = tag.split(":", 2)
|
40
|
+
raise Error, "Group not found: #{group}" unless group?(group)
|
38
41
|
|
39
|
-
|
40
|
-
#
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
add_table(tables, group, id, args[1])
|
42
|
+
# if id
|
43
|
+
# # TODO show group name and value
|
44
|
+
# log colorize("`pgsync group:value` is deprecated and will have a different function in 0.6.0.", :yellow)
|
45
|
+
# log colorize("Use `pgsync group --var 1=value` instead.", :yellow)
|
46
|
+
# end
|
47
|
+
|
48
|
+
@groups[group].each do |table|
|
49
|
+
table_sql = nil
|
50
|
+
if table.is_a?(Array)
|
51
|
+
table, table_sql = table
|
50
52
|
end
|
53
|
+
add_table(tables, table, id, sql || table_sql)
|
51
54
|
end
|
52
55
|
end
|
53
56
|
|
54
|
-
|
57
|
+
tables2.each do |tag|
|
58
|
+
table, id = tag.split(":", 2)
|
59
|
+
raise Error, "Cannot use parameters with tables" if id
|
60
|
+
add_table(tables, table, id, sql)
|
61
|
+
end
|
62
|
+
|
63
|
+
if !opts[:groups] && !opts[:tables] && !args[0]
|
55
64
|
exclude = to_arr(opts[:exclude])
|
56
65
|
exclude = source.fully_resolve_tables(exclude).keys if exclude.any?
|
57
66
|
|
@@ -61,7 +70,9 @@ module PgSync
|
|
61
70
|
tabs.select! { |t| schemas.include?(t.split(".", 2)[0]) }
|
62
71
|
end
|
63
72
|
|
64
|
-
|
73
|
+
(tabs - exclude).each do |k|
|
74
|
+
tables[k] = {}
|
75
|
+
end
|
65
76
|
end
|
66
77
|
|
67
78
|
source.fully_resolve_tables(tables)
|
@@ -79,60 +90,47 @@ module PgSync
|
|
79
90
|
end
|
80
91
|
end
|
81
92
|
|
82
|
-
def
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
t.each do |table|
|
90
|
-
sql = nil
|
91
|
-
if table.is_a?(Array)
|
92
|
-
table, sql = table
|
93
|
+
def add_table(tables, table, id, sql)
|
94
|
+
tables2 =
|
95
|
+
if table.include?("*")
|
96
|
+
regex = Regexp.new('\A' + Regexp.escape(table).gsub('\*','[^\.]*') + '\z')
|
97
|
+
source.tables.select { |t| regex.match(t) || regex.match(t.split(".", 2).last) }
|
98
|
+
else
|
99
|
+
[table]
|
93
100
|
end
|
94
|
-
|
101
|
+
|
102
|
+
tables2.each do |tab|
|
103
|
+
tables[tab] = {}
|
104
|
+
tables[tab][:sql] = table_sql(sql, id) if sql
|
95
105
|
end
|
96
106
|
end
|
97
107
|
|
98
|
-
def
|
99
|
-
|
100
|
-
|
101
|
-
t2 = source.tables.select { |t| regex.match(t) }
|
102
|
-
t2.each do |tab|
|
103
|
-
add_table(tables, tab, id, boom, true)
|
104
|
-
end
|
105
|
-
else
|
106
|
-
tables[table] = {}
|
107
|
-
if boom
|
108
|
-
sql = boom.dup
|
109
|
-
# vars must match \w
|
110
|
-
missing_vars = sql.scan(/{\w+}/).map { |v| v[1..-2] }
|
111
|
-
|
112
|
-
vars = {}
|
113
|
-
|
114
|
-
# legacy
|
115
|
-
if id
|
116
|
-
vars["id"] = cast(id)
|
117
|
-
vars["1"] = cast(id)
|
118
|
-
end
|
108
|
+
def table_sql(sql, id)
|
109
|
+
# vars must match \w
|
110
|
+
missing_vars = sql.scan(/{\w+}/).map { |v| v[1..-2] }
|
119
111
|
|
120
|
-
|
121
|
-
# k, v = value.split("=", 2)
|
122
|
-
# vars[k] = v
|
123
|
-
# end
|
112
|
+
vars = {}
|
124
113
|
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
114
|
+
# legacy
|
115
|
+
if id
|
116
|
+
vars["id"] = cast(id)
|
117
|
+
vars["1"] = cast(id)
|
118
|
+
end
|
130
119
|
|
131
|
-
|
120
|
+
# opts[:var].each do |value|
|
121
|
+
# k, v = value.split("=", 2)
|
122
|
+
# vars[k] = v
|
123
|
+
# end
|
132
124
|
|
133
|
-
|
134
|
-
|
125
|
+
sql = sql.dup
|
126
|
+
vars.each do |k, v|
|
127
|
+
# only sub if in var list
|
128
|
+
sql.gsub!("{#{k}}", cast(v)) if missing_vars.delete(k)
|
135
129
|
end
|
130
|
+
|
131
|
+
raise Error, "Missing variables: #{missing_vars.uniq.join(", ")}" if missing_vars.any?
|
132
|
+
|
133
|
+
sql
|
136
134
|
end
|
137
135
|
|
138
136
|
# TODO quote vars in next major version
|
data/lib/pgsync/table_sync.rb
CHANGED
@@ -4,173 +4,150 @@ module PgSync
|
|
4
4
|
|
5
5
|
def sync(config, table, opts, source_url, destination_url)
|
6
6
|
start_time = Time.now
|
7
|
+
|
7
8
|
source = DataSource.new(source_url, timeout: 0)
|
8
9
|
destination = DataSource.new(destination_url, timeout: 0)
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
to_connection = destination.conn
|
13
|
-
|
14
|
-
bad_fields = opts[:no_rules] ? [] : config["data_rules"]
|
15
|
-
|
16
|
-
from_fields = source.columns(table)
|
17
|
-
to_fields = destination.columns(table)
|
18
|
-
shared_fields = to_fields & from_fields
|
19
|
-
extra_fields = to_fields - from_fields
|
20
|
-
missing_fields = from_fields - to_fields
|
11
|
+
from_connection = source.conn
|
12
|
+
to_connection = destination.conn
|
21
13
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
to_sequences = destination.sequences(table, shared_fields)
|
28
|
-
end
|
14
|
+
from_fields = source.columns(table)
|
15
|
+
to_fields = destination.columns(table)
|
16
|
+
shared_fields = to_fields & from_fields
|
17
|
+
extra_fields = to_fields - from_fields
|
18
|
+
missing_fields = from_fields - to_fields
|
29
19
|
|
30
|
-
|
31
|
-
|
32
|
-
|
20
|
+
if opts[:no_sequences]
|
21
|
+
from_sequences = []
|
22
|
+
to_sequences = []
|
23
|
+
else
|
24
|
+
from_sequences = source.sequences(table, shared_fields)
|
25
|
+
to_sequences = destination.sequences(table, shared_fields)
|
26
|
+
end
|
33
27
|
|
34
|
-
|
28
|
+
shared_sequences = to_sequences & from_sequences
|
29
|
+
extra_sequences = to_sequences - from_sequences
|
30
|
+
missing_sequences = from_sequences - to_sequences
|
35
31
|
|
36
|
-
|
37
|
-
|
38
|
-
end
|
32
|
+
sql_clause = String.new("")
|
33
|
+
sql_clause << " #{opts[:sql]}" if opts[:sql]
|
39
34
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
35
|
+
notes = []
|
36
|
+
notes << "Extra columns: #{extra_fields.join(", ")}" if extra_fields.any?
|
37
|
+
notes << "Missing columns: #{missing_fields.join(", ")}" if missing_fields.any?
|
38
|
+
notes << "Extra sequences: #{extra_sequences.join(", ")}" if extra_sequences.any?
|
39
|
+
notes << "Missing sequences: #{missing_sequences.join(", ")}" if missing_sequences.any?
|
45
40
|
|
46
|
-
|
47
|
-
return {status: "success", message: "No fields to copy"}
|
48
|
-
end
|
41
|
+
return {status: "success", message: "No fields to copy"} if shared_fields.empty?
|
49
42
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
43
|
+
bad_fields = opts[:no_rules] ? [] : config["data_rules"]
|
44
|
+
primary_key = destination.primary_key(table)
|
45
|
+
copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, _| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], table, f, primary_key)} AS #{quote_ident(f)}" : "#{quote_ident_full(table)}.#{quote_ident(f)}" }.join(", ")
|
46
|
+
fields = shared_fields.map { |f| quote_ident(f) }.join(", ")
|
54
47
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
48
|
+
seq_values = {}
|
49
|
+
shared_sequences.each do |seq|
|
50
|
+
seq_values[seq] = source.last_value(seq)
|
51
|
+
end
|
59
52
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
53
|
+
copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quote_ident_full(table)}#{sql_clause}) TO STDOUT"
|
54
|
+
if opts[:in_batches]
|
55
|
+
raise Error, "Cannot use --overwrite with --in-batches" if opts[:overwrite]
|
56
|
+
raise Error, "No primary key" unless primary_key
|
64
57
|
|
65
|
-
|
58
|
+
destination.truncate(table) if opts[:truncate]
|
66
59
|
|
67
|
-
|
68
|
-
|
60
|
+
from_max_id = source.max_id(table, primary_key)
|
61
|
+
to_max_id = destination.max_id(table, primary_key) + 1
|
69
62
|
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
63
|
+
if to_max_id == 1
|
64
|
+
from_min_id = source.min_id(table, primary_key)
|
65
|
+
to_max_id = from_min_id if from_min_id > 0
|
66
|
+
end
|
74
67
|
|
75
|
-
|
76
|
-
|
68
|
+
starting_id = to_max_id
|
69
|
+
batch_size = opts[:batch_size]
|
77
70
|
|
78
|
-
|
79
|
-
|
71
|
+
i = 1
|
72
|
+
batch_count = ((from_max_id - starting_id + 1) / batch_size.to_f).ceil
|
80
73
|
|
81
|
-
|
82
|
-
|
83
|
-
|
74
|
+
while starting_id <= from_max_id
|
75
|
+
where = "#{quote_ident(primary_key)} >= #{starting_id} AND #{quote_ident(primary_key)} < #{starting_id + batch_size}"
|
76
|
+
log " #{i}/#{batch_count}: #{where}"
|
84
77
|
|
85
|
-
|
86
|
-
|
78
|
+
# TODO be smarter for advance sql clauses
|
79
|
+
batch_sql_clause = " #{sql_clause.length > 0 ? "#{sql_clause} AND" : "WHERE"} #{where}"
|
87
80
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
|
97
|
-
starting_id += batch_size
|
98
|
-
i += 1
|
99
|
-
|
100
|
-
if opts[:sleep] && starting_id <= from_max_id
|
101
|
-
sleep(opts[:sleep])
|
81
|
+
batch_copy_to_command = "COPY (SELECT #{copy_fields} FROM #{quote_ident_full(table)}#{batch_sql_clause}) TO STDOUT"
|
82
|
+
to_connection.copy_data "COPY #{quote_ident_full(table)} (#{fields}) FROM STDIN" do
|
83
|
+
from_connection.copy_data batch_copy_to_command do
|
84
|
+
while (row = from_connection.get_copy_data)
|
85
|
+
to_connection.put_copy_data(row)
|
102
86
|
end
|
103
87
|
end
|
88
|
+
end
|
104
89
|
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
from_connection.copy_data copy_to_command do
|
113
|
-
while (row = from_connection.get_copy_data)
|
114
|
-
file.write(row)
|
115
|
-
end
|
116
|
-
end
|
117
|
-
file.rewind
|
90
|
+
starting_id += batch_size
|
91
|
+
i += 1
|
92
|
+
|
93
|
+
if opts[:sleep] && starting_id <= from_max_id
|
94
|
+
sleep(opts[:sleep])
|
95
|
+
end
|
96
|
+
end
|
118
97
|
|
119
|
-
|
120
|
-
|
98
|
+
log # add extra line for spinner
|
99
|
+
elsif !opts[:truncate] && (opts[:overwrite] || opts[:preserve] || !sql_clause.empty?)
|
100
|
+
raise Error, "No primary key" unless primary_key
|
121
101
|
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
to_connection.put_copy_data(row)
|
126
|
-
end
|
127
|
-
end
|
102
|
+
# create a temp table
|
103
|
+
temp_table = "pgsync_#{rand(1_000_000_000)}"
|
104
|
+
to_connection.exec("CREATE TEMPORARY TABLE #{quote_ident_full(temp_table)} AS SELECT * FROM #{quote_ident_full(table)} WITH NO DATA")
|
128
105
|
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
to_connection.exec("DELETE FROM #{quote_ident_full(table)} WHERE #{quote_ident(primary_key)} IN (SELECT #{quote_ident(primary_key)} FROM #{quote_ident_full(temp_table)})")
|
135
|
-
to_connection.exec("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident(temp_table)})")
|
136
|
-
end
|
137
|
-
end
|
138
|
-
ensure
|
139
|
-
file.close
|
140
|
-
file.unlink
|
141
|
-
end
|
142
|
-
else
|
143
|
-
destination.truncate(table)
|
144
|
-
to_connection.copy_data "COPY #{quote_ident_full(table)} (#{fields}) FROM STDIN" do
|
145
|
-
from_connection.copy_data copy_to_command do
|
146
|
-
while (row = from_connection.get_copy_data)
|
147
|
-
to_connection.put_copy_data(row)
|
148
|
-
end
|
149
|
-
end
|
106
|
+
# load data
|
107
|
+
to_connection.copy_data "COPY #{quote_ident_full(temp_table)} (#{fields}) FROM STDIN" do
|
108
|
+
from_connection.copy_data copy_to_command do
|
109
|
+
while (row = from_connection.get_copy_data)
|
110
|
+
to_connection.put_copy_data(row)
|
150
111
|
end
|
151
112
|
end
|
152
|
-
seq_values.each do |seq, value|
|
153
|
-
to_connection.exec("SELECT setval(#{escape(seq)}, #{escape(value)})")
|
154
|
-
end
|
155
113
|
end
|
156
114
|
|
157
|
-
|
158
|
-
|
159
|
-
|
115
|
+
if opts[:preserve]
|
116
|
+
# insert into
|
117
|
+
to_connection.exec("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident_full(temp_table)} WHERE NOT EXISTS (SELECT 1 FROM #{quote_ident_full(table)} WHERE #{quote_ident_full(table)}.#{quote_ident(primary_key)} = #{quote_ident_full(temp_table)}.#{quote_ident(primary_key)}))")
|
118
|
+
else
|
119
|
+
to_connection.transaction do
|
120
|
+
to_connection.exec("DELETE FROM #{quote_ident_full(table)} WHERE #{quote_ident(primary_key)} IN (SELECT #{quote_ident(primary_key)} FROM #{quote_ident_full(temp_table)})")
|
121
|
+
to_connection.exec("INSERT INTO #{quote_ident_full(table)} (SELECT * FROM #{quote_ident(temp_table)})")
|
122
|
+
end
|
160
123
|
end
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
124
|
+
else
|
125
|
+
destination.truncate(table)
|
126
|
+
to_connection.copy_data "COPY #{quote_ident_full(table)} (#{fields}) FROM STDIN" do
|
127
|
+
from_connection.copy_data copy_to_command do
|
128
|
+
while (row = from_connection.get_copy_data)
|
129
|
+
to_connection.put_copy_data(row)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
seq_values.each do |seq, value|
|
135
|
+
to_connection.exec("SELECT setval(#{escape(seq)}, #{escape(value)})")
|
166
136
|
end
|
137
|
+
|
138
|
+
message = nil
|
139
|
+
message = notes.join(", ") if notes.any?
|
140
|
+
|
141
|
+
{status: "success", message: message, time: (Time.now - start_time).round(1)}
|
167
142
|
rescue => e
|
168
143
|
message =
|
169
144
|
case e
|
170
|
-
when PG::
|
145
|
+
when PG::ConnectionBad
|
171
146
|
# likely fine to show simplified message here
|
172
147
|
# the full message will be shown when first trying to connect
|
173
148
|
"Connection failed"
|
149
|
+
when PG::Error
|
150
|
+
e.message.sub("ERROR: ", "")
|
174
151
|
when Error
|
175
152
|
e.message
|
176
153
|
else
|
@@ -178,6 +155,9 @@ module PgSync
|
|
178
155
|
end
|
179
156
|
|
180
157
|
{status: "error", message: message}
|
158
|
+
ensure
|
159
|
+
source.close if source
|
160
|
+
destination.close if destination
|
181
161
|
end
|
182
162
|
|
183
163
|
private
|
data/lib/pgsync/utils.rb
CHANGED
@@ -1,43 +1,43 @@
|
|
1
1
|
module PgSync
|
2
2
|
module Utils
|
3
|
+
COLOR_CODES = {
|
4
|
+
red: 31,
|
5
|
+
green: 32,
|
6
|
+
yellow: 33
|
7
|
+
}
|
8
|
+
|
3
9
|
def log(message = nil)
|
4
|
-
|
10
|
+
output.puts message
|
5
11
|
end
|
6
12
|
|
7
|
-
def colorize(message,
|
8
|
-
if
|
9
|
-
"\e[#{
|
13
|
+
def colorize(message, color)
|
14
|
+
if output.tty?
|
15
|
+
"\e[#{COLOR_CODES[color]}m#{message}\e[0m"
|
10
16
|
else
|
11
17
|
message
|
12
18
|
end
|
13
19
|
end
|
14
20
|
|
15
|
-
def
|
16
|
-
|
21
|
+
def output
|
22
|
+
$stderr
|
23
|
+
end
|
17
24
|
|
18
|
-
|
19
|
-
|
20
|
-
if @options[:db]
|
21
|
-
db_config_file(@options[:db])
|
22
|
-
else
|
23
|
-
@options[:config] || ".pgsync.yml"
|
24
|
-
end
|
25
|
-
)
|
25
|
+
def config_file
|
26
|
+
search_tree(db_config_file(@options[:db]) || @options[:config] || ".pgsync.yml")
|
26
27
|
end
|
27
28
|
|
28
29
|
def db_config_file(db)
|
29
|
-
|
30
|
-
".pgsync-#{db}.yml"
|
30
|
+
".pgsync-#{db}.yml" if db
|
31
31
|
end
|
32
32
|
|
33
33
|
def search_tree(file)
|
34
|
+
return file if File.exists?(file)
|
35
|
+
|
34
36
|
path = Dir.pwd
|
35
37
|
# prevent infinite loop
|
36
38
|
20.times do
|
37
39
|
absolute_file = File.join(path, file)
|
38
|
-
if File.exist?(absolute_file)
|
39
|
-
break absolute_file
|
40
|
-
end
|
40
|
+
break absolute_file if File.exist?(absolute_file)
|
41
41
|
path = File.dirname(path)
|
42
42
|
break if path == "/"
|
43
43
|
end
|
data/lib/pgsync/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pgsync
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-04-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: parallel
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 4.
|
47
|
+
version: 4.8.1
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 4.
|
54
|
+
version: 4.8.1
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: tty-spinner
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|