pgsync 0.6.6 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -0
- data/LICENSE.txt +1 -1
- data/README.md +11 -3
- data/lib/pgsync/client.rb +5 -3
- data/lib/pgsync/data_source.rb +17 -0
- data/lib/pgsync/schema_sync.rb +31 -14
- data/lib/pgsync/sync.rb +10 -1
- data/lib/pgsync/table_sync.rb +3 -3
- data/lib/pgsync/task.rb +1 -1
- data/lib/pgsync/version.rb +1 -1
- data/lib/pgsync.rb +1 -1
- metadata +5 -47
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ba1091cc780447fdf3369431b15d87cd3b6e47bb047f4a34e2a36a70bb873669
|
4
|
+
data.tar.gz: bf343a9fd3e785cb077953654656d51191827216b741b844736043bfaa2b7d8b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f30e0cfdfa4a5f2f3d48db16d66dea9ff101025b8a95fd383f0eec8b60bce111d0f41cef64467c50df3d4820a55230e64f0f04e68ef09f40ea57d658b3584da6
|
7
|
+
data.tar.gz: 7feb71d51de95fba8d32d0080a43432a11f96080782aad0515dade6ce6f2b81def36f7b060a30aeeadcea14cc16dbf34fc1f94b02ebdb53c2453d1ff0ba6dd30
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,18 @@
|
|
1
|
+
## 0.7.0 (2022-03-10)
|
2
|
+
|
3
|
+
- Changed `--defer-constraints` to `--defer-constraints-v1`
|
4
|
+
- Changed `--defer-constraints-v2` to `--defer-constraints`
|
5
|
+
- Fixed unknown alias error with Ruby 3.1
|
6
|
+
- Dropped support for Ruby < 2.5
|
7
|
+
|
8
|
+
## 0.6.8 (2021-09-21)
|
9
|
+
|
10
|
+
- Fixed error when schema missing in destination with `--schema-first` and `--schema-only`
|
11
|
+
|
12
|
+
## 0.6.7 (2021-04-26)
|
13
|
+
|
14
|
+
- Fixed connection security for `--schema-first` and `--schema-only` - [more info](https://github.com/ankane/pgsync/issues/121)
|
15
|
+
|
1
16
|
## 0.6.6 (2020-10-29)
|
2
17
|
|
3
18
|
- Added support for tables with generated columns
|
data/LICENSE.txt
CHANGED
data/README.md
CHANGED
@@ -9,7 +9,7 @@ Sync data from one Postgres database to another (like `pg_dump`/`pg_restore`). D
|
|
9
9
|
|
10
10
|
:tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
|
11
11
|
|
12
|
-
[![Build Status](https://
|
12
|
+
[![Build Status](https://github.com/ankane/pgsync/workflows/build/badge.svg?branch=master)](https://github.com/ankane/pgsync/actions)
|
13
13
|
|
14
14
|
## Installation
|
15
15
|
|
@@ -19,7 +19,7 @@ pgsync is a command line tool. To install, run:
|
|
19
19
|
gem install pgsync
|
20
20
|
```
|
21
21
|
|
22
|
-
This will give you the `pgsync` command. If installation fails, you may need to install [dependencies](#dependencies).
|
22
|
+
This will give you the `pgsync` command. You can also install it with [Homebrew](#homebrew). If installation fails, you may need to install [dependencies](#dependencies).
|
23
23
|
|
24
24
|
## Setup
|
25
25
|
|
@@ -205,7 +205,7 @@ Foreign keys can make it difficult to sync data. Three options are:
|
|
205
205
|
To defer constraints, use:
|
206
206
|
|
207
207
|
```sh
|
208
|
-
pgsync --defer-constraints
|
208
|
+
pgsync --defer-constraints
|
209
209
|
```
|
210
210
|
|
211
211
|
To manually specify the order of tables, use `--jobs 1` so tables are synced one-at-a-time.
|
@@ -347,6 +347,14 @@ Bundler.with_unbundled_env do
|
|
347
347
|
end
|
348
348
|
```
|
349
349
|
|
350
|
+
## Homebrew
|
351
|
+
|
352
|
+
On Mac, you can use:
|
353
|
+
|
354
|
+
```sh
|
355
|
+
brew install ankane/brew/pgsync
|
356
|
+
```
|
357
|
+
|
350
358
|
## Dependencies
|
351
359
|
|
352
360
|
If installation fails, your system may be missing Ruby or libpq.
|
data/lib/pgsync/client.rb
CHANGED
@@ -11,6 +11,7 @@ module PgSync
|
|
11
11
|
result = Slop::Parser.new(slop_options).parse(@args)
|
12
12
|
arguments = result.arguments
|
13
13
|
options = result.to_h
|
14
|
+
options[:defer_constraints_v2] ||= options[:defer_constraints]
|
14
15
|
|
15
16
|
raise Error, "Specify either --db or --config, not both" if options[:db] && options[:config]
|
16
17
|
raise Error, "Cannot use --overwrite with --in-batches" if options[:overwrite] && options[:in_batches]
|
@@ -59,12 +60,13 @@ module PgSync
|
|
59
60
|
|
60
61
|
o.separator ""
|
61
62
|
o.separator "Foreign key options:"
|
62
|
-
o.boolean "--defer-constraints
|
63
|
+
o.boolean "--defer-constraints", "defer constraints", default: false
|
63
64
|
o.boolean "--disable-integrity", "disable foreign key triggers", default: false
|
64
65
|
o.integer "-j", "--jobs", "number of tables to sync at a time"
|
65
66
|
|
66
|
-
#
|
67
|
-
o.boolean "--defer-constraints", "defer constraints", default: false, help: false
|
67
|
+
# legacy
|
68
|
+
o.boolean "--defer-constraints-v1", "defer constraints", default: false, help: false
|
69
|
+
o.boolean "--defer-constraints-v2", "defer constraints", default: false, help: false
|
68
70
|
# private, for testing
|
69
71
|
o.boolean "--disable-integrity-v2", "disable foreign key triggers", default: false, help: false
|
70
72
|
|
data/lib/pgsync/data_source.rb
CHANGED
@@ -68,6 +68,23 @@ module PgSync
|
|
68
68
|
execute("TRUNCATE #{quote_ident_full(table)} CASCADE")
|
69
69
|
end
|
70
70
|
|
71
|
+
def schemas
|
72
|
+
@schemas ||= begin
|
73
|
+
query = <<~SQL
|
74
|
+
SELECT
|
75
|
+
schema_name
|
76
|
+
FROM
|
77
|
+
information_schema.schemata
|
78
|
+
ORDER BY 1
|
79
|
+
SQL
|
80
|
+
execute(query).map { |row| row["schema_name"] }
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def create_schema(schema)
|
85
|
+
execute("CREATE SCHEMA #{quote_ident(schema)}")
|
86
|
+
end
|
87
|
+
|
71
88
|
def triggers(table)
|
72
89
|
query = <<~SQL
|
73
90
|
SELECT
|
data/lib/pgsync/schema_sync.rb
CHANGED
@@ -24,10 +24,12 @@ module PgSync
|
|
24
24
|
spinner.auto_spin
|
25
25
|
end
|
26
26
|
|
27
|
+
create_schemas if specify_tables?
|
28
|
+
|
27
29
|
# if spinner, capture lines to show on error
|
28
30
|
lines = []
|
29
31
|
success =
|
30
|
-
run_command
|
32
|
+
run_command do |line|
|
31
33
|
if show_spinner
|
32
34
|
lines << line
|
33
35
|
else
|
@@ -49,12 +51,14 @@ module PgSync
|
|
49
51
|
|
50
52
|
private
|
51
53
|
|
52
|
-
def run_command
|
53
|
-
|
54
|
-
|
54
|
+
def run_command
|
55
|
+
err_r, err_w = IO.pipe
|
56
|
+
Open3.pipeline_start(dump_command, restore_command, err: err_w) do |wait_thrs|
|
57
|
+
err_w.close
|
58
|
+
err_r.each do |line|
|
55
59
|
yield line
|
56
60
|
end
|
57
|
-
|
61
|
+
wait_thrs.all? { |t| t.value.success? }
|
58
62
|
end
|
59
63
|
end
|
60
64
|
|
@@ -65,19 +69,32 @@ module PgSync
|
|
65
69
|
end
|
66
70
|
|
67
71
|
def dump_command
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
[]
|
72
|
+
cmd = ["pg_dump", "-Fc", "--verbose", "--schema-only", "--no-owner", "--no-acl"]
|
73
|
+
if specify_tables?
|
74
|
+
@tasks.each do |task|
|
75
|
+
cmd.concat(["-t", task.quoted_table])
|
73
76
|
end
|
74
|
-
|
75
|
-
|
77
|
+
end
|
78
|
+
cmd.concat(["-d", @source.url])
|
76
79
|
end
|
77
80
|
|
78
81
|
def restore_command
|
79
|
-
|
80
|
-
|
82
|
+
cmd = ["pg_restore", "--verbose", "--no-owner", "--no-acl", "--clean"]
|
83
|
+
cmd << "--if-exists" if Gem::Version.new(pg_restore_version) >= Gem::Version.new("9.4.0")
|
84
|
+
cmd.concat(["-d", @destination.url])
|
85
|
+
end
|
86
|
+
|
87
|
+
# pg_dump -t won't create schemas (even with -n)
|
88
|
+
# not ideal that this happens outside restore transaction
|
89
|
+
def create_schemas
|
90
|
+
schemas = @tasks.map { |t| t.table.schema }.uniq - @destination.schemas
|
91
|
+
schemas.sort.each do |schema|
|
92
|
+
@destination.create_schema(schema)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def specify_tables?
|
97
|
+
!opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude] || opts[:schemas]
|
81
98
|
end
|
82
99
|
end
|
83
100
|
end
|
data/lib/pgsync/sync.rb
CHANGED
@@ -78,7 +78,16 @@ module PgSync
|
|
78
78
|
file = config_file
|
79
79
|
if file
|
80
80
|
begin
|
81
|
-
YAML.load_file
|
81
|
+
# same options as YAML.load_file
|
82
|
+
File.open(file, "r:bom|utf-8") do |f|
|
83
|
+
# changed to keyword arguments in 3.1.0.pre1
|
84
|
+
# https://github.com/ruby/psych/commit/c79ed445b4b3f8c9adf3da13bca3c976ddfae258
|
85
|
+
if Psych::VERSION.to_f >= 3.1
|
86
|
+
YAML.safe_load(f, aliases: true, filename: file) || {}
|
87
|
+
else
|
88
|
+
YAML.safe_load(f, [], [], true, file) || {}
|
89
|
+
end
|
90
|
+
end
|
82
91
|
rescue Psych::SyntaxError => e
|
83
92
|
raise Error, e.message
|
84
93
|
rescue Errno::ENOENT
|
data/lib/pgsync/table_sync.rb
CHANGED
@@ -125,7 +125,7 @@ module PgSync
|
|
125
125
|
end
|
126
126
|
|
127
127
|
# for non-deferrable constraints
|
128
|
-
if opts[:
|
128
|
+
if opts[:defer_constraints_v1]
|
129
129
|
constraints = non_deferrable_constraints(destination)
|
130
130
|
constraints = tasks.flat_map { |t| constraints[t.table] || [] }
|
131
131
|
warning "Non-deferrable constraints: #{constraints.join(", ")}" if constraints.any?
|
@@ -230,7 +230,7 @@ module PgSync
|
|
230
230
|
|
231
231
|
# disable multiple jobs for defer constraints and disable integrity
|
232
232
|
# so we can use a transaction to ensure a consistent snapshot
|
233
|
-
if opts[:debug] || opts[:in_batches] || opts[:
|
233
|
+
if opts[:debug] || opts[:in_batches] || opts[:defer_constraints_v1] || opts[:defer_constraints_v2] || opts[:disable_integrity] || opts[:disable_integrity_v2]
|
234
234
|
warning "--jobs ignored" if jobs
|
235
235
|
jobs = 0
|
236
236
|
end
|
@@ -268,7 +268,7 @@ module PgSync
|
|
268
268
|
source.transaction do
|
269
269
|
yield
|
270
270
|
end
|
271
|
-
elsif opts[:
|
271
|
+
elsif opts[:defer_constraints_v1] || opts[:defer_constraints_v2]
|
272
272
|
destination.transaction do
|
273
273
|
if opts[:defer_constraints_v2]
|
274
274
|
table_constraints = non_deferrable_constraints(destination)
|
data/lib/pgsync/task.rb
CHANGED
@@ -149,7 +149,7 @@ module PgSync
|
|
149
149
|
destination.execute("INSERT INTO #{quoted_table} (#{fields}) (SELECT #{fields} FROM #{quote_ident_full(temp_table)}) ON CONFLICT (#{on_conflict}) DO #{action}")
|
150
150
|
else
|
151
151
|
# use delete instead of truncate for foreign keys
|
152
|
-
if opts[:
|
152
|
+
if opts[:defer_constraints_v1] || opts[:defer_constraints_v2]
|
153
153
|
destination.execute("DELETE FROM #{quoted_table}")
|
154
154
|
else
|
155
155
|
destination.truncate(table)
|
data/lib/pgsync/version.rb
CHANGED
data/lib/pgsync.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pgsync
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-03-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: parallel
|
@@ -66,50 +66,8 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: bundler
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - ">="
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '0'
|
76
|
-
type: :development
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - ">="
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '0'
|
83
|
-
- !ruby/object:Gem::Dependency
|
84
|
-
name: minitest
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - ">="
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: '0'
|
90
|
-
type: :development
|
91
|
-
prerelease: false
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
93
|
-
requirements:
|
94
|
-
- - ">="
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
version: '0'
|
97
|
-
- !ruby/object:Gem::Dependency
|
98
|
-
name: rake
|
99
|
-
requirement: !ruby/object:Gem::Requirement
|
100
|
-
requirements:
|
101
|
-
- - ">="
|
102
|
-
- !ruby/object:Gem::Version
|
103
|
-
version: '0'
|
104
|
-
type: :development
|
105
|
-
prerelease: false
|
106
|
-
version_requirements: !ruby/object:Gem::Requirement
|
107
|
-
requirements:
|
108
|
-
- - ">="
|
109
|
-
- !ruby/object:Gem::Version
|
110
|
-
version: '0'
|
111
69
|
description:
|
112
|
-
email: andrew@
|
70
|
+
email: andrew@ankane.org
|
113
71
|
executables:
|
114
72
|
- pgsync
|
115
73
|
extensions: []
|
@@ -145,14 +103,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
145
103
|
requirements:
|
146
104
|
- - ">="
|
147
105
|
- !ruby/object:Gem::Version
|
148
|
-
version: '2.
|
106
|
+
version: '2.5'
|
149
107
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
150
108
|
requirements:
|
151
109
|
- - ">="
|
152
110
|
- !ruby/object:Gem::Version
|
153
111
|
version: '0'
|
154
112
|
requirements: []
|
155
|
-
rubygems_version: 3.
|
113
|
+
rubygems_version: 3.3.7
|
156
114
|
signing_key:
|
157
115
|
specification_version: 4
|
158
116
|
summary: Sync Postgres data between databases
|