pgsync 0.5.5 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pgsync might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -3
- data/LICENSE.txt +1 -1
- data/README.md +87 -32
- data/config.yml +4 -0
- data/lib/pgsync.rb +6 -1
- data/lib/pgsync/client.rb +55 -57
- data/lib/pgsync/data_source.rb +55 -104
- data/lib/pgsync/init.rb +50 -6
- data/lib/pgsync/schema_sync.rb +83 -0
- data/lib/pgsync/sequence.rb +29 -0
- data/lib/pgsync/sync.rb +82 -199
- data/lib/pgsync/table.rb +28 -0
- data/lib/pgsync/table_sync.rb +254 -207
- data/lib/pgsync/task.rb +325 -0
- data/lib/pgsync/task_resolver.rb +237 -0
- data/lib/pgsync/utils.rb +55 -14
- data/lib/pgsync/version.rb +1 -1
- metadata +7 -3
- data/lib/pgsync/table_list.rb +0 -141
data/lib/pgsync/utils.rb
CHANGED
@@ -3,7 +3,8 @@ module PgSync
|
|
3
3
|
COLOR_CODES = {
|
4
4
|
red: 31,
|
5
5
|
green: 32,
|
6
|
-
yellow: 33
|
6
|
+
yellow: 33,
|
7
|
+
cyan: 36
|
7
8
|
}
|
8
9
|
|
9
10
|
def log(message = nil)
|
@@ -18,29 +19,69 @@ module PgSync
|
|
18
19
|
end
|
19
20
|
end
|
20
21
|
|
22
|
+
def warning(message)
|
23
|
+
log colorize(message, :yellow)
|
24
|
+
end
|
25
|
+
|
26
|
+
def deprecated(message)
|
27
|
+
warning "[DEPRECATED] #{message}"
|
28
|
+
end
|
29
|
+
|
21
30
|
def output
|
22
31
|
$stderr
|
23
32
|
end
|
24
33
|
|
25
|
-
def
|
26
|
-
|
34
|
+
def db_config_file(db)
|
35
|
+
".pgsync-#{db}.yml"
|
27
36
|
end
|
28
37
|
|
29
|
-
def
|
30
|
-
|
38
|
+
def confirm_tables_exist(data_source, tasks, description)
|
39
|
+
tasks.map(&:table).each do |table|
|
40
|
+
unless data_source.table_exists?(table)
|
41
|
+
raise Error, "Table not found in #{description}: #{table}"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def first_schema
|
47
|
+
@first_schema ||= source.search_path.find { |sp| sp != "pg_catalog" }
|
48
|
+
end
|
49
|
+
|
50
|
+
def task_name(task)
|
51
|
+
friendly_name(task.table)
|
31
52
|
end
|
32
53
|
|
33
|
-
def
|
34
|
-
|
54
|
+
def friendly_name(table)
|
55
|
+
if table.schema == first_schema
|
56
|
+
table.name
|
57
|
+
else
|
58
|
+
table.full_name
|
59
|
+
end
|
60
|
+
end
|
35
61
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
path = File.dirname(path)
|
42
|
-
break if path == "/"
|
62
|
+
def quote_ident_full(ident)
|
63
|
+
if ident.is_a?(Table) || ident.is_a?(Sequence)
|
64
|
+
[quote_ident(ident.schema), quote_ident(ident.name)].join(".")
|
65
|
+
else # temp table names are strings
|
66
|
+
quote_ident(ident)
|
43
67
|
end
|
44
68
|
end
|
69
|
+
|
70
|
+
def quote_ident(value)
|
71
|
+
PG::Connection.quote_ident(value)
|
72
|
+
end
|
73
|
+
|
74
|
+
def escape(value)
|
75
|
+
if value.is_a?(String)
|
76
|
+
"'#{quote_string(value)}'"
|
77
|
+
else
|
78
|
+
value
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
# activerecord
|
83
|
+
def quote_string(s)
|
84
|
+
s.gsub(/\\/, '\&\&').gsub(/'/, "''")
|
85
|
+
end
|
45
86
|
end
|
46
87
|
end
|
data/lib/pgsync/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pgsync
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-06-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: parallel
|
@@ -124,9 +124,13 @@ files:
|
|
124
124
|
- lib/pgsync/client.rb
|
125
125
|
- lib/pgsync/data_source.rb
|
126
126
|
- lib/pgsync/init.rb
|
127
|
+
- lib/pgsync/schema_sync.rb
|
128
|
+
- lib/pgsync/sequence.rb
|
127
129
|
- lib/pgsync/sync.rb
|
128
|
-
- lib/pgsync/
|
130
|
+
- lib/pgsync/table.rb
|
129
131
|
- lib/pgsync/table_sync.rb
|
132
|
+
- lib/pgsync/task.rb
|
133
|
+
- lib/pgsync/task_resolver.rb
|
130
134
|
- lib/pgsync/utils.rb
|
131
135
|
- lib/pgsync/version.rb
|
132
136
|
homepage: https://github.com/ankane/pgsync
|
data/lib/pgsync/table_list.rb
DELETED
@@ -1,141 +0,0 @@
|
|
1
|
-
module PgSync
|
2
|
-
class TableList
|
3
|
-
include Utils
|
4
|
-
|
5
|
-
attr_reader :args, :opts, :source, :config
|
6
|
-
|
7
|
-
def initialize(args, options, source, config)
|
8
|
-
@args = args
|
9
|
-
@opts = options
|
10
|
-
@source = source
|
11
|
-
@config = config
|
12
|
-
@groups = config["groups"] || {}
|
13
|
-
end
|
14
|
-
|
15
|
-
def group?(group)
|
16
|
-
@groups.key?(group)
|
17
|
-
end
|
18
|
-
|
19
|
-
def tables
|
20
|
-
tables = {}
|
21
|
-
sql = args[1]
|
22
|
-
|
23
|
-
groups = to_arr(opts[:groups])
|
24
|
-
tables2 = to_arr(opts[:tables])
|
25
|
-
|
26
|
-
if args[0]
|
27
|
-
# could be a group, table, or mix
|
28
|
-
to_arr(args[0]).each do |tag|
|
29
|
-
group, id = tag.split(":", 2)
|
30
|
-
if group?(group)
|
31
|
-
groups << tag
|
32
|
-
else
|
33
|
-
tables2 << tag
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
groups.each do |tag|
|
39
|
-
group, id = tag.split(":", 2)
|
40
|
-
raise Error, "Group not found: #{group}" unless group?(group)
|
41
|
-
|
42
|
-
# if id
|
43
|
-
# # TODO show group name and value
|
44
|
-
# log colorize("`pgsync group:value` is deprecated and will have a different function in 0.6.0.", :yellow)
|
45
|
-
# log colorize("Use `pgsync group --var 1=value` instead.", :yellow)
|
46
|
-
# end
|
47
|
-
|
48
|
-
@groups[group].each do |table|
|
49
|
-
table_sql = nil
|
50
|
-
if table.is_a?(Array)
|
51
|
-
table, table_sql = table
|
52
|
-
end
|
53
|
-
add_table(tables, table, id, sql || table_sql)
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
tables2.each do |tag|
|
58
|
-
table, id = tag.split(":", 2)
|
59
|
-
raise Error, "Cannot use parameters with tables" if id
|
60
|
-
add_table(tables, table, id, sql)
|
61
|
-
end
|
62
|
-
|
63
|
-
if !opts[:groups] && !opts[:tables] && !args[0]
|
64
|
-
exclude = to_arr(opts[:exclude])
|
65
|
-
exclude = source.fully_resolve_tables(exclude).keys if exclude.any?
|
66
|
-
|
67
|
-
tabs = source.tables
|
68
|
-
unless opts[:all_schemas]
|
69
|
-
schemas = Set.new(opts[:schemas] ? to_arr(opts[:schemas]) : source.search_path)
|
70
|
-
tabs.select! { |t| schemas.include?(t.split(".", 2)[0]) }
|
71
|
-
end
|
72
|
-
|
73
|
-
(tabs - exclude).each do |k|
|
74
|
-
tables[k] = {}
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
source.fully_resolve_tables(tables)
|
79
|
-
end
|
80
|
-
|
81
|
-
private
|
82
|
-
|
83
|
-
def to_arr(value)
|
84
|
-
if value.is_a?(Array)
|
85
|
-
value
|
86
|
-
else
|
87
|
-
# Split by commas, but don't use commas inside double quotes
|
88
|
-
# https://stackoverflow.com/questions/21105360/regex-find-comma-not-inside-quotes
|
89
|
-
value.to_s.split(/(?!\B"[^"]*),(?![^"]*"\B)/)
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
|
-
def add_table(tables, table, id, sql)
|
94
|
-
tables2 =
|
95
|
-
if table.include?("*")
|
96
|
-
regex = Regexp.new('\A' + Regexp.escape(table).gsub('\*','[^\.]*') + '\z')
|
97
|
-
source.tables.select { |t| regex.match(t) || regex.match(t.split(".", 2).last) }
|
98
|
-
else
|
99
|
-
[table]
|
100
|
-
end
|
101
|
-
|
102
|
-
tables2.each do |tab|
|
103
|
-
tables[tab] = {}
|
104
|
-
tables[tab][:sql] = table_sql(sql, id) if sql
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
def table_sql(sql, id)
|
109
|
-
# vars must match \w
|
110
|
-
missing_vars = sql.scan(/{\w+}/).map { |v| v[1..-2] }
|
111
|
-
|
112
|
-
vars = {}
|
113
|
-
|
114
|
-
# legacy
|
115
|
-
if id
|
116
|
-
vars["id"] = cast(id)
|
117
|
-
vars["1"] = cast(id)
|
118
|
-
end
|
119
|
-
|
120
|
-
# opts[:var].each do |value|
|
121
|
-
# k, v = value.split("=", 2)
|
122
|
-
# vars[k] = v
|
123
|
-
# end
|
124
|
-
|
125
|
-
sql = sql.dup
|
126
|
-
vars.each do |k, v|
|
127
|
-
# only sub if in var list
|
128
|
-
sql.gsub!("{#{k}}", cast(v)) if missing_vars.delete(k)
|
129
|
-
end
|
130
|
-
|
131
|
-
raise Error, "Missing variables: #{missing_vars.uniq.join(", ")}" if missing_vars.any?
|
132
|
-
|
133
|
-
sql
|
134
|
-
end
|
135
|
-
|
136
|
-
# TODO quote vars in next major version
|
137
|
-
def cast(value)
|
138
|
-
value.to_s.gsub(/\A\"|\"\z/, '')
|
139
|
-
end
|
140
|
-
end
|
141
|
-
end
|