pgmove 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 27cd367d56ae51a5f2cfc6203ba5abbeba5a4224
4
+ data.tar.gz: 250375ea08025b5c8a18e6010de89bca19a0e13f
5
+ SHA512:
6
+ metadata.gz: 7a049e3df27c553b96bc908bfd1561b6f493f0cdc30b68fadb4c2619ec72eed46b78efe00b4e560a77bb73b467e663a11eae3725e91e2c44ef06c97aee48f65d
7
+ data.tar.gz: 49ea1266efabe4f624efb2cd044ece943aa5ced0fa477201ddb19adfde4716b66088fb940698fe40ce4dac806eac6033a67ff5a40b2d3e5ed98cafcca16b3954
data/.gitignore ADDED
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in pgmove.gemspec
4
+ gemspec
data/README.md ADDED
@@ -0,0 +1,36 @@
1
+ # Pgmove
2
+
3
+ Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/pgmove`. To experiment with that code, run `bin/console` for an interactive prompt.
4
+
5
+ TODO: Delete this and the text above, and describe your gem
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'pgmove'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install pgmove
22
+
23
+ ## Usage
24
+
25
+ TODO: Write usage instructions here
26
+
27
+ ## Development
28
+
29
+ After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
30
+
31
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
32
+
33
+ ## Contributing
34
+
35
+ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/pgmove.
36
+
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require "bundler/gem_tasks"
2
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "pgmove"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
data/dev_exe/pgmove ADDED
@@ -0,0 +1,4 @@
1
+ #!/usr/bin/env ruby
2
+ lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
3
+ $LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
4
+ load __dir__ + '/../exe/pgmove'
data/exe/pgmove ADDED
@@ -0,0 +1,110 @@
1
+ #!/usr/bin/env ruby
2
+ Signal.trap("INT") { exit 1 }
3
+
4
+ def pgmove_dir?
5
+ File.exist? ".pgmove"
6
+ end
7
+
8
+ def init(argv)
9
+ dir = argv.first
10
+ abort "usage: pgmove init <dir>" if not dir
11
+ FileUtils.mkdir dir
12
+ FileUtils.touch "#{dir}/.pgmove"
13
+ puts "initialized #{dir}"
14
+ end
15
+
16
+ def read_config(file)
17
+ YAML.load_file file
18
+ end
19
+
20
+ require 'fileutils'
21
+ require 'pgmove/logger'
22
+ require 'optparse'
23
+
24
+ logger = ::Pgmove::Logger.logger
25
+
26
+ options = {}
27
+ opts_parser = OptionParser.new do |opts|
28
+
29
+ banner = []
30
+ banner << "Usage: pgmove [global options] command args"
31
+ banner << "Commands: init start status bucardo compare finalize"
32
+
33
+ banner << "Options: "
34
+ opts.banner = banner.join("\n")
35
+
36
+ opts.on("-v", "--version", "Show version") do |v|
37
+ puts ::Pgmove::VERSION
38
+ exit
39
+ end
40
+
41
+ opts.on("--debug", "Show debug messages") do
42
+ options[:debug] = true
43
+ logger.level = ::Logger::DEBUG
44
+ end
45
+
46
+ opts.on("--trace", "Show debug messages and exception stack trace") do
47
+ options[:debug] = true
48
+ options[:trace] = true
49
+ logger.level = ::Logger::DEBUG
50
+ logger.trace_mode = true
51
+ end
52
+
53
+ opts.on_tail("-h", "--help", "Show this message") do
54
+ puts opts
55
+ exit
56
+ end
57
+ end
58
+ opts_parser.parse!(ARGV)
59
+
60
+ command = ARGV.shift
61
+ if command == "init"
62
+ init(ARGV)
63
+ exit
64
+ elsif not pgmove_dir?
65
+ abort "not running from pgmove directory, use 'pgmove init <dir>'" if not pgmove_dir?
66
+ end
67
+
68
+ require 'pgmove/db'
69
+ require 'pgmove/bucardo'
70
+ require 'yaml'
71
+
72
+ config = read_config("config.yml")
73
+
74
+
75
+ src_db = Pgmove::Db.new(
76
+ name: config["src"]["name"],
77
+ user: config["src"]["user"],
78
+ pass: config["src"]["pass"],
79
+ host: config["src"]["host"],
80
+ port: config["src"]["port"]
81
+ )
82
+
83
+ dest_db = Pgmove::Db.new(
84
+ name: config["dest"]["name"],
85
+ user: config["dest"]["user"],
86
+ pass: config["dest"]["pass"],
87
+ host: config["dest"]["host"],
88
+ port: config["dest"]["port"],
89
+ use_tmp: true
90
+ )
91
+
92
+ bucardo = Pgmove::Bucardo.new(src_db, dest_db)
93
+
94
+ case command
95
+ when "start"
96
+ bucardo.setup
97
+ bucardo.start_sync
98
+ when "status"
99
+ bucardo.status
100
+ when "reset"
101
+ bucardo.reset
102
+ when "compare"
103
+ bucardo.compare
104
+ when "finalize"
105
+ bucardo.finalize
106
+ when "bucardo", "b"
107
+ bucardo.bucardo ARGV.join(" ")
108
+ else
109
+ puts opts_parser
110
+ end
@@ -0,0 +1,128 @@
1
+ require_relative 'helper'
2
+ require 'fileutils'
3
+
4
+ module Pgmove
5
+ class Bucardo
6
+ include Helper
7
+
8
+ SCHEMA_PATH = "tmp/schema.sql"
9
+ PGPASS_PATH = "#{Dir.home}/.pgpass"
10
+ RELGROUP = "pgmove"
11
+ SYNC_NAME = "pgmove"
12
+
13
+ attr_reader :src_db, :dest_db
14
+
15
+ def initialize(src_db, dest_db)
16
+ @src_db = src_db
17
+ @dest_db = dest_db
18
+ end
19
+
20
+ def setup
21
+ create_dirs
22
+ reset
23
+ install
24
+ copy_schema
25
+ add_db
26
+ add_tables
27
+ add_sync
28
+ end
29
+
30
+ def bucardo(command, db: nil, env: {})
31
+ db ||= "bucardo"
32
+ system! "bucardo -U #{@src_db.user} -P #{@src_db.pass} -h #{@src_db.host} -p #{@src_db.port} -d #{db} #{command}", env: env
33
+ end
34
+
35
+ def reset
36
+ stop
37
+ reset_pgpass
38
+ system! %(psql "#{@src_db.conn_str}" -c 'DROP schema if exists bucardo cascade')
39
+ system! %(psql "#{@src_db.conn_str}" -c 'DROP database IF EXISTS bucardo')
40
+ system! %(psql "#{@src_db.conn_str}" -c 'drop role IF EXISTS bucardo')
41
+ @dest_db.reset
42
+ end
43
+
44
+ def start_sync
45
+ bucardo "start --log-destination log"
46
+ end
47
+
48
+ def status
49
+ bucardo "status pgmove"
50
+ end
51
+
52
+ def stop
53
+ if Dir.glob("tmp/*.pid").size > 0
54
+ bucardo "stop"
55
+ sleep 5
56
+ end
57
+ end
58
+
59
+ def compare
60
+ begin
61
+ sconn = @src_db.pg_conn
62
+ dconn = @dest_db.pg_conn
63
+ format = "%-75s | %15s | %15s | %5s\n"
64
+ printf format, "table", "source", "dest", "diff"
65
+ puts "-" * (75 + 15 + 15 + 5 + 9)
66
+ @src_db.tables.each do |t|
67
+ src_count = @src_db.row_count(t, conn: sconn)
68
+ dest_count = @dest_db.row_count(t, conn: dconn)
69
+ marker = src_count == dest_count ? '' : "*"
70
+ printf format, t, src_count, dest_count, marker
71
+ end
72
+ ensure
73
+ sconn.close if sconn
74
+ dconn.close if dconn
75
+ end
76
+ end
77
+
78
+ def finalize
79
+ stop
80
+ @dest_db.finalize
81
+ end
82
+
83
+ private
84
+
85
+ def check_deps
86
+ raise "#{@src_db.user} not a superuser" if not @src_db.superuser?
87
+ raise "#{@dest_db.user} not a superuser" if not @dest_db.superuser?
88
+
89
+ end
90
+
91
+ def create_dirs
92
+ FileUtils.mkdir_p "tmp"
93
+ FileUtils.mkdir_p "log"
94
+ end
95
+
96
+ def reset_pgpass
97
+ File.open(PGPASS_PATH, "w") do |f|
98
+ f.write "#{@src_db.host}:#{@src_db.port}:postgres:#{@src_db.user}:#{@src_db.pass}\n"
99
+ f.chmod 0600
100
+ end
101
+ end
102
+
103
+ def install
104
+ env = { "DBUSER" => @src_db.user }
105
+ bucardo "install --batch --pid-dir tmp", db: "postgres", env: env
106
+ end
107
+
108
+ def copy_schema
109
+ system! %(pg_dump "#{@src_db.conn_str}" -N bucardo --schema-only > #{SCHEMA_PATH})
110
+ @dest_db.load_schema(SCHEMA_PATH)
111
+ end
112
+
113
+ def add_db
114
+ bucardo "add db source_db #{@src_db.bucardo_conn_str}"
115
+ bucardo "add db dest_db #{@dest_db.bucardo_conn_str}"
116
+ end
117
+
118
+ def add_tables
119
+ bucardo "add all tables db=source_db relgroup=#{RELGROUP} -T public.spatial_ref_sys -N postgis"
120
+ bucardo "add all sequences db=source_db relgroup=#{RELGROUP}"
121
+ end
122
+
123
+ def add_sync
124
+ bucardo "add sync #{SYNC_NAME} relgroup=#{RELGROUP} dbs=source_db:source,dest_db:target onetimecopy=2"
125
+ end
126
+
127
+ end
128
+ end
@@ -0,0 +1,42 @@
1
+ require 'logger'
2
+ require_relative 'ext/string_ext'
3
+
4
+ module Pgmove
5
+ class CustomLogger < ::Logger
6
+
7
+ using StringExt
8
+ attr_accessor :trace_mode
9
+
10
+ def initialize(file)
11
+ super(file)
12
+ @level = ::Logger::INFO
13
+ end
14
+
15
+ def format_message(severity, timestamp, progname, msg)
16
+ case severity
17
+ when "INFO"
18
+ "#{msg}\n"
19
+ when "ERROR"
20
+ "#{severity.bold.red} #{msg}\n"
21
+ when "WARN"
22
+ "#{severity.downcase.bold.yellow} #{msg}\n"
23
+ else
24
+ "#{severity[0].bold.blue} #{msg}\n"
25
+ end
26
+ end
27
+
28
+ def bullet(msg)
29
+ info "#{"\u2219".bold.blue} #{msg}"
30
+ end
31
+
32
+ def trace(msg)
33
+ return if not @trace_mode
34
+ info %(#{"T".bold.blue} #{msg}\n)
35
+ end
36
+
37
+ def silence!
38
+ @logdev = nil
39
+ end
40
+
41
+ end
42
+ end
data/lib/pgmove/db.rb ADDED
@@ -0,0 +1,124 @@
1
+ require_relative 'helper'
2
+ require 'pg'
3
+
4
+ module Pgmove
5
+ class Db
6
+
7
+ include Helper
8
+ attr_reader :user, :pass, :host, :port, :name
9
+
10
+ def initialize(name:, user:, pass:, host:, port:, use_tmp: false)
11
+ @user = user
12
+ @pass = pass
13
+ @host = host
14
+ @port = port
15
+
16
+ if use_tmp
17
+ @name = "#{name}_bucardo_tmp"
18
+ @final_name = name
19
+ else
20
+ @name = name
21
+ end
22
+ end
23
+
24
+ def reset
25
+ psql "DROP DATABASE IF EXISTS #{@name}", db: "postgres"
26
+ psql "CREATE DATABASE #{@name}", db: "postgres"
27
+ end
28
+
29
+ def load_schema(path)
30
+ psql_raw "-f #{path}"
31
+ end
32
+
33
+ def finalize
34
+ psql "ALTER DATABASE #{@name} RENAME to #{@final_name}", db: "postgres"
35
+ end
36
+
37
+ def superuser?
38
+ rows = psql_query "select usesuper from pg_user where usename = CURRENT_USER"
39
+ rows[0][0] == 't'
40
+ end
41
+
42
+ def conn_str(db: nil)
43
+ db ||= @name
44
+ "host=#{@host} port=#{@port} dbname=#{db} user=#{@user} password=#{@pass}"
45
+ end
46
+
47
+ def bucardo_conn_str
48
+ "dbhost=#{@host} dbport=#{@port} dbname=#{@name} dbuser=#{@user} password=#{@pass}"
49
+ end
50
+
51
+ def tables
52
+ tables = []
53
+ schemas.each do |s|
54
+ sql = "SELECT table_name FROM information_schema.tables \
55
+ WHERE table_schema='#{s}' AND table_type='BASE TABLE'"
56
+ rows = psql_query sql
57
+ rows.each { |r| tables << "#{s}.#{r[0]}" }
58
+ end
59
+ tables.sort
60
+ end
61
+
62
+ def schemas
63
+ sql = "SELECT n.nspname FROM pg_catalog.pg_namespace n WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' AND n.nspname <> 'bucardo'"
64
+ rows = psql_query sql
65
+ rows.map { |r| r[0] }
66
+ end
67
+
68
+ def row_count(table, conn:)
69
+ sql = "select count(*) from #{table}"
70
+ conn.exec(sql)[0]["count"].to_i
71
+ rescue
72
+ -1
73
+ end
74
+
75
+ def compare(other_db)
76
+ row_counts = row_counts()
77
+ other_row_counts = other_db.row_counts
78
+ row_counts.each do |k, v|
79
+ printf "%-60s %15d %15d\n", k, row_counts[k], other_row_counts[k]
80
+ end
81
+ end
82
+
83
+ def pg_conn
84
+ conn = PG::Connection.open(
85
+ :dbname => @name,
86
+ :user => @user,
87
+ :password => @pass,
88
+ :host => @host,
89
+ :port => @port
90
+ )
91
+ if block_given?
92
+ yield conn
93
+ else
94
+ conn
95
+ end
96
+ ensure
97
+ if block_given?
98
+ conn.close if conn
99
+ end
100
+ end
101
+
102
+ private
103
+
104
+ def psql(sql, db: nil)
105
+ psql_raw %(-c "#{sql}"), db: db
106
+ end
107
+
108
+ def psql_raw(params, db: nil)
109
+ command = %(psql "#{conn_str(db: db)}" #{params})
110
+ if @final_name
111
+ system! command
112
+ else
113
+ raise "wont run on live db: #{command}"
114
+ end
115
+ end
116
+
117
+ def psql_query(sql, db: nil)
118
+ command = %(psql "#{conn_str(db: db)}" -t -A -c "#{sql}")
119
+ rows = `#{command}`.split("\n")
120
+ rows.map { |i| i.split('|') }
121
+ end
122
+
123
+ end
124
+ end
@@ -0,0 +1,37 @@
1
+ module Pgmove
2
+ module StringExt
3
+ refine String do
4
+ def colorize(color_code)
5
+ "\e[#{color_code}m#{self}\e[0m"
6
+ end
7
+
8
+ def bold
9
+ "\e[1m#{self}\e[0m"
10
+ end
11
+
12
+ def white
13
+ colorize(37)
14
+ end
15
+
16
+ def green
17
+ colorize(32)
18
+ end
19
+
20
+ def yellow
21
+ colorize(33)
22
+ end
23
+
24
+ def red
25
+ colorize(31)
26
+ end
27
+
28
+ def blue
29
+ colorize(34)
30
+ end
31
+
32
+ def grey
33
+ colorize("90")
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,15 @@
1
+ require_relative 'logger'
2
+
3
+ module Pgmove
4
+ module Helper
5
+
6
+ include Logger
7
+
8
+ def system!(command, display: true, env: {})
9
+ logger.bullet command if display
10
+ ok = system env, command
11
+ raise "Non zero exit: #{command}" if not ok
12
+ end
13
+
14
+ end
15
+ end
@@ -0,0 +1,26 @@
1
+ require_relative 'custom_logger'
2
+ module Pgmove
3
+ module Logger
4
+
5
+ class << self
6
+ attr_writer :logger
7
+ end
8
+
9
+ def self.logger
10
+ @logger ||= CustomLogger.new(STDOUT)
11
+ end
12
+
13
+ def self.stderr
14
+ @stderr ||= ::Logger.new(STDERR)
15
+ end
16
+
17
+ def logger
18
+ ::Pgmove::Logger.logger
19
+ end
20
+
21
+ def stderr
22
+ ::Pgmove::Logger.stderr
23
+ end
24
+
25
+ end
26
+ end
@@ -0,0 +1,3 @@
1
+ module Pgmove
2
+ VERSION = "0.1.0"
3
+ end
data/lib/pgmove.rb ADDED
@@ -0,0 +1,5 @@
1
+ require "pgmove/version"
2
+
3
+ module Pgmove
4
+ # Your code goes here...
5
+ end
data/pgmove.gemspec ADDED
@@ -0,0 +1,23 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'pgmove/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "pgmove"
8
+ spec.version = Pgmove::VERSION
9
+ spec.authors = ["Neeraj"]
10
+ spec.email = [""]
11
+
12
+ spec.summary = %q{A tool to move postgres databases b/w clusters using bucardo}
13
+ spec.description = %q{A tool to move postgres databases b/w clusters using bucardo}
14
+
15
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
16
+ spec.bindir = "exe"
17
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_dependency "pg", "~> 0.19.0"
21
+ spec.add_development_dependency "bundler", "~> 1.12"
22
+ spec.add_development_dependency "rake", "~> 10.0"
23
+ end
metadata ADDED
@@ -0,0 +1,103 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: pgmove
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Neeraj
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2016-11-23 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: pg
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 0.19.0
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 0.19.0
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.12'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.12'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '10.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '10.0'
55
+ description: A tool to move postgres databases b/w clusters using bucardo
56
+ email:
57
+ - ''
58
+ executables:
59
+ - pgmove
60
+ extensions: []
61
+ extra_rdoc_files: []
62
+ files:
63
+ - ".gitignore"
64
+ - Gemfile
65
+ - README.md
66
+ - Rakefile
67
+ - bin/console
68
+ - bin/setup
69
+ - dev_exe/pgmove
70
+ - exe/pgmove
71
+ - lib/pgmove.rb
72
+ - lib/pgmove/bucardo.rb
73
+ - lib/pgmove/custom_logger.rb
74
+ - lib/pgmove/db.rb
75
+ - lib/pgmove/ext/string_ext.rb
76
+ - lib/pgmove/helper.rb
77
+ - lib/pgmove/logger.rb
78
+ - lib/pgmove/version.rb
79
+ - pgmove.gemspec
80
+ homepage:
81
+ licenses: []
82
+ metadata: {}
83
+ post_install_message:
84
+ rdoc_options: []
85
+ require_paths:
86
+ - lib
87
+ required_ruby_version: !ruby/object:Gem::Requirement
88
+ requirements:
89
+ - - ">="
90
+ - !ruby/object:Gem::Version
91
+ version: '0'
92
+ required_rubygems_version: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ requirements: []
98
+ rubyforge_project:
99
+ rubygems_version: 2.5.1
100
+ signing_key:
101
+ specification_version: 4
102
+ summary: A tool to move postgres databases b/w clusters using bucardo
103
+ test_files: []