pgmove 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 27cd367d56ae51a5f2cfc6203ba5abbeba5a4224
+  data.tar.gz: 250375ea08025b5c8a18e6010de89bca19a0e13f
+SHA512:
+  metadata.gz: 7a049e3df27c553b96bc908bfd1561b6f493f0cdc30b68fadb4c2619ec72eed46b78efe00b4e560a77bb73b467e663a11eae3725e91e2c44ef06c97aee48f65d
+  data.tar.gz: 49ea1266efabe4f624efb2cd044ece943aa5ced0fa477201ddb19adfde4716b66088fb940698fe40ce4dac806eac6033a67ff5a40b2d3e5ed98cafcca16b3954

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pgmove.gemspec
+gemspec

data/README.md

@@ -0,0 +1,36 @@
+# Pgmove
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/pgmove`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pgmove'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pgmove
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/pgmove.
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "pgmove"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/dev_exe/pgmove

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+load __dir__ + '/../exe/pgmove'

data/exe/pgmove

@@ -0,0 +1,110 @@
+#!/usr/bin/env ruby
+Signal.trap("INT") { exit 1 }
+
+def pgmove_dir?
+  File.exist? ".pgmove"
+end
+
+def init(argv)
+  dir = argv.first
+  abort "usage: pgmove init <dir>" if not dir
+  FileUtils.mkdir dir
+  FileUtils.touch "#{dir}/.pgmove"
+  puts "initialized #{dir}"
+end
+
+def read_config(file)
+  YAML.load_file file
+end
+
+require 'fileutils'
+require 'pgmove/logger'
+require 'optparse'
+
+logger = ::Pgmove::Logger.logger
+
+options = {}
+opts_parser = OptionParser.new do |opts| 
+    
+    banner = []  
+    banner << "Usage: pgmove [global options] command args"
+    banner << "Commands: init start status bucardo compare finalize"
+    
+    banner << "Options: "
+    opts.banner = banner.join("\n")
+
+    opts.on("-v", "--version", "Show version") do |v| 
+      puts ::Pgmove::VERSION
+      exit
+    end 
+
+    opts.on("--debug", "Show debug messages") do
+      options[:debug] = true
+      logger.level = ::Logger::DEBUG
+    end 
+
+    opts.on("--trace", "Show debug messages and exception stack trace") do
+      options[:debug] = true
+      options[:trace] = true
+      logger.level = ::Logger::DEBUG
+      logger.trace_mode = true
+    end 
+
+    opts.on_tail("-h", "--help", "Show this message") do
+      puts opts
+      exit
+    end
+end
+opts_parser.parse!(ARGV)
+
+command = ARGV.shift
+if command == "init"
+  init(ARGV)
+  exit
+elsif not pgmove_dir?
+  abort "not running from pgmove directory, use 'pgmove init <dir>'" if not pgmove_dir?
+end
+
+require 'pgmove/db'
+require 'pgmove/bucardo'
+require 'yaml'
+
+config = read_config("config.yml")
+
+
+src_db = Pgmove::Db.new(
+  name: config["src"]["name"],
+  user: config["src"]["user"],
+  pass: config["src"]["pass"],
+  host: config["src"]["host"],
+  port: config["src"]["port"]
+)
+
+dest_db = Pgmove::Db.new(
+  name: config["dest"]["name"],
+  user: config["dest"]["user"],
+  pass: config["dest"]["pass"],
+  host: config["dest"]["host"],
+  port: config["dest"]["port"],
+  use_tmp: true
+)
+
+bucardo = Pgmove::Bucardo.new(src_db, dest_db)
+
+case command
+when "start"
+  bucardo.setup
+  bucardo.start_sync
+when "status"
+  bucardo.status
+when "reset"
+  bucardo.reset
+when "compare"
+  bucardo.compare
+when "finalize"
+  bucardo.finalize
+when "bucardo", "b"
+  bucardo.bucardo ARGV.join(" ")
+else
+  puts opts_parser
+end

data/lib/pgmove/bucardo.rb

@@ -0,0 +1,128 @@
+require_relative 'helper'
+require 'fileutils'
+
+module Pgmove
+  class Bucardo
+    include Helper
+
+    SCHEMA_PATH = "tmp/schema.sql"
+    PGPASS_PATH = "#{Dir.home}/.pgpass"
+    RELGROUP = "pgmove"
+    SYNC_NAME = "pgmove"
+
+    attr_reader :src_db, :dest_db
+
+    def initialize(src_db, dest_db)
+      @src_db = src_db
+      @dest_db = dest_db
+    end
+
+    def setup
+      create_dirs
+      reset
+      install
+      copy_schema
+      add_db
+      add_tables
+      add_sync
+    end
+
+    def bucardo(command, db: nil, env: {})
+      db ||= "bucardo"
+      system! "bucardo -U #{@src_db.user} -P #{@src_db.pass} -h #{@src_db.host} -p #{@src_db.port} -d #{db} #{command}", env: env
+    end
+
+    def reset
+      stop
+      reset_pgpass
+      system! %(psql "#{@src_db.conn_str}" -c 'DROP schema if exists bucardo cascade')
+      system! %(psql "#{@src_db.conn_str}" -c 'DROP database IF EXISTS bucardo')
+      system! %(psql "#{@src_db.conn_str}" -c 'drop role IF EXISTS bucardo')
+      @dest_db.reset
+    end
+
+    def start_sync
+      bucardo "start --log-destination log"
+    end
+
+    def status
+      bucardo "status pgmove"
+    end
+
+    def stop
+      if Dir.glob("tmp/*.pid").size > 0
+        bucardo "stop"
+        sleep 5
+      end
+    end
+
+    def compare
+      begin
+        sconn = @src_db.pg_conn
+        dconn = @dest_db.pg_conn
+        format = "%-75s | %15s | %15s | %5s\n"
+        printf format, "table", "source", "dest", "diff"
+        puts "-" * (75 + 15 + 15 + 5 + 9)
+        @src_db.tables.each do |t|
+          src_count = @src_db.row_count(t, conn: sconn)
+          dest_count = @dest_db.row_count(t, conn: dconn)
+          marker = src_count == dest_count ? '' : "*"
+          printf format, t, src_count, dest_count, marker
+        end
+      ensure
+        sconn.close if sconn
+        dconn.close if dconn
+      end
+    end
+
+    def finalize
+      stop
+      @dest_db.finalize
+    end
+
+    private 
+
+    def check_deps
+      raise "#{@src_db.user} not a superuser" if not @src_db.superuser?
+      raise "#{@dest_db.user} not a superuser" if not @dest_db.superuser?
+
+    end
+
+    def create_dirs
+      FileUtils.mkdir_p "tmp"
+      FileUtils.mkdir_p "log"
+    end
+
+    def reset_pgpass
+      File.open(PGPASS_PATH, "w") do |f| 
+        f.write "#{@src_db.host}:#{@src_db.port}:postgres:#{@src_db.user}:#{@src_db.pass}\n"
+        f.chmod 0600
+      end
+    end
+
+    def install
+      env = { "DBUSER" => @src_db.user }
+      bucardo "install --batch --pid-dir tmp", db: "postgres", env: env
+    end
+
+    def copy_schema
+      system! %(pg_dump "#{@src_db.conn_str}" -N bucardo --schema-only > #{SCHEMA_PATH})
+      @dest_db.load_schema(SCHEMA_PATH)
+    end
+
+    def add_db
+      bucardo "add db source_db #{@src_db.bucardo_conn_str}"
+      bucardo "add db dest_db #{@dest_db.bucardo_conn_str}"
+    end
+
+    def add_tables
+      bucardo "add all tables db=source_db relgroup=#{RELGROUP} -T public.spatial_ref_sys -N postgis"
+      bucardo "add all sequences db=source_db relgroup=#{RELGROUP}"
+    end
+
+    def add_sync
+      bucardo "add sync #{SYNC_NAME} relgroup=#{RELGROUP} dbs=source_db:source,dest_db:target onetimecopy=2"
+    end
+
+  end
+end

data/lib/pgmove/custom_logger.rb

@@ -0,0 +1,42 @@
+require 'logger'
+require_relative 'ext/string_ext'
+
+module Pgmove
+  class CustomLogger < ::Logger
+
+    using StringExt
+    attr_accessor :trace_mode
+
+    def initialize(file)
+      super(file)
+      @level = ::Logger::INFO
+    end
+
+    def format_message(severity, timestamp, progname, msg)
+      case severity
+      when "INFO"
+        "#{msg}\n"
+      when "ERROR"
+        "#{severity.bold.red} #{msg}\n"
+      when "WARN"
+        "#{severity.downcase.bold.yellow} #{msg}\n"
+      else
+        "#{severity[0].bold.blue} #{msg}\n"
+      end
+    end
+
+    def bullet(msg)
+      info "#{"\u2219".bold.blue} #{msg}"
+    end
+
+    def trace(msg)
+      return if not @trace_mode
+      info %(#{"T".bold.blue} #{msg}\n)
+    end
+
+    def silence!
+      @logdev = nil
+    end
+
+  end
+end

data/lib/pgmove/db.rb

@@ -0,0 +1,124 @@
+require_relative 'helper'
+require 'pg'
+
+module Pgmove
+  class Db
+
+    include Helper
+    attr_reader :user, :pass, :host, :port, :name
+
+    def initialize(name:, user:, pass:, host:, port:, use_tmp: false)
+      @user = user
+      @pass = pass
+      @host = host
+      @port = port
+
+      if use_tmp
+        @name = "#{name}_bucardo_tmp"
+        @final_name = name
+      else
+        @name = name
+      end
+    end
+
+    def reset
+      psql "DROP DATABASE IF EXISTS #{@name}", db: "postgres"
+      psql "CREATE DATABASE #{@name}", db: "postgres"
+    end
+
+    def load_schema(path)
+      psql_raw "-f #{path}"
+    end
+
+    def finalize
+      psql "ALTER DATABASE #{@name} RENAME to #{@final_name}", db: "postgres"
+    end
+
+    def superuser?
+      rows = psql_query "select usesuper from pg_user where usename = CURRENT_USER"
+      rows[0][0] == 't'
+    end
+
+    def conn_str(db: nil)
+      db ||= @name
+      "host=#{@host} port=#{@port} dbname=#{db} user=#{@user} password=#{@pass}"
+    end
+
+    def bucardo_conn_str
+      "dbhost=#{@host} dbport=#{@port} dbname=#{@name} dbuser=#{@user} password=#{@pass}"
+    end
+
+    def tables
+      tables = []
+      schemas.each do |s|
+        sql = "SELECT table_name FROM information_schema.tables \
+             WHERE table_schema='#{s}' AND table_type='BASE TABLE'"
+         rows = psql_query sql
+         rows.each { |r| tables << "#{s}.#{r[0]}" }
+      end
+      tables.sort
+    end
+
+    def schemas
+      sql = "SELECT n.nspname FROM pg_catalog.pg_namespace n WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' AND n.nspname <> 'bucardo'"
+      rows = psql_query sql
+      rows.map { |r| r[0] }
+    end
+
+    def row_count(table, conn:)
+      sql = "select count(*) from #{table}"
+      conn.exec(sql)[0]["count"].to_i
+    rescue
+      -1
+    end
+
+    def compare(other_db)
+      row_counts = row_counts()
+      other_row_counts = other_db.row_counts
+      row_counts.each do |k, v|
+        printf "%-60s %15d %15d\n", k, row_counts[k], other_row_counts[k]
+      end
+    end
+
+    def pg_conn
+      conn = PG::Connection.open(
+        :dbname => @name, 
+        :user => @user, 
+        :password => @pass, 
+        :host => @host, 
+        :port => @port
+      )
+      if block_given?
+        yield conn
+      else
+        conn
+      end
+    ensure
+      if block_given?
+        conn.close if conn
+      end
+    end
+
+    private
+ 
+    def psql(sql, db: nil)
+      psql_raw %(-c "#{sql}"), db: db
+    end
+
+    def psql_raw(params, db: nil)
+      command = %(psql "#{conn_str(db: db)}" #{params})
+      if @final_name
+        system! command
+      else
+        raise "wont run on live db: #{command}"
+      end
+    end
+
+    def psql_query(sql, db: nil)
+      command = %(psql "#{conn_str(db: db)}" -t -A -c "#{sql}")
+      rows = `#{command}`.split("\n")
+      rows.map { |i| i.split('|') }
+    end
+
+  end
+end

data/lib/pgmove/ext/string_ext.rb

@@ -0,0 +1,37 @@
+module Pgmove
+  module StringExt
+    refine String do
+      def colorize(color_code)
+        "\e[#{color_code}m#{self}\e[0m"
+      end 
+
+      def bold
+        "\e[1m#{self}\e[0m"
+      end 
+      
+      def white
+        colorize(37)
+      end 
+
+      def green
+        colorize(32)
+      end 
+
+      def yellow
+        colorize(33)
+      end 
+
+      def red 
+        colorize(31)
+      end 
+
+      def blue
+        colorize(34)
+      end 
+
+      def grey
+        colorize("90")
+      end
+    end
+  end
+end

data/lib/pgmove/helper.rb

@@ -0,0 +1,15 @@
+require_relative 'logger'
+
+module Pgmove
+  module Helper
+
+    include Logger
+
+    def system!(command, display: true, env: {})
+      logger.bullet command if display
+      ok = system env, command
+      raise "Non zero exit: #{command}" if not ok
+    end
+
+  end
+end

data/lib/pgmove/logger.rb

@@ -0,0 +1,26 @@
+require_relative 'custom_logger'
+module Pgmove
+  module Logger
+
+    class << self
+      attr_writer :logger
+    end
+
+    def self.logger
+      @logger ||= CustomLogger.new(STDOUT)
+    end
+
+    def self.stderr
+      @stderr ||= ::Logger.new(STDERR)
+    end
+
+    def logger
+      ::Pgmove::Logger.logger
+    end
+
+    def stderr
+      ::Pgmove::Logger.stderr
+    end
+
+  end
+end

data/lib/pgmove/version.rb

@@ -0,0 +1,3 @@
+module Pgmove
+  VERSION = "0.1.0"
+end

data/lib/pgmove.rb

@@ -0,0 +1,5 @@
+require "pgmove/version"
+
+module Pgmove
+  # Your code goes here...
+end

data/pgmove.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pgmove/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "pgmove"
+  spec.version       = Pgmove::VERSION
+  spec.authors       = ["Neeraj"]
+  spec.email         = [""]
+
+  spec.summary       = %q{A tool to move postgres databases b/w clusters using bucardo}
+  spec.description   = %q{A tool to move postgres databases b/w clusters using bucardo}
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "pg", "~> 0.19.0"
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: pgmove
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Neeraj
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-11-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: A tool to move postgres databases b/w clusters using bucardo
+email:
+- ''
+executables:
+- pgmove
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- dev_exe/pgmove
+- exe/pgmove
+- lib/pgmove.rb
+- lib/pgmove/bucardo.rb
+- lib/pgmove/custom_logger.rb
+- lib/pgmove/db.rb
+- lib/pgmove/ext/string_ext.rb
+- lib/pgmove/helper.rb
+- lib/pgmove/logger.rb
+- lib/pgmove/version.rb
+- pgmove.gemspec
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A tool to move postgres databases b/w clusters using bucardo
+test_files: []