pgsync 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 58a427f20bdf42cd97ca4101964a571ab7f5da76
+  data.tar.gz: 13da001cf27f6ddda55a96b5f9ec3a8e27577eb5
+SHA512:
+  metadata.gz: 7399365349da94533619effdeee4c80d05817384b75377af12c4d2f05d1cc2af6b9681dcc0e9b5d4cf8c3616e9f7f048b6b3fd7ca765d74b97437f29a503e04c
+  data.tar.gz: 15abafa358b77760f493d28f7c20ad8c44cf2e5a6e3ed7f5432f2b2b50c0aa67c465219026f705e1e37da4830c6881fc579ccb756ea694a05429c92c4f270112

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in pgsync.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Andrew Kane
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,149 @@
+# pgsync
+
+Quickly and securely sync data between environments
+
+## Installation
+
+```sh
+gem install pgsync
+```
+
+And in your project directory, run:
+
+```sh
+pgsync setup
+```
+
+This creates a `.pgsync.yml` for you to customize. We recommend checking this into your version control (assuming it doesn’t contain sensitive information).
+
+## How to Use
+
+Fetch all tables
+
+```sh
+pgsync
+```
+
+Fetch specific tables
+
+```sh
+pgsync table1,table2
+```
+
+Fetch specific rows (truncates destination table first)
+
+```sh
+pgsync products --where "id < 100"
+```
+
+### Exclude Tables
+
+```sh
+pgsync --exclude users
+```
+
+To always exclude, add to `.pgsync.yml`.
+
+```yml
+exclude:
+  - table1
+  - table2
+```
+
+For Rails, you probably want to exclude schema migrations.
+
+```yml
+exclude:
+  - schema_migrations
+```
+
+### Schema
+
+Fetch schema
+
+```sh
+pgsync schema
+```
+
+Specify tables
+
+```sh
+pgsync schema table1,table2
+```
+
+### Groups
+
+Define groups in `.pgsync.yml`:
+
+```yml
+groups:
+  group1:
+    - table1
+    - table2
+  group2:
+    - table3
+    - table4
+```
+
+And run:
+
+```sh
+pgsync groups group1,group2
+```
+
+## Senstive Information
+
+Prevent sensitive information - like passwords and email addresses - from leaving the remote server.
+
+Define rules in `.pgsync.yml`:
+
+```yml
+data_rules:
+  email: unique_email
+  last_name: random_letter
+  birthday: random_date
+  users.auth_token:
+    value: secret
+  visits_count:
+    statement: "(RANDOM() * 10)::int"
+  encrypted_*: null
+```
+
+`last_name` matches all columns named `last_name` and `users.last_name` matches only the users table. Wildcards are supported, and the first matching rule is applied.
+
+Options for replacement are:
+
+- null
+- value
+- statement
+- unique_email
+- unique_phone
+- random_letter
+- random_int
+- random_date
+- random_time
+- random_ip
+- untouched
+
+## Safety
+
+To keep you from accidentally overwriting production, the destination is limited to `localhost` or `127.0.0.1` by default.
+
+To use another host, add `to_safe: true` to your `.pgsync.yml`.
+
+## Thanks
+
+Inspired by [heroku-pg-transfer](https://github.com/ddollar/heroku-pg-transfer).
+
+## TODO
+
+- Support for schemas other than `public`
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/pgsync/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/pgsync/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+task :default => :test
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+end

data/bin/pgsync

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require "pgsync"
+begin
+  PgSync::Client.new(ARGV).perform
+rescue PgSync::Error => e
+  abort e.message
+end

data/config.yml

@@ -0,0 +1,27 @@
+# source database URL
+# database URLs take the format of:
+#   postgres://user:password@host:port/dbname
+# we recommend a command which outputs a database URL
+# so sensitive information is not included in this file
+from: $(heroku config:get DATABASE_URL)
+
+# destination database URL
+to: postgres://localhost:5432/myapp_development
+
+# exclude tables
+# exclude:
+#   - schema_migrations
+
+# define groups
+# groups:
+#   group1:
+#     - table1
+#     - table2
+
+# protect sensitive information
+data_rules:
+  email: unique_email
+  phone: unique_phone
+  last_name: random_letter
+  birthday: random_date
+  encrypted_*: null

data/lib/pgsync.rb

@@ -0,0 +1,366 @@
+require "pgsync/version"
+require "yaml"
+require "slop"
+require "uri"
+require "erb"
+require "pg"
+require "parallel"
+require "multiprocessing"
+require "fileutils"
+
+module PgSync
+  class Error < StandardError; end
+  class Rollback < StandardError; end
+
+  class Client
+    def initialize(args)
+      $stdout.sync = true
+      @arguments, @options = parse_args(args)
+      @config_file = @options[:config]
+      @mutex = MultiProcessing::Mutex.new
+    end
+
+    # TODO clean up this mess
+    def perform
+      start_time = Time.now
+
+      args, opts = @arguments, @options
+      [:to, :from, :to_safe, :exclude].each do |opt|
+        opts[opt] ||= config[opt.to_s]
+      end
+      command = args[0]
+
+      if command == "setup"
+        setup
+      else
+        source = parse_source(opts[:from])
+        abort "No source" unless source
+        source_uri = parse_uri(source)
+
+        destination = parse_source(opts[:to])
+        abort "No destination" unless destination
+        destination_uri = parse_uri(destination)
+        abort "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1" unless %(localhost 127.0.0.1).include?(destination_uri.host) || opts[:to_safe]
+
+        print_uri("From", source_uri)
+        print_uri("To", destination_uri)
+
+        if args[0] == "schema"
+          time =
+            benchmark do
+              puts "* Dumping schema"
+              tables = to_arr(args[1]).map { |t| "-t #{t}" }.join(" ")
+              dump_command = "pg_dump --verbose --schema-only --no-owner --no-acl --clean #{tables} #{to_url(source_uri)}"
+              restore_command = "psql -q -d #{to_url(destination_uri)}"
+              system("#{dump_command} | #{restore_command}")
+            end
+
+          puts "* DONE (#{time.round(1)}s)"
+        else
+          from_uri = source_uri
+          to_uri = destination_uri
+
+          tables =
+            if args[0] == "groups"
+              specified_groups = to_arr(args[1])
+              specified_groups.map do |group|
+                if (tables = config["groups"][group])
+                  tables
+                else
+                  abort "Group not found: #{group}"
+                end
+              end.flatten
+            elsif args[0] == "tables"
+              to_arr(args[1])
+            elsif args[0]
+              to_arr(args[0])
+            else
+              nil
+            end
+
+          with_connection(from_uri, timeout: 3) do |conn|
+            tables ||= self.tables(conn, "public") - to_arr(opts[:exclude])
+
+            tables.each do |table|
+              unless table_exists?(conn, table, "public")
+                abort "Table does not exist in source: #{table}"
+              end
+            end
+          end
+
+          with_connection(to_uri, timeout: 3) do |conn|
+            tables.each do |table|
+              unless table_exists?(conn, table, "public")
+                abort "Table does not exist in destination: #{table}"
+              end
+            end
+          end
+
+          Parallel.each(tables) do |table|
+            time =
+              benchmark do
+                with_connection(from_uri) do |from_connection|
+                  with_connection(to_uri) do |to_connection|
+                    bad_fields = config["data_rules"]
+
+                    from_fields = columns(from_connection, table, "public")
+                    to_fields = columns(to_connection, table, "public")
+                    shared_fields = to_fields & from_fields
+                    extra_fields = to_fields - from_fields
+                    missing_fields = from_fields - to_fields
+
+                    from_sequences = sequences(from_connection, table, shared_fields)
+                    to_sequences = sequences(to_connection, table, shared_fields)
+                    shared_sequences = to_sequences & from_sequences
+                    extra_sequences = to_sequences - from_sequences
+                    missing_sequences = from_sequences - to_sequences
+
+                    where = opts[:where]
+
+                    @mutex.synchronize do
+                      puts "* Syncing #{table}"
+                      if where
+                        puts "    #{where}"
+                        where = " WHERE #{opts[:where]}"
+                      end
+                      puts "EXTRA COLUMNS: #{extra_fields}" if extra_fields.any?
+                      puts "MISSING COLUMNS: #{missing_fields}" if missing_fields.any?
+                      puts "EXTRA SEQUENCES: #{extra_sequences}" if extra_sequences.any?
+                      puts "MISSING SEQUENCES: #{missing_sequences}" if missing_sequences.any?
+                    end
+
+                    if shared_fields.empty?
+                      abort "No fields to copy: #{table}"
+                    end
+
+                    copy_fields = shared_fields.map { |f| f2 = bad_fields.to_a.find { |bf, bk| rule_match?(table, f, bf) }; f2 ? "#{apply_strategy(f2[1], f, from_connection)} AS #{escape_identifier(f)}" : escape_identifier(f) }.join(", ")
+                    fields = shared_fields.map { |f| escape_identifier(f) }.join(", ")
+
+                    seq_values = {}
+                    shared_sequences.each do |seq|
+                      seq_values[seq] = from_connection.exec("select last_value from #{seq}").to_a[0]["last_value"]
+                    end
+
+                    # use transaction to revert statement timeout
+                    begin
+                      from_connection.transaction do |fconn|
+                        fconn.exec("SET statement_timeout = 0")
+                        to_connection.exec("TRUNCATE #{table} CASCADE")
+                        to_connection.copy_data "COPY #{table} (#{fields}) FROM STDIN" do
+                          fconn.copy_data "COPY (SELECT #{copy_fields} FROM #{table}#{where}) TO STDOUT" do
+                            while row = fconn.get_copy_data
+                              to_connection.put_copy_data(row)
+                            end
+                          end
+                        end
+                        seq_values.each do |seq, value|
+                          to_connection.exec("SELECT setval(#{escape(seq)}, #{escape(value)})")
+                        end
+                        raise PgSync::Rollback
+                      end
+                    rescue PgSync::Rollback
+                      # success
+                    end
+                  end
+                end
+              end
+
+            @mutex.synchronize do
+              puts "* DONE #{table} (#{time.round(1)}s)"
+            end
+          end
+
+          time = Time.now - start_time
+          puts "Completed in #{time.round(1)}s"
+        end
+      end
+      true
+    end
+
+    protected
+
+    def parse_args(args)
+      opts = Slop.parse(args) do |o|
+        o.banner = "usage: pgsync [options]"
+        o.string "--from", "source"
+        o.string "--to", "destination"
+        o.string "--where", "where"
+        o.string "--exclude", "exclude tables"
+        o.string "--config", "config file", default: ".pgsync.yml"
+        # TODO much better name for this option
+        o.boolean "--to-safe", "accept danger", default: false
+        o.on "-v", "--version", "print the version" do
+          puts PgSync::VERSION
+          exit
+        end
+        o.on "-h", "--help", "prints help" do
+          puts o
+          exit
+        end
+      end
+      [opts.arguments, opts.to_hash]
+    rescue Slop::Error => e
+      abort e.message
+    end
+
+    # TODO look down path
+    def config
+      @config ||= begin
+        if File.exist?(@config_file)
+          begin
+            YAML.load_file(@config_file) || {}
+          rescue Psych::SyntaxError => e
+            raise PgSync::Error, e.message
+          end
+        else
+          {}
+        end
+      end
+    end
+
+    def parse_source(source)
+      if source && source[0..1] == "$(" && source[-1] == ")"
+        source = `#{source[2..-2]}`.chomp
+      end
+      source
+    end
+
+    def setup
+      if File.exist?(@config_file)
+        abort "#{@config_file} exists."
+      else
+        FileUtils.cp(File.dirname(__FILE__) + "/../config.yml", @config_file)
+        puts "#{@config_file} created. Add your database credentials."
+      end
+    end
+
+    def with_connection(uri, timeout: 0)
+      conn =
+        PG::Connection.new(
+          host: uri.host,
+          port: uri.port,
+          dbname: uri.path.sub(/\A\//, ""),
+          user: uri.user,
+          password: uri.password,
+          connect_timeout: timeout
+        )
+      begin
+        yield conn
+      ensure
+        conn.close
+      end
+    rescue PG::ConnectionBad => e
+      puts
+      abort e.message
+    end
+
+    def benchmark
+      start_time = Time.now
+      yield
+      Time.now - start_time
+    end
+
+    def tables(conn, schema)
+      query = "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = $1 ORDER BY tablename ASC"
+      conn.exec_params(query, [schema]).to_a.map { |row| row["tablename"] }
+    end
+
+    def columns(conn, table, schema)
+      query = "SELECT column_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2"
+      conn.exec_params(query, [schema, table]).to_a.map { |row| row["column_name"] }
+    end
+
+    def table_exists?(conn, table, schema)
+      query = "SELECT 1 FROM information_schema.tables WHERE table_schema = $1 AND table_name = $2"
+      conn.exec_params(query, [schema, table]).to_a.size > 0
+    end
+
+    # TODO better performance
+    def rule_match?(table, column, rule)
+      regex = Regexp.new('\A' + Regexp.escape(rule).gsub('\*','[^\.]*') + '\z')
+      regex.match(column) || regex.match("#{table}.#{column}")
+    end
+
+    # TODO wildcard rules
+    def apply_strategy(rule, column, conn)
+      if rule.is_a?(Hash)
+        if rule.key?("value")
+          escape(rule["value"])
+        elsif rule.key?("statement")
+          rule["statement"]
+        else
+          abort "Unknown rule #{rule.inspect} for column #{column}"
+        end
+      else
+        strategies = {
+          "unique_email" => "'email' || id || '@example.org'",
+          "untouched" => escape_identifier(column),
+          "unique_phone" => "(id + 1000000000)::text",
+          "random_int" => "(RAND() * 10)::int",
+          "random_date" => "'1970-01-01'",
+          "random_time" => "NOW()",
+          "unique_secret" => "'secret' || id",
+          "random_ip" => "'127.0.0.1'",
+          "random_letter" => "'A'",
+          "null" => "NULL",
+          nil => "NULL"
+        }
+        if strategies[rule]
+          strategies[rule]
+        else
+          abort "Unknown rule #{rule} for column #{column}"
+        end
+      end
+    end
+
+    def escape_identifier(value)
+      PG::Connection.quote_ident(value)
+    end
+
+    def escape(value)
+      if value.is_a?(String)
+        "'#{quote_string(value)}'"
+      else
+        value
+      end
+    end
+
+    # activerecord
+    def quote_string(s)
+      s.gsub(/\\/, '\&\&').gsub(/'/, "''")
+    end
+
+    def to_arr(value)
+      if value.is_a?(Array)
+        value
+      else
+        value.to_s.split(",")
+      end
+    end
+
+    def parse_uri(url)
+      uri = URI.parse(url)
+      uri.host ||= "localhost"
+      uri.port ||= 5432
+      uri
+    end
+
+    def print_uri(prefix, uri)
+      puts "#{prefix}: #{uri.path.sub(/\A\//, '')} on #{uri.host}:#{uri.port}"
+    end
+
+    def to_url(uri)
+      uri = uri.dup
+      uri.query = nil
+      uri.to_s
+    end
+
+    def abort(message)
+      raise PgSync::Error, message
+    end
+
+    def sequences(conn, table, columns)
+      conn.exec("SELECT #{columns.map { |f| "pg_get_serial_sequence(#{escape(table)}, #{escape(f)}) AS #{f}" }.join(", ")}").to_a[0].values.compact
+    end
+  end
+end

data/lib/pgsync/version.rb

@@ -0,0 +1,3 @@
+module PgSync
+  VERSION = "0.1.0"
+end

data/pgsync.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "pgsync/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "pgsync"
+  spec.version       = PgSync::VERSION
+  spec.authors       = ["Andrew Kane"]
+  spec.email         = ["andrew@chartkick.com"]
+
+  spec.summary       = "Quickly and securely sync data between environments"
+  spec.homepage      = "https://github.com/ankane/pgsync"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "slop", ">= 4.2.0"
+  spec.add_dependency "pg"
+  spec.add_dependency "parallel"
+  spec.add_dependency "multiprocessing"
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+end

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: pgsync
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andrew Kane
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: multiprocessing
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- andrew@chartkick.com
+executables:
+- pgsync
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/pgsync
+- config.yml
+- lib/pgsync.rb
+- lib/pgsync/version.rb
+- pgsync.gemspec
+homepage: https://github.com/ankane/pgsync
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Quickly and securely sync data between environments
+test_files: []
+has_rdoc: