pgsync 0.5.3 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1c278b92d9c3c7848f655be4148555df1739194e959b8aff56ef39228c2656d
-  data.tar.gz: e2620166fef4fd1a785863d28796390ee4c29a7bc301a7b32800116306d9a15b
+  metadata.gz: 531d7975b4942abc5670b8c4ad43717da1326eacacc3b95f5d5797236cb82bb3
+  data.tar.gz: e32dfb82469fa2ebeeea844f0a09e87b8068bd7a201111b90d374eef779381af
 SHA512:
-  metadata.gz: bcd912263fce83a965d790eed4af30f59227b2b55e82cc46e90c109df8c1e3a6e0d7d25f29d14c946bbdc578a3817373a4eac938fe75282586d939a2fba522ea
-  data.tar.gz: 4606494cd7608080cf7863432eaeee6ab1a49fba221ef82da906ccce264e07003eb6458600134f398bb717e556b83339b0b2362d55d7bb62a21ee645678c653f
+  metadata.gz: 4a33a680baaf706ad25fed35eaee3a8c3b191c7a5f6600b4ada42e55143a12226b0e1be189cf4c29d4866982e4f8c7b4915c4517375913083f3210f5fb131598
+  data.tar.gz: a3825c52c5e96c869ea06ed0ec969e1835aec5614f8bd1eb9255995972bdb3e526dd2996802e4d467b1497cfe371509b8a973cc8e3b61988d55f7fb0824d5293

data/CHANGELOG.md

@@ -1,3 +1,40 @@
+## 0.6.2 (2020-06-09)
+
+- Added support for `--disable-integrity` on Amazon RDS
+- Fixed error when excluded table not found in source
+
+## 0.6.1 (2020-06-07)
+
+- Added Django and Laravel integrations
+
+## 0.6.0 (2020-06-07)
+
+- Added messages for different column types and non-deferrable constraints
+- Added support for wildcards to `--exclude`
+- Improved `--overwrite` and `--preserve` options for foreign keys
+- Improved output for schema sync
+- Fixed `--overwrite` and `--preserve` options for multicolumn primary keys
+- Fixed output for notices
+
+Breaking
+
+- Syncs shared tables instead of raising an error when tables missing in destination
+- Raise an error when `--config` or `--db` option provided and config not found
+- Removed deprecated options
+- Dropped support for Postgres < 9.5
+
+## 0.5.5 (2020-05-13)
+
+- Added `--jobs` option
+- Added `--defer-constraints` option
+- Added `--disable-user-triggers` option
+- Added `--disable-integrity` option
+- Improved error message for older libpq
+
+## 0.5.4 (2020-05-09)
+
+- Fixed output for `--in-batches`
+
 ## 0.5.3 (2020-04-03)
 
 - Improved Postgres error messages

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2019 Andrew Kane
+Copyright (c) 2015-2020 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -33,20 +33,26 @@ This creates `.pgsync.yml` for you to customize. We recommend checking this into
 
 ## How to Use
 
-Sync all tables
+First, make sure your schema is set up in both databases. We recommend using a schema migration tool for this, but pgsync also provides a few [convenience methods](#schema). Once that’s done, you’re ready to sync data.
+
+Sync tables
 
 ```sh
 pgsync
 ```
 
-**Note:** pgsync assumes your schema is setup in your `to` database. See the [schema section](#schema) if that’s not the case.
-
 Sync specific tables
 
 ```sh
 pgsync table1,table2
 ```
 
+Works with wildcards as well
+
+```sh
+pgsync "table*"
+```
+
 Sync specific rows (existing rows are overwritten)
 
 ```sh
@@ -65,13 +71,15 @@ Or truncate them
 pgsync products "where store_id = 1" --truncate
 ```
 
-### Exclude Tables
+## Tables
+
+Exclude specific tables
 
 ```sh
-pgsync --exclude users
+pgsync --exclude table1,table2
 ```
 
-To always exclude, add to `.pgsync.yml`.
+Add to `.pgsync.yml` to exclude by default
 
 ```yml
 exclude:
@@ -79,15 +87,17 @@ exclude:
   - table2
 ```
 
-For Rails, you probably want to exclude schema migrations and ActiveRecord metadata.
+Sync tables from all schemas or specific schemas (by default, only the search path is synced)
 
-```yml
-exclude:
-  - schema_migrations
-  - ar_internal_metadata
+```sh
+pgsync --all-schemas
+# or
+pgsync --schemas public,other
+# or
+pgsync public.table1,other.table2
 ```
 
-### Groups
+## Groups
 
 Define groups in `.pgsync.yml`:
 
@@ -104,6 +114,8 @@ And run:
 pgsync group1
 ```
 
+## Variables
+
 You can also use groups to sync a specific record and associated records in other tables.
 
 To get product `123` with its reviews, last 10 coupons, and store, use:
@@ -123,18 +135,14 @@ And run:
 pgsync product:123
 ```
 
-### Schema
-
-**Note:** pgsync is designed to sync data. You should use a schema migration tool to manage schema changes. The methods in this section are provided for convenience but not recommended.
+## Schema
 
-Sync schema before the data
+Sync schema before the data (this wipes out existing data)
 
 ```sh
 pgsync --schema-first
 ```
 
-**Note:** This wipes out existing data
-
 Specify tables
 
 ```sh
@@ -149,13 +157,9 @@ pgsync --schema-only
 
 pgsync does not try to sync Postgres extensions.
 
-## Data Protection
-
-Always make sure your [connection is secure](https://ankane.org/postgres-sslmode-explained) when connecting to a database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
-
-## Sensitive Information
+## Sensitive Data
 
-Prevent sensitive information like email addresses from leaving the remote server.
+Prevent sensitive data like email addresses from leaving the remote server.
 
 Define rules in `.pgsync.yml`:
 
@@ -188,7 +192,63 @@ Options for replacement are:
 - `null`
 - `untouched`
 
-Rules starting with `unique_` require the table to have a primary key. `unique_phone` requires a numeric primary key.
+Rules starting with `unique_` require the table to have a single column primary key. `unique_phone` requires a numeric primary key.
+
+## Foreign Keys
+
+Foreign keys can make it difficult to sync data. Three options are:
+
+1. Manually specify the order of tables
+2. Use deferrable constraints
+3. Disable foreign key triggers, which can silently break referential integrity
+
+When manually specifying the order, use `--jobs 1` so tables are synced one-at-a-time.
+
+```sh
+pgsync table1,table2,table3 --jobs 1
+```
+
+If your tables have [deferrable constraints](https://begriffs.com/posts/2017-08-27-deferrable-sql-constraints.html), use:
+
+```sh
+pgsync --defer-constraints
+```
+
+To disable foreign key triggers and potentially break referential integrity, use:
+
+```sh
+pgsync --disable-integrity
+```
+
+This requires superuser privileges on the `to` database. If syncing to (not from) Amazon RDS, use the `rds_superuser` role. If syncing to (not from) Heroku, there doesn’t appear to be a way to disable integrity.
+
+## Triggers
+
+Disable user triggers with:
+
+```sh
+pgsync --disable-user-triggers
+```
+
+## Append-Only Tables
+
+For extremely large, append-only tables, sync in batches.
+
+```sh
+pgsync large_table --in-batches
+```
+
+The script will resume where it left off when run again, making it great for backfills.
+
+## Connection Security
+
+Always make sure your [connection is secure](https://ankane.org/postgres-sslmode-explained) when connecting to a database over a network you don’t fully trust. Your best option is to connect over SSH or a VPN. Another option is to use `sslmode=verify-full`. If you don’t do this, your database credentials can be compromised.
+
+## Safety
+
+To keep you from accidentally overwriting production, the destination is limited to `localhost` or `127.0.0.1` by default.
+
+To use another host, add `to_safe: true` to your `.pgsync.yml`.
 
 ## Multiple Databases
 
@@ -204,31 +264,50 @@ This creates `.pgsync-db2.yml` for you to edit. Specify a database in commands w
 pgsync --db db2
 ```
 
-## Safety
+## Integrations
 
-To keep you from accidentally overwriting production, the destination is limited to `localhost` or `127.0.0.1` by default.
+- [Django](#django)
+- [Heroku](#heroku)
+- [Laravel](#laravel)
+- [Rails](#rails)
 
-To use another host, add `to_safe: true` to your `.pgsync.yml`.
+### Django
+
+If you run `pgsync --init` in a Django project, migrations will be excluded in `.pgsync.yml`.
+
+```yml
+exclude:
+  - django_migrations
+```
 
-## Large Tables
+### Heroku
 
-For extremely large tables, sync in batches.
+If you run `pgsync --init` in a Heroku project, the `from` database will be set in `.pgsync.yml`.
 
-```sh
-pgsync large_table --in-batches
+```yml
+from: $(heroku config:get DATABASE_URL)?sslmode=require
 ```
 
-The script will resume where it left off when run again, making it great for backfills.
+### Laravel
 
-## Foreign Keys
+If you run `pgsync --init` in a Laravel project, migrations will be excluded in `.pgsync.yml`.
 
-By default, tables are copied in parallel. If you use foreign keys, this can cause violations. You can specify tables to be copied serially with:
+```yml
+exclude:
+  - migrations
+```
 
-```sh
-pgsync group1 --debug
+### Rails
+
+If you run `pgsync --init` in a Rails project, Active Record metadata and schema migrations will be excluded in `.pgsync.yml`.
+
+```yml
+exclude:
+  - ar_internal_metadata
+  - schema_migrations
 ```
 
-## Reference
+## Other Commands
 
 Help
 
@@ -242,6 +321,12 @@ Version
 pgsync --version
 ```
 
+List tables
+
+```sh
+pgsync --list
+```
+
 ## Scripts
 
 Use groups when possible to take advantage of parallelism.

data/config.yml

@@ -20,6 +20,10 @@ to: postgres://localhost:5432/myapp_development
 #     - table1
 #     - table2
 
+# sync specific schemas
+# schemas:
+#   - public
+
 # protect sensitive information
 data_rules:
   email: unique_email

data/lib/pgsync.rb

@@ -10,15 +10,19 @@ require "shellwords"
 require "tempfile"
 require "uri"
 require "yaml"
+require "open3"
 
 # modules
 require "pgsync/utils"
 require "pgsync/client"
 require "pgsync/data_source"
 require "pgsync/init"
+require "pgsync/schema_sync"
 require "pgsync/sync"
-require "pgsync/table_list"
+require "pgsync/table"
 require "pgsync/table_sync"
+require "pgsync/task"
+require "pgsync/task_resolver"
 require "pgsync/version"
 
 module PgSync

data/lib/pgsync/client.rb

@@ -7,73 +7,74 @@ module PgSync
       output.sync = true
     end
 
-    def perform(testing: true)
-      opts = parse_args
+    def perform
+      result = Slop::Parser.new(slop_options).parse(@args)
+      arguments = result.arguments
+      options = result.to_h
 
-      # TODO throw error in 0.6.0
-      warn "Specify either --db or --config, not both" if opts[:db] && opts[:config]
+      raise Error, "Specify either --db or --config, not both" if options[:db] && options[:config]
+      raise Error, "Cannot use --overwrite with --in-batches" if options[:overwrite] && options[:in_batches]
 
-      if opts.version?
+      if options[:version]
         log VERSION
-      elsif opts.help?
-        log opts
-      # TODO remove deprecated conditions (last two)
-      elsif opts.init? || opts.setup? || opts.arguments[0] == "setup"
-        Init.new.perform(opts)
+      elsif options[:help]
+        log slop_options
+      elsif options[:init]
+        Init.new(arguments, options).perform
       else
-        Sync.new.perform(opts)
+        Sync.new(arguments, options).perform
       end
-    rescue Error, PG::ConnectionBad => e
-      raise e if testing
-      abort colorize(e.message, :red)
+    rescue => e
+      # Error, PG::ConnectionBad, Slop::Error
+      raise e if options && options[:debug]
+      abort colorize(e.message.strip, :red)
     end
 
     def self.start
-      new(ARGV).perform(testing: false)
+      new(ARGV).perform
     end
 
     protected
 
-    def parse_args
-      Slop.parse(@args) do |o|
-        o.banner = %{Usage:
-    pgsync [options]
+    def slop_options
+      o = Slop::Options.new
+      o.banner = %{Usage:
+  pgsync [options]
 
 Options:}
-        o.string "-d", "--db", "database"
-        o.string "-t", "--tables", "tables to sync"
-        o.string "-g", "--groups", "groups to sync"
-        o.string "--schemas", "schemas to sync"
-        o.string "--from", "source"
-        o.string "--to", "destination"
-        o.string "--where", "where", help: false
-        o.integer "--limit", "limit", help: false
-        o.string "--exclude", "exclude tables"
-        o.string "--config", "config file"
-        # TODO much better name for this option
-        o.boolean "--to-safe", "accept danger", default: false
-        o.boolean "--debug", "debug", default: false
-        o.boolean "--list", "list", default: false
-        o.boolean "--overwrite", "overwrite existing rows", default: false, help: false
-        o.boolean "--preserve", "preserve existing rows", default: false
-        o.boolean "--truncate", "truncate existing rows", default: false
-        o.boolean "--schema-first", "schema first", default: false
-        o.boolean "--schema-only", "schema only", default: false
-        o.boolean "--all-schemas", "all schemas", default: false
-        o.boolean "--no-rules", "do not apply data rules", default: false
-        o.boolean "--no-sequences", "do not sync sequences", default: false
-        o.boolean "--init", "init", default: false
-        o.boolean "--setup", "setup", default: false, help: false
-        o.boolean "--in-batches", "in batches", default: false, help: false
-        o.integer "--batch-size", "batch size", default: 10000, help: false
-        o.float "--sleep", "sleep", default: 0, help: false
-        o.boolean "--fail-fast", "stop on the first failed table", default: false
-        # o.array "--var", "pass a variable"
-        o.boolean "-v", "--version", "print the version"
-        o.boolean "-h", "--help", "prints help"
-      end
-    rescue Slop::Error => e
-      raise Error, e.message
+      o.string "-d", "--db", "database"
+      o.string "-t", "--tables", "tables to sync"
+      o.string "-g", "--groups", "groups to sync"
+      o.integer "-j", "--jobs", "number of tables to sync at a time"
+      o.string "--schemas", "schemas to sync"
+      o.string "--from", "source"
+      o.string "--to", "destination"
+      o.string "--exclude", "exclude tables"
+      o.string "--config", "config file"
+      o.boolean "--to-safe", "accept danger", default: false
+      o.boolean "--debug", "debug", default: false
+      o.boolean "--list", "list", default: false
+      o.boolean "--overwrite", "overwrite existing rows", default: false, help: false
+      o.boolean "--preserve", "preserve existing rows", default: false
+      o.boolean "--truncate", "truncate existing rows", default: false
+      o.boolean "--schema-first", "schema first", default: false
+      o.boolean "--schema-only", "schema only", default: false
+      o.boolean "--all-schemas", "all schemas", default: false
+      o.boolean "--no-rules", "do not apply data rules", default: false
+      o.boolean "--no-sequences", "do not sync sequences", default: false
+      o.boolean "--init", "init", default: false
+      o.boolean "--in-batches", "in batches", default: false, help: false
+      o.integer "--batch-size", "batch size", default: 10000, help: false
+      o.float "--sleep", "sleep", default: 0, help: false
+      o.boolean "--fail-fast", "stop on the first failed table", default: false
+      o.boolean "--defer-constraints", "defer constraints", default: false
+      o.boolean "--disable-user-triggers", "disable non-system triggers", default: false
+      o.boolean "--disable-integrity", "disable foreign key triggers", default: false
+      # private, for testing
+      o.boolean "--disable-integrity-v2", "disable foreign key triggers", default: false, help: false
+      o.boolean "-v", "--version", "print the version"
+      o.boolean "-h", "--help", "prints help"
+      o
     end
   end
 end