pgsync 0.5.5 → 0.6.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32d1e60a133659dde98d3564d21fca4f1866e8f7b58518ef71f6dcaf59cf8c82
-  data.tar.gz: 650b50aee614ed737d2e99278cbb8910f7980a041dcf58d18288f6bc53826635
+  metadata.gz: 0ecb467f4d3112edfcfebe19913e29daeddfe1e4cc10e18bd9e8850fbceced57
+  data.tar.gz: 4d8309dec9e8238074267baf583fe333c7b1d3300ad80398748e15905eb1e539
 SHA512:
-  metadata.gz: 8bd3e4373558c8f8519f0c118f51b55ac27a3a82fb1ab9a38e85bfee858e08a80fb2c37cab490878d2dda06121c13f45d81b3df6c87b8ede0e0060a50b94fee6
-  data.tar.gz: 922ecb8f57615025e5b92a93f82010bb667fbc968db12ebddeb49507ce90d6dd4c3c16ea3cd5777817bb321a8fefa54f6483b6ffae872f897e7ada64d68831b0
+  metadata.gz: 145eb31e2565257a5adedc05c1308692dffb1e0785728ecd38230bb23c34c4ba6ecf08512201f707f457b136e0b7a22ca511b46fb7834f2045c041ff43dd6ccc
+  data.tar.gz: cde51dee36149f9f8c21e26eecd504de68d8a47c2524f084b74962d7ceae1a01b59841261fc6c112cf94b31a89f55b9a991c05d1af0db41c46ed615fc56e888f

data/CHANGELOG.md

@@ -1,9 +1,44 @@
+## 0.6.4 (2020-06-10)
+
+- Log SQL with `--debug` option
+- Improved sequence queries
+
+## 0.6.3 (2020-06-09)
+
+- Added `--defer-constraints-v2` option
+- Ensure consistent source snapshot with `--disable-integrity`
+
+## 0.6.2 (2020-06-09)
+
+- Added support for `--disable-integrity` on Amazon RDS
+- Fixed error when excluded table not found in source
+
+## 0.6.1 (2020-06-07)
+
+- Added Django and Laravel integrations
+
+## 0.6.0 (2020-06-07)
+
+- Added messages for different column types and non-deferrable constraints
+- Added support for wildcards to `--exclude`
+- Improved `--overwrite` and `--preserve` options for foreign keys
+- Improved output for schema sync
+- Fixed `--overwrite` and `--preserve` options for multicolumn primary keys
+- Fixed output for notices
+
+Breaking
+
+- Syncs shared tables instead of raising an error when tables missing in destination
+- Raise an error when `--config` or `--db` option provided and config not found
+- Removed deprecated options
+- Dropped support for Postgres < 9.5
+
 ## 0.5.5 (2020-05-13)
 
 - Added `--jobs` option
-- Added experimental `--defer-constraints` option
-- Added experimental `--disable-user-triggers` option
-- Added experimental `--disable-integrity` option
+- Added `--defer-constraints` option
+- Added `--disable-user-triggers` option
+- Added `--disable-integrity` option
 - Improved error message for older libpq
 
 ## 0.5.4 (2020-05-09)

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2019 Andrew Kane
+Copyright (c) 2015-2020 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -33,20 +33,26 @@ This creates `.pgsync.yml` for you to customize. We recommend checking this into
 
 ## How to Use
 
-Sync all tables
+First, make sure your schema is set up in both databases. We recommend using a schema migration tool for this, but pgsync also provides a few [convenience methods](#schema). Once that’s done, you’re ready to sync data.
+
+Sync tables
 
 ```sh
 pgsync
 ```
 
-**Note:** pgsync assumes your schema is setup in your `to` database. See the [schema section](#schema) if that’s not the case.
-
 Sync specific tables
 
 ```sh
 pgsync table1,table2
 ```
 
+Works with wildcards as well
+
+```sh
+pgsync "table*"
+```
+
 Sync specific rows (existing rows are overwritten)
 
 ```sh
@@ -65,13 +71,15 @@ Or truncate them
 pgsync products "where store_id = 1" --truncate
 ```
 
-## Exclude Tables
+## Tables
+
+Exclude specific tables
 
 ```sh
-pgsync --exclude users
+pgsync --exclude table1,table2
 ```
 
-To always exclude, add to `.pgsync.yml`.
+Add to `.pgsync.yml` to exclude by default
 
 ```yml
 exclude:
@@ -79,12 +87,14 @@ exclude:
   - table2
 ```
 
-For Rails, you probably want to exclude schema migrations and Active Record metadata.
+Sync tables from all schemas or specific schemas (by default, only the search path is synced)
 
-```yml
-exclude:
-  - schema_migrations
-  - ar_internal_metadata
+```sh
+pgsync --all-schemas
+# or
+pgsync --schemas public,other
+# or
+pgsync public.table1,other.table2
 ```
 
 ## Groups
@@ -104,6 +114,8 @@ And run:
 pgsync group1
 ```
 
+## Variables
+
 You can also use groups to sync a specific record and associated records in other tables.
 
 To get product `123` with its reviews, last 10 coupons, and store, use:
@@ -125,16 +137,12 @@ pgsync product:123
 
 ## Schema
 
-**Note:** pgsync is designed to sync data. You should use a schema migration tool to manage schema changes. The methods in this section are provided for convenience but not recommended.
-
-Sync schema before the data
+Sync schema before the data (this wipes out existing data)
 
 ```sh
 pgsync --schema-first
 ```
 
-**Note:** This wipes out existing data
-
 Specify tables
 
 ```sh
@@ -149,9 +157,9 @@ pgsync --schema-only
 
 pgsync does not try to sync Postgres extensions.
 
-## Sensitive Information
+## Sensitive Data
 
-Prevent sensitive information like email addresses from leaving the remote server.
+Prevent sensitive data like email addresses from leaving the remote server.
 
 Define rules in `.pgsync.yml`:
 
@@ -184,37 +192,35 @@ Options for replacement are:
 - `null`
 - `untouched`
 
-Rules starting with `unique_` require the table to have a primary key. `unique_phone` requires a numeric primary key.
+Rules starting with `unique_` require the table to have a single column primary key. `unique_phone` requires a numeric primary key.
 
 ## Foreign Keys
 
 Foreign keys can make it difficult to sync data. Three options are:
 
-1. Manually specify the order of tables
-2. Use deferrable constraints
-3. Disable triggers, which can silently break referential integrity
+1. Defer constraints (recommended)
+2. Manually specify the order of tables
+3. Disable foreign key triggers, which can silently break referential integrity (not recommended)
 
-When manually specifying the order, use `--jobs 1` so tables are synced one-at-a-time.
+To defer constraints, use:
 
 ```sh
-pgsync table1,table2,table3 --jobs 1
+pgsync --defer-constraints-v2
 ```
 
-If your tables have [deferrable constraints](https://begriffs.com/posts/2017-08-27-deferrable-sql-constraints.html), use:
+To manually specify the order of tables, use `--jobs 1` so tables are synced one-at-a-time.
 
 ```sh
-pgsync --defer-constraints
+pgsync table1,table2,table3 --jobs 1
 ```
 
-**Note:** This feature is currently experimental.
-
-To disable triggers and potentially break referential integrity, use:
+To disable foreign key triggers and potentially break referential integrity, use:
 
 ```sh
 pgsync --disable-integrity
 ```
 
-**Note:** This feature is currently experimental.
+This requires superuser privileges on the `to` database. If syncing to (not from) Amazon RDS, use the `rds_superuser` role. If syncing to (not from) Heroku, there doesn’t appear to be a way to disable integrity.
 
 ## Triggers
 
@@ -224,8 +230,6 @@ Disable user triggers with:
 pgsync --disable-user-triggers
 ```
 
-**Note:** This feature is currently experimental.
-
 ## Append-Only Tables
 
 For extremely large, append-only tables, sync in batches.
@@ -260,6 +264,57 @@ This creates `.pgsync-db2.yml` for you to edit. Specify a database in commands w
 pgsync --db db2
 ```
 
+## Integrations
+
+- [Django](#django)
+- [Heroku](#heroku)
+- [Laravel](#laravel)
+- [Rails](#rails)
+
+### Django
+
+If you run `pgsync --init` in a Django project, migrations will be excluded in `.pgsync.yml`.
+
+```yml
+exclude:
+  - django_migrations
+```
+
+### Heroku
+
+If you run `pgsync --init` in a Heroku project, the `from` database will be set in `.pgsync.yml`.
+
+```yml
+from: $(heroku config:get DATABASE_URL)?sslmode=require
+```
+
+### Laravel
+
+If you run `pgsync --init` in a Laravel project, migrations will be excluded in `.pgsync.yml`.
+
+```yml
+exclude:
+  - migrations
+```
+
+### Rails
+
+If you run `pgsync --init` in a Rails project, Active Record metadata and schema migrations will be excluded in `.pgsync.yml`.
+
+```yml
+exclude:
+  - ar_internal_metadata
+  - schema_migrations
+```
+
+## Debugging
+
+To view the SQL that’s run, use:
+
+```sh
+pgsync --debug
+```
+
 ## Other Commands
 
 Help

data/config.yml

@@ -20,6 +20,10 @@ to: postgres://localhost:5432/myapp_development
 #     - table1
 #     - table2
 
+# sync specific schemas
+# schemas:
+#   - public
+
 # protect sensitive information
 data_rules:
   email: unique_email

data/lib/pgsync.rb

@@ -10,15 +10,20 @@ require "shellwords"
 require "tempfile"
 require "uri"
 require "yaml"
+require "open3"
 
 # modules
 require "pgsync/utils"
 require "pgsync/client"
 require "pgsync/data_source"
 require "pgsync/init"
+require "pgsync/schema_sync"
+require "pgsync/sequence"
 require "pgsync/sync"
-require "pgsync/table_list"
+require "pgsync/table"
 require "pgsync/table_sync"
+require "pgsync/task"
+require "pgsync/task_resolver"
 require "pgsync/version"
 
 module PgSync

data/lib/pgsync/client.rb

@@ -7,77 +7,75 @@ module PgSync
       output.sync = true
     end
 
-    def perform(testing: true)
-      opts = parse_args
+    def perform
+      result = Slop::Parser.new(slop_options).parse(@args)
+      arguments = result.arguments
+      options = result.to_h
 
-      # TODO throw error in 0.6.0
-      warn "Specify either --db or --config, not both" if opts[:db] && opts[:config]
+      raise Error, "Specify either --db or --config, not both" if options[:db] && options[:config]
+      raise Error, "Cannot use --overwrite with --in-batches" if options[:overwrite] && options[:in_batches]
 
-      if opts.version?
+      if options[:version]
         log VERSION
-      elsif opts.help?
-        log opts
-      # TODO remove deprecated conditions (last two)
-      elsif opts.init? || opts.setup? || opts.arguments[0] == "setup"
-        Init.new.perform(opts)
+      elsif options[:help]
+        log slop_options
+      elsif options[:init]
+        Init.new(arguments, options).perform
       else
-        Sync.new.perform(opts)
+        Sync.new(arguments, options).perform
       end
-    rescue Error, PG::ConnectionBad => e
-      raise e if testing
-      abort colorize(e.message, :red)
+    rescue => e
+      # Error, PG::ConnectionBad, Slop::Error
+      raise e if options && options[:debug]
+      abort colorize(e.message.strip, :red)
     end
 
     def self.start
-      new(ARGV).perform(testing: false)
+      new(ARGV).perform
     end
 
     protected
 
-    def parse_args
-      Slop.parse(@args) do |o|
-        o.banner = %{Usage:
-    pgsync [options]
+    def slop_options
+      o = Slop::Options.new
+      o.banner = %{Usage:
+  pgsync [options]
 
 Options:}
-        o.string "-d", "--db", "database"
-        o.string "-t", "--tables", "tables to sync"
-        o.string "-g", "--groups", "groups to sync"
-        o.integer "-j", "--jobs", "number of tables to sync at a time"
-        o.string "--schemas", "schemas to sync"
-        o.string "--from", "source"
-        o.string "--to", "destination"
-        o.string "--where", "where", help: false
-        o.integer "--limit", "limit", help: false
-        o.string "--exclude", "exclude tables"
-        o.string "--config", "config file"
-        # TODO much better name for this option
-        o.boolean "--to-safe", "accept danger", default: false
-        o.boolean "--debug", "debug", default: false
-        o.boolean "--list", "list", default: false
-        o.boolean "--overwrite", "overwrite existing rows", default: false, help: false
-        o.boolean "--preserve", "preserve existing rows", default: false
-        o.boolean "--truncate", "truncate existing rows", default: false
-        o.boolean "--schema-first", "schema first", default: false
-        o.boolean "--schema-only", "schema only", default: false
-        o.boolean "--all-schemas", "all schemas", default: false
-        o.boolean "--no-rules", "do not apply data rules", default: false
-        o.boolean "--no-sequences", "do not sync sequences", default: false
-        o.boolean "--init", "init", default: false
-        o.boolean "--setup", "setup", default: false, help: false
-        o.boolean "--in-batches", "in batches", default: false, help: false
-        o.integer "--batch-size", "batch size", default: 10000, help: false
-        o.float "--sleep", "sleep", default: 0, help: false
-        o.boolean "--fail-fast", "stop on the first failed table", default: false
-        o.boolean "--defer-constraints", "defer constraints", default: false
-        o.boolean "--disable-user-triggers", "disable non-system triggers", default: false
-        o.boolean "--disable-integrity", "disable foreign key triggers", default: false
-        # o.array "--var", "pass a variable"
-        o.boolean "-v", "--version", "print the version"
-        o.boolean "-h", "--help", "prints help"
-      end
-    rescue Slop::Error => e
-      raise Error, e.message
+      o.string "-d", "--db", "database"
+      o.string "-t", "--tables", "tables to sync"
+      o.string "-g", "--groups", "groups to sync"
+      o.integer "-j", "--jobs", "number of tables to sync at a time"
+      o.string "--schemas", "schemas to sync"
+      o.string "--from", "source"
+      o.string "--to", "destination"
+      o.string "--exclude", "exclude tables"
+      o.string "--config", "config file"
+      o.boolean "--to-safe", "accept danger", default: false
+      o.boolean "--debug", "debug", default: false
+      o.boolean "--list", "list", default: false
+      o.boolean "--overwrite", "overwrite existing rows", default: false, help: false
+      o.boolean "--preserve", "preserve existing rows", default: false
+      o.boolean "--truncate", "truncate existing rows", default: false
+      o.boolean "--schema-first", "schema first", default: false
+      o.boolean "--schema-only", "schema only", default: false
+      o.boolean "--all-schemas", "all schemas", default: false
+      o.boolean "--no-rules", "do not apply data rules", default: false
+      o.boolean "--no-sequences", "do not sync sequences", default: false
+      o.boolean "--init", "init", default: false
+      o.boolean "--in-batches", "in batches", default: false, help: false
+      o.integer "--batch-size", "batch size", default: 10000, help: false
+      o.float "--sleep", "sleep", default: 0, help: false
+      o.boolean "--fail-fast", "stop on the first failed table", default: false
+      o.boolean "--defer-constraints", "defer constraints", default: false, help: false
+      o.boolean "--defer-constraints-v2", "defer constraints", default: false
+      o.boolean "--disable-user-triggers", "disable non-system triggers", default: false
+      o.boolean "--disable-integrity", "disable foreign key triggers", default: false
+      # private, for testing
+      o.boolean "--disable-integrity-v2", "disable foreign key triggers", default: false, help: false
+      o.boolean "-v", "--version", "print the version"
+      o.boolean "-h", "--help", "prints help"
+      o
     end
   end
 end