pgsync 0.6.4 → 0.6.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ecb467f4d3112edfcfebe19913e29daeddfe1e4cc10e18bd9e8850fbceced57
-  data.tar.gz: 4d8309dec9e8238074267baf583fe333c7b1d3300ad80398748e15905eb1e539
+  metadata.gz: 07f27052ca6f110b7b6ace2ad5b56399876ee1e3905dd0fc21f6c7d459ec84ca
+  data.tar.gz: 897b2b49d184ba7adf4610c4bb3bd779102982ee0f42d1592fb7b4f5bcf1a5fd
 SHA512:
-  metadata.gz: 145eb31e2565257a5adedc05c1308692dffb1e0785728ecd38230bb23c34c4ba6ecf08512201f707f457b136e0b7a22ca511b46fb7834f2045c041ff43dd6ccc
-  data.tar.gz: cde51dee36149f9f8c21e26eecd504de68d8a47c2524f084b74962d7ceae1a01b59841261fc6c112cf94b31a89f55b9a991c05d1af0db41c46ed615fc56e888f
+  metadata.gz: ae0a54b59689868a51a212436d064ac59431cd5fe86606031f7f20797fd2e66effac1d2387a2531700b7bc62a6909d2dc1091700fccebbf6631972e901d46166
+  data.tar.gz: 847b20e665fa5184975177c193dc0bf74b1c368008db663d69ba526602508c7d18cae61330487e892e8b0faf799966e74c8f5c2d7ffc1a23067bffb3b61f0b45

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 0.6.8 (2021-09-21)
+
+- Fixed error when schema missing in destination with `--schema-first` and `--schema-only`
+
+## 0.6.7 (2021-04-26)
+
+- Fixed connection security for `--schema-first` and `--schema-only` - [more info](https://github.com/ankane/pgsync/issues/121)
+
+## 0.6.6 (2020-10-29)
+
+- Added support for tables with generated columns
+
+## 0.6.5 (2020-07-10)
+
+- Improved help
+
 ## 0.6.4 (2020-06-10)
 
 - Log SQL with `--debug` option

data/README.md

@@ -9,7 +9,7 @@ Sync data from one Postgres database to another (like `pg_dump`/`pg_restore`). D
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
-[![Build Status](https://travis-ci.org/ankane/pgsync.svg?branch=master)](https://travis-ci.org/ankane/pgsync)
+[![Build Status](https://github.com/ankane/pgsync/workflows/build/badge.svg?branch=master)](https://github.com/ankane/pgsync/actions)
 
 ## Installation
 
@@ -390,6 +390,10 @@ Also check out:
 
 Inspired by [heroku-pg-transfer](https://github.com/ddollar/heroku-pg-transfer).
 
+## History
+
+View the [changelog](https://github.com/ankane/pgsync/blob/master/CHANGELOG.md)
+
 ## Contributing
 
 Everyone is encouraged to help improve this project. Here are a few ways you can help:

data/lib/pgsync/client.rb

@@ -39,42 +39,74 @@ module PgSync
     def slop_options
       o = Slop::Options.new
       o.banner = %{Usage:
-  pgsync [options]
+    pgsync [tables,groups] [sql] [options]}
 
-Options:}
-      o.string "-d", "--db", "database"
-      o.string "-t", "--tables", "tables to sync"
-      o.string "-g", "--groups", "groups to sync"
-      o.integer "-j", "--jobs", "number of tables to sync at a time"
+      # not shown
+      o.string "-t", "--tables", "tables to sync", help: false
+      o.string "-g", "--groups", "groups to sync", help: false
+
+      o.separator ""
+      o.separator "Table options:"
+      o.string "--exclude", "tables to exclude"
       o.string "--schemas", "schemas to sync"
-      o.string "--from", "source"
-      o.string "--to", "destination"
-      o.string "--exclude", "exclude tables"
-      o.string "--config", "config file"
-      o.boolean "--to-safe", "accept danger", default: false
-      o.boolean "--debug", "debug", default: false
-      o.boolean "--list", "list", default: false
-      o.boolean "--overwrite", "overwrite existing rows", default: false, help: false
+      o.boolean "--all-schemas", "sync all schemas", default: false
+
+      o.separator ""
+      o.separator "Row options:"
+      o.boolean "--overwrite", "overwrite existing rows", default: false
       o.boolean "--preserve", "preserve existing rows", default: false
       o.boolean "--truncate", "truncate existing rows", default: false
-      o.boolean "--schema-first", "schema first", default: false
-      o.boolean "--schema-only", "schema only", default: false
-      o.boolean "--all-schemas", "all schemas", default: false
-      o.boolean "--no-rules", "do not apply data rules", default: false
-      o.boolean "--no-sequences", "do not sync sequences", default: false
-      o.boolean "--init", "init", default: false
-      o.boolean "--in-batches", "in batches", default: false, help: false
-      o.integer "--batch-size", "batch size", default: 10000, help: false
-      o.float "--sleep", "sleep", default: 0, help: false
-      o.boolean "--fail-fast", "stop on the first failed table", default: false
-      o.boolean "--defer-constraints", "defer constraints", default: false, help: false
+
+      o.separator ""
+      o.separator "Foreign key options:"
       o.boolean "--defer-constraints-v2", "defer constraints", default: false
-      o.boolean "--disable-user-triggers", "disable non-system triggers", default: false
       o.boolean "--disable-integrity", "disable foreign key triggers", default: false
+      o.integer "-j", "--jobs", "number of tables to sync at a time"
+
+      # replaced by v2
+      o.boolean "--defer-constraints", "defer constraints", default: false, help: false
       # private, for testing
       o.boolean "--disable-integrity-v2", "disable foreign key triggers", default: false, help: false
-      o.boolean "-v", "--version", "print the version"
-      o.boolean "-h", "--help", "prints help"
+
+      o.separator ""
+      o.separator "Schema options:"
+      o.boolean "--schema-first", "sync schema first", default: false
+      o.boolean "--schema-only", "sync schema only", default: false
+
+      o.separator ""
+      o.separator "Config options:"
+      # technically, defaults to searching path for .pgsync.yml, but this is simpler
+      o.string "--config", "config file (defaults to .pgsync.yml)"
+      o.string "-d", "--db", "database-specific config file"
+
+      o.separator ""
+      o.separator "Connection options:"
+      o.string "--from", "source database URL"
+      o.string "--to", "destination database URL"
+      o.boolean "--to-safe", "confirms destination is safe (when not localhost)", default: false
+
+      o.separator ""
+      o.separator "Other options:"
+      o.boolean "--debug", "show SQL statements", default: false
+      o.boolean "--disable-user-triggers", "disable non-system triggers", default: false
+      o.boolean "--fail-fast", "stop on the first failed table", default: false
+      o.boolean "--no-rules", "don't apply data rules", default: false
+      o.boolean "--no-sequences", "don't sync sequences", default: false
+
+      # not shown in help
+      # o.separator ""
+      # o.separator "Append-only table options:"
+      o.boolean "--in-batches", "sync in batches", default: false, help: false
+      o.integer "--batch-size", "batch size", default: 10000, help: false
+      o.float "--sleep", "time to sleep between batches", default: 0, help: false
+
+      o.separator ""
+      o.separator "Other commands:"
+      o.boolean "--init", "create config file", default: false
+      o.boolean "--list", "list tables", default: false
+      o.boolean "-h", "--help", "print help"
+      o.boolean "-v", "--version", "print version"
+
       o
     end
   end

data/lib/pgsync/data_source.rb

@@ -68,6 +68,23 @@ module PgSync
       execute("TRUNCATE #{quote_ident_full(table)} CASCADE")
     end
 
+    def schemas
+      @schemas ||= begin
+        query = <<~SQL
+          SELECT
+            schema_name
+          FROM
+            information_schema.schemata
+          ORDER BY 1
+        SQL
+        execute(query).map { |row| row["schema_name"] }
+      end
+    end
+
+    def create_schema(schema)
+      execute("CREATE SCHEMA #{quote_ident(schema)}")
+    end
+
     def triggers(table)
       query = <<~SQL
         SELECT

data/lib/pgsync/schema_sync.rb

@@ -24,10 +24,12 @@ module PgSync
         spinner.auto_spin
       end
 
+      create_schemas if specify_tables?
+
       # if spinner, capture lines to show on error
       lines = []
       success =
-        run_command("#{dump_command} | #{restore_command}") do |line|
+        run_command do |line|
           if show_spinner
             lines << line
           else
@@ -49,12 +51,14 @@ module PgSync
 
     private
 
-    def run_command(command)
-      Open3.popen2e(command) do |stdin, stdout, wait_thr|
-        stdout.each do |line|
+    def run_command
+      err_r, err_w = IO.pipe
+      Open3.pipeline_start(dump_command, restore_command, err: err_w) do |wait_thrs|
+        err_w.close
+        err_r.each do |line|
           yield line
         end
-        wait_thr.value.success?
+        wait_thrs.all? { |t| t.value.success? }
       end
     end
 
@@ -65,19 +69,32 @@ module PgSync
     end
 
     def dump_command
-      tables =
-        if !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude] || opts[:schemas]
-          @tasks.map { |task| "-t #{Shellwords.escape(task.quoted_table)}" }
-        else
-          []
+      cmd = ["pg_dump", "-Fc", "--verbose", "--schema-only", "--no-owner", "--no-acl"]
+      if specify_tables?
+        @tasks.each do |task|
+          cmd.concat(["-t", task.quoted_table])
         end
-
-      "pg_dump -Fc --verbose --schema-only --no-owner --no-acl #{tables.join(" ")} -d #{@source.url}"
+      end
+      cmd.concat(["-d", @source.url])
     end
 
     def restore_command
-      if_exists = Gem::Version.new(pg_restore_version) >= Gem::Version.new("9.4.0")
-      "pg_restore --verbose --no-owner --no-acl --clean #{if_exists ? "--if-exists" : nil} -d #{@destination.url}"
+      cmd = ["pg_restore", "--verbose", "--no-owner", "--no-acl", "--clean"]
+      cmd << "--if-exists" if Gem::Version.new(pg_restore_version) >= Gem::Version.new("9.4.0")
+      cmd.concat(["-d", @destination.url])
+    end
+
+    # pg_dump -t won't create schemas (even with -n)
+    # not ideal that this happens outside restore transaction
+    def create_schemas
+      schemas = @tasks.map { |t| t.table.schema }.uniq - @destination.schemas
+      schemas.sort.each do |schema|
+        @destination.create_schema(schema)
+      end
+    end
+
+    def specify_tables?
+      !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude] || opts[:schemas]
     end
   end
 end

data/lib/pgsync/table_sync.rb

@@ -141,6 +141,8 @@ module PgSync
           data_type AS type
         FROM
           information_schema.columns
+        WHERE
+          is_generated = 'NEVER'
         ORDER BY 1, 2, 3
       SQL
       data_source.execute(query).group_by { |r| Table.new(r["schema"], r["table"]) }.map do |k, v|

data/lib/pgsync/task.rb

@@ -140,9 +140,13 @@ module PgSync
             "NOTHING"
           else # overwrite or sql clause
             setter = shared_fields.reject { |f| primary_key.include?(f) }.map { |f| "#{quote_ident(f)} = EXCLUDED.#{quote_ident(f)}" }
-            "UPDATE SET #{setter.join(", ")}"
+            if setter.any?
+              "UPDATE SET #{setter.join(", ")}"
+            else
+              "NOTHING"
+            end
           end
-        destination.execute("INSERT INTO #{quoted_table} (SELECT * FROM #{quote_ident_full(temp_table)}) ON CONFLICT (#{on_conflict}) DO #{action}")
+        destination.execute("INSERT INTO #{quoted_table} (#{fields}) (SELECT #{fields} FROM #{quote_ident_full(temp_table)}) ON CONFLICT (#{on_conflict}) DO #{action}")
       else
         # use delete instead of truncate for foreign keys
         if opts[:defer_constraints] || opts[:defer_constraints_v2]

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.6.4"
+  VERSION = "0.6.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.6.4
+  version: 0.6.8
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-06-11 00:00:00.000000000 Z
+date: 2021-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.8.1
+        version: 4.8.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.8.1
+        version: 4.8.2
 - !ruby/object:Gem::Dependency
   name: tty-spinner
   requirement: !ruby/object:Gem::Requirement
@@ -66,49 +66,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: 
+description:
 email: andrew@chartkick.com
 executables:
 - pgsync
@@ -137,7 +95,7 @@ homepage: https://github.com/ankane/pgsync
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -152,8 +110,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: Sync Postgres data between databases
 test_files: []