pgsync 0.6.6 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1097ac9939cf312566746d7d3ff06604ee5997c9e2b85ae38f7d46d77dbdefc9
-  data.tar.gz: 06b70ef312aa796c9bf083b89a7cd6ff3c8a99920d8b1eabdd290cd67d956559
+  metadata.gz: ba1091cc780447fdf3369431b15d87cd3b6e47bb047f4a34e2a36a70bb873669
+  data.tar.gz: bf343a9fd3e785cb077953654656d51191827216b741b844736043bfaa2b7d8b
 SHA512:
-  metadata.gz: 815aeffa2bd01469b0fbc229a25b8f2ff8b69f28c3bfac21c61f571875bf74914cdaabfe9150d3db271ac2f10dcef4c0c4f542b72f306618b4b2a0985d59d388
-  data.tar.gz: 128cd39679c96354d93213e611d679275d2c3de75a29b847aa8cb911c03d13f3b9231f7318a5ea3a4b8bc98684eb6a3ed07975159a9d89ee4aa2ac53746365f7
+  metadata.gz: f30e0cfdfa4a5f2f3d48db16d66dea9ff101025b8a95fd383f0eec8b60bce111d0f41cef64467c50df3d4820a55230e64f0f04e68ef09f40ea57d658b3584da6
+  data.tar.gz: 7feb71d51de95fba8d32d0080a43432a11f96080782aad0515dade6ce6f2b81def36f7b060a30aeeadcea14cc16dbf34fc1f94b02ebdb53c2453d1ff0ba6dd30

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 0.7.0 (2022-03-10)
+
+- Changed `--defer-constraints` to `--defer-constraints-v1`
+- Changed `--defer-constraints-v2` to `--defer-constraints`
+- Fixed unknown alias error with Ruby 3.1
+- Dropped support for Ruby < 2.5
+
+## 0.6.8 (2021-09-21)
+
+- Fixed error when schema missing in destination with `--schema-first` and `--schema-only`
+
+## 0.6.7 (2021-04-26)
+
+- Fixed connection security for `--schema-first` and `--schema-only` - [more info](https://github.com/ankane/pgsync/issues/121)
+
 ## 0.6.6 (2020-10-29)
 
 - Added support for tables with generated columns

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2020 Andrew Kane
+Copyright (c) 2015-2022 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -9,7 +9,7 @@ Sync data from one Postgres database to another (like `pg_dump`/`pg_restore`). D
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
-[![Build Status](https://travis-ci.org/ankane/pgsync.svg?branch=master)](https://travis-ci.org/ankane/pgsync)
+[![Build Status](https://github.com/ankane/pgsync/workflows/build/badge.svg?branch=master)](https://github.com/ankane/pgsync/actions)
 
 ## Installation
 
@@ -19,7 +19,7 @@ pgsync is a command line tool. To install, run:
 gem install pgsync
 ```
 
-This will give you the `pgsync` command. If installation fails, you may need to install [dependencies](#dependencies).
+This will give you the `pgsync` command. You can also install it with [Homebrew](#homebrew). If installation fails, you may need to install [dependencies](#dependencies).
 
 ## Setup
 
@@ -205,7 +205,7 @@ Foreign keys can make it difficult to sync data. Three options are:
 To defer constraints, use:
 
 ```sh
-pgsync --defer-constraints-v2
+pgsync --defer-constraints
 ```
 
 To manually specify the order of tables, use `--jobs 1` so tables are synced one-at-a-time.
@@ -347,6 +347,14 @@ Bundler.with_unbundled_env do
 end
 ```
 
+## Homebrew
+
+On Mac, you can use:
+
+```sh
+brew install ankane/brew/pgsync
+```
+
 ## Dependencies
 
 If installation fails, your system may be missing Ruby or libpq.

data/lib/pgsync/client.rb

@@ -11,6 +11,7 @@ module PgSync
       result = Slop::Parser.new(slop_options).parse(@args)
       arguments = result.arguments
       options = result.to_h
+      options[:defer_constraints_v2] ||= options[:defer_constraints]
 
       raise Error, "Specify either --db or --config, not both" if options[:db] && options[:config]
       raise Error, "Cannot use --overwrite with --in-batches" if options[:overwrite] && options[:in_batches]
@@ -59,12 +60,13 @@ module PgSync
 
       o.separator ""
       o.separator "Foreign key options:"
-      o.boolean "--defer-constraints-v2", "defer constraints", default: false
+      o.boolean "--defer-constraints", "defer constraints", default: false
       o.boolean "--disable-integrity", "disable foreign key triggers", default: false
       o.integer "-j", "--jobs", "number of tables to sync at a time"
 
-      # replaced by v2
-      o.boolean "--defer-constraints", "defer constraints", default: false, help: false
+      # legacy
+      o.boolean "--defer-constraints-v1", "defer constraints", default: false, help: false
+      o.boolean "--defer-constraints-v2", "defer constraints", default: false, help: false
       # private, for testing
       o.boolean "--disable-integrity-v2", "disable foreign key triggers", default: false, help: false
 

data/lib/pgsync/data_source.rb

@@ -68,6 +68,23 @@ module PgSync
       execute("TRUNCATE #{quote_ident_full(table)} CASCADE")
     end
 
+    def schemas
+      @schemas ||= begin
+        query = <<~SQL
+          SELECT
+            schema_name
+          FROM
+            information_schema.schemata
+          ORDER BY 1
+        SQL
+        execute(query).map { |row| row["schema_name"] }
+      end
+    end
+
+    def create_schema(schema)
+      execute("CREATE SCHEMA #{quote_ident(schema)}")
+    end
+
     def triggers(table)
       query = <<~SQL
         SELECT

data/lib/pgsync/schema_sync.rb

@@ -24,10 +24,12 @@ module PgSync
         spinner.auto_spin
       end
 
+      create_schemas if specify_tables?
+
       # if spinner, capture lines to show on error
       lines = []
       success =
-        run_command("#{dump_command} | #{restore_command}") do |line|
+        run_command do |line|
           if show_spinner
             lines << line
           else
@@ -49,12 +51,14 @@ module PgSync
 
     private
 
-    def run_command(command)
-      Open3.popen2e(command) do |stdin, stdout, wait_thr|
-        stdout.each do |line|
+    def run_command
+      err_r, err_w = IO.pipe
+      Open3.pipeline_start(dump_command, restore_command, err: err_w) do |wait_thrs|
+        err_w.close
+        err_r.each do |line|
           yield line
         end
-        wait_thr.value.success?
+        wait_thrs.all? { |t| t.value.success? }
       end
     end
 
@@ -65,19 +69,32 @@ module PgSync
     end
 
     def dump_command
-      tables =
-        if !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude] || opts[:schemas]
-          @tasks.map { |task| "-t #{Shellwords.escape(task.quoted_table)}" }
-        else
-          []
+      cmd = ["pg_dump", "-Fc", "--verbose", "--schema-only", "--no-owner", "--no-acl"]
+      if specify_tables?
+        @tasks.each do |task|
+          cmd.concat(["-t", task.quoted_table])
         end
-
-      "pg_dump -Fc --verbose --schema-only --no-owner --no-acl #{tables.join(" ")} -d #{@source.url}"
+      end
+      cmd.concat(["-d", @source.url])
     end
 
     def restore_command
-      if_exists = Gem::Version.new(pg_restore_version) >= Gem::Version.new("9.4.0")
-      "pg_restore --verbose --no-owner --no-acl --clean #{if_exists ? "--if-exists" : nil} -d #{@destination.url}"
+      cmd = ["pg_restore", "--verbose", "--no-owner", "--no-acl", "--clean"]
+      cmd << "--if-exists" if Gem::Version.new(pg_restore_version) >= Gem::Version.new("9.4.0")
+      cmd.concat(["-d", @destination.url])
+    end
+
+    # pg_dump -t won't create schemas (even with -n)
+    # not ideal that this happens outside restore transaction
+    def create_schemas
+      schemas = @tasks.map { |t| t.table.schema }.uniq - @destination.schemas
+      schemas.sort.each do |schema|
+        @destination.create_schema(schema)
+      end
+    end
+
+    def specify_tables?
+      !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude] || opts[:schemas]
     end
   end
 end

data/lib/pgsync/sync.rb

@@ -78,7 +78,16 @@ module PgSync
         file = config_file
         if file
           begin
-            YAML.load_file(file) || {}
+            # same options as YAML.load_file
+            File.open(file, "r:bom|utf-8") do |f|
+              # changed to keyword arguments in 3.1.0.pre1
+              # https://github.com/ruby/psych/commit/c79ed445b4b3f8c9adf3da13bca3c976ddfae258
+              if Psych::VERSION.to_f >= 3.1
+                YAML.safe_load(f, aliases: true, filename: file) || {}
+              else
+                YAML.safe_load(f, [], [], true, file) || {}
+              end
+            end
           rescue Psych::SyntaxError => e
             raise Error, e.message
           rescue Errno::ENOENT

data/lib/pgsync/table_sync.rb

@@ -125,7 +125,7 @@ module PgSync
       end
 
       # for non-deferrable constraints
-      if opts[:defer_constraints]
+      if opts[:defer_constraints_v1]
         constraints = non_deferrable_constraints(destination)
         constraints = tasks.flat_map { |t| constraints[t.table] || [] }
         warning "Non-deferrable constraints: #{constraints.join(", ")}" if constraints.any?
@@ -230,7 +230,7 @@ module PgSync
 
       # disable multiple jobs for defer constraints and disable integrity
       # so we can use a transaction to ensure a consistent snapshot
-      if opts[:debug] || opts[:in_batches] || opts[:defer_constraints] || opts[:defer_constraints_v2] || opts[:disable_integrity] || opts[:disable_integrity_v2]
+      if opts[:debug] || opts[:in_batches] || opts[:defer_constraints_v1] || opts[:defer_constraints_v2] || opts[:disable_integrity] || opts[:disable_integrity_v2]
         warning "--jobs ignored" if jobs
         jobs = 0
       end
@@ -268,7 +268,7 @@ module PgSync
         source.transaction do
           yield
         end
-      elsif opts[:defer_constraints] || opts[:defer_constraints_v2]
+      elsif opts[:defer_constraints_v1] || opts[:defer_constraints_v2]
         destination.transaction do
           if opts[:defer_constraints_v2]
             table_constraints = non_deferrable_constraints(destination)

data/lib/pgsync/task.rb

@@ -149,7 +149,7 @@ module PgSync
         destination.execute("INSERT INTO #{quoted_table} (#{fields}) (SELECT #{fields} FROM #{quote_ident_full(temp_table)}) ON CONFLICT (#{on_conflict}) DO #{action}")
       else
         # use delete instead of truncate for foreign keys
-        if opts[:defer_constraints] || opts[:defer_constraints_v2]
+        if opts[:defer_constraints_v1] || opts[:defer_constraints_v2]
           destination.execute("DELETE FROM #{quoted_table}")
         else
           destination.truncate(table)

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.6.6"
+  VERSION = "0.7.0"
 end

data/lib/pgsync.rb

@@ -5,12 +5,12 @@ require "slop"
 require "tty-spinner"
 
 # stdlib
+require "open3"
 require "set"
 require "shellwords"
 require "tempfile"
 require "uri"
 require "yaml"
-require "open3"
 
 # modules
 require "pgsync/utils"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.6.6
+  version: 0.7.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-10-30 00:00:00.000000000 Z
+date: 2022-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -66,50 +66,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description:
-email: andrew@chartkick.com
+email: andrew@ankane.org
 executables:
 - pgsync
 extensions: []
@@ -145,14 +103,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.2'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Sync Postgres data between databases