pgsync 0.6.7 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4af2ef537f2130fb3b60ce705c17dc0ea2292a6d12f074ac14dafdc18e5419e8
-  data.tar.gz: 441442b39578dd09b4cbe22a9514f3eda59b7876a8d0c1541eda39631ac9a757
+  metadata.gz: 042c6ef2c6e17c85523588c23fb7d92dc23c625eeffd9f2988e56e346b6a46ec
+  data.tar.gz: 567458066a079635a0a887941d0c8da492acacae24f79f44d48195425a397d32
 SHA512:
-  metadata.gz: '0357909b28157593f8fcc21678efa53c767454c41a2f858b577da9e802aa29d49ffd3f8f835554098d670d82b018e27d7299a091e41022fd1b98a7eb1529232a'
-  data.tar.gz: acb5faf68e112427b406b03285d1bf2aa1c42034a186c8a37b617670037f06156e8e8f5dbb222b5fac3ce97e2b6a9d69014be9ca9c9395230c0139028c91e194
+  metadata.gz: c065a560b972175c371552ca69e7824d81d4bd1ebe72ed045ef5a5a59bcc28b738993019ca58231792f711a1f1acae4b42cb7ef68b73de83505c007a015e833d
+  data.tar.gz: c38db8ae37eca93353a5f354035fcb604e3e423b4b3fd58719b95c955fad7970990549109454f49392133860903d34d6ac0ca477f82ea1824b23435a9ae6db7e

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 0.7.1 (2022-07-06)
+
+- Fixed random letter data rule generating non-letter
+
+## 0.7.0 (2022-03-10)
+
+- Changed `--defer-constraints` to `--defer-constraints-v1`
+- Changed `--defer-constraints-v2` to `--defer-constraints`
+- Fixed unknown alias error with Ruby 3.1
+- Dropped support for Ruby < 2.5
+
+## 0.6.8 (2021-09-21)
+
+- Fixed error when schema missing in destination with `--schema-first` and `--schema-only`
+
 ## 0.6.7 (2021-04-26)
 
 - Fixed connection security for `--schema-first` and `--schema-only` - [more info](https://github.com/ankane/pgsync/issues/121)

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015-2020 Andrew Kane
+Copyright (c) 2015-2022 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -19,7 +19,7 @@ pgsync is a command line tool. To install, run:
 gem install pgsync
 ```
 
-This will give you the `pgsync` command. If installation fails, you may need to install [dependencies](#dependencies).
+This will give you the `pgsync` command. You can also install it with [Homebrew](#homebrew). If installation fails, you may need to install [dependencies](#dependencies).
 
 ## Setup
 
@@ -205,7 +205,7 @@ Foreign keys can make it difficult to sync data. Three options are:
 To defer constraints, use:
 
 ```sh
-pgsync --defer-constraints-v2
+pgsync --defer-constraints
 ```
 
 To manually specify the order of tables, use `--jobs 1` so tables are synced one-at-a-time.
@@ -347,6 +347,14 @@ Bundler.with_unbundled_env do
 end
 ```
 
+## Homebrew
+
+On Mac, you can use:
+
+```sh
+brew install ankane/brew/pgsync
+```
+
 ## Dependencies
 
 If installation fails, your system may be missing Ruby or libpq.

data/lib/pgsync/client.rb

@@ -11,6 +11,7 @@ module PgSync
       result = Slop::Parser.new(slop_options).parse(@args)
       arguments = result.arguments
       options = result.to_h
+      options[:defer_constraints_v2] ||= options[:defer_constraints]
 
       raise Error, "Specify either --db or --config, not both" if options[:db] && options[:config]
       raise Error, "Cannot use --overwrite with --in-batches" if options[:overwrite] && options[:in_batches]
@@ -59,12 +60,13 @@ module PgSync
 
       o.separator ""
       o.separator "Foreign key options:"
-      o.boolean "--defer-constraints-v2", "defer constraints", default: false
+      o.boolean "--defer-constraints", "defer constraints", default: false
       o.boolean "--disable-integrity", "disable foreign key triggers", default: false
       o.integer "-j", "--jobs", "number of tables to sync at a time"
 
-      # replaced by v2
-      o.boolean "--defer-constraints", "defer constraints", default: false, help: false
+      # legacy
+      o.boolean "--defer-constraints-v1", "defer constraints", default: false, help: false
+      o.boolean "--defer-constraints-v2", "defer constraints", default: false, help: false
       # private, for testing
       o.boolean "--disable-integrity-v2", "disable foreign key triggers", default: false, help: false
 

data/lib/pgsync/data_source.rb

@@ -68,6 +68,23 @@ module PgSync
       execute("TRUNCATE #{quote_ident_full(table)} CASCADE")
     end
 
+    def schemas
+      @schemas ||= begin
+        query = <<~SQL
+          SELECT
+            schema_name
+          FROM
+            information_schema.schemata
+          ORDER BY 1
+        SQL
+        execute(query).map { |row| row["schema_name"] }
+      end
+    end
+
+    def create_schema(schema)
+      execute("CREATE SCHEMA #{quote_ident(schema)}")
+    end
+
     def triggers(table)
       query = <<~SQL
         SELECT

data/lib/pgsync/schema_sync.rb

@@ -24,6 +24,8 @@ module PgSync
         spinner.auto_spin
       end
 
+      create_schemas if specify_tables?
+
       # if spinner, capture lines to show on error
       lines = []
       success =
@@ -68,7 +70,7 @@ module PgSync
 
     def dump_command
       cmd = ["pg_dump", "-Fc", "--verbose", "--schema-only", "--no-owner", "--no-acl"]
-      if !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude] || opts[:schemas]
+      if specify_tables?
         @tasks.each do |task|
           cmd.concat(["-t", task.quoted_table])
         end
@@ -81,5 +83,18 @@ module PgSync
       cmd << "--if-exists" if Gem::Version.new(pg_restore_version) >= Gem::Version.new("9.4.0")
       cmd.concat(["-d", @destination.url])
     end
+
+    # pg_dump -t won't create schemas (even with -n)
+    # not ideal that this happens outside restore transaction
+    def create_schemas
+      schemas = @tasks.map { |t| t.table.schema }.uniq - @destination.schemas
+      schemas.sort.each do |schema|
+        @destination.create_schema(schema)
+      end
+    end
+
+    def specify_tables?
+      !opts[:all_schemas] || opts[:tables] || opts[:groups] || args[0] || opts[:exclude] || opts[:schemas]
+    end
   end
 end

data/lib/pgsync/sync.rb

@@ -78,7 +78,16 @@ module PgSync
         file = config_file
         if file
           begin
-            YAML.load_file(file) || {}
+            # same options as YAML.load_file
+            File.open(file, "r:bom|utf-8") do |f|
+              # changed to keyword arguments in 3.1.0.pre1
+              # https://github.com/ruby/psych/commit/c79ed445b4b3f8c9adf3da13bca3c976ddfae258
+              if Psych::VERSION.to_f >= 3.1
+                YAML.safe_load(f, aliases: true, filename: file) || {}
+              else
+                YAML.safe_load(f, [], [], true, file) || {}
+              end
+            end
           rescue Psych::SyntaxError => e
             raise Error, e.message
           rescue Errno::ENOENT

data/lib/pgsync/table_sync.rb

@@ -125,7 +125,7 @@ module PgSync
       end
 
       # for non-deferrable constraints
-      if opts[:defer_constraints]
+      if opts[:defer_constraints_v1]
         constraints = non_deferrable_constraints(destination)
         constraints = tasks.flat_map { |t| constraints[t.table] || [] }
         warning "Non-deferrable constraints: #{constraints.join(", ")}" if constraints.any?
@@ -230,7 +230,7 @@ module PgSync
 
       # disable multiple jobs for defer constraints and disable integrity
       # so we can use a transaction to ensure a consistent snapshot
-      if opts[:debug] || opts[:in_batches] || opts[:defer_constraints] || opts[:defer_constraints_v2] || opts[:disable_integrity] || opts[:disable_integrity_v2]
+      if opts[:debug] || opts[:in_batches] || opts[:defer_constraints_v1] || opts[:defer_constraints_v2] || opts[:disable_integrity] || opts[:disable_integrity_v2]
         warning "--jobs ignored" if jobs
         jobs = 0
       end
@@ -268,7 +268,7 @@ module PgSync
         source.transaction do
           yield
         end
-      elsif opts[:defer_constraints] || opts[:defer_constraints_v2]
+      elsif opts[:defer_constraints_v1] || opts[:defer_constraints_v2]
         destination.transaction do
           if opts[:defer_constraints_v2]
             table_constraints = non_deferrable_constraints(destination)

data/lib/pgsync/task.rb

@@ -149,7 +149,7 @@ module PgSync
         destination.execute("INSERT INTO #{quoted_table} (#{fields}) (SELECT #{fields} FROM #{quote_ident_full(temp_table)}) ON CONFLICT (#{on_conflict}) DO #{action}")
       else
         # use delete instead of truncate for foreign keys
-        if opts[:defer_constraints] || opts[:defer_constraints_v2]
+        if opts[:defer_constraints_v1] || opts[:defer_constraints_v2]
           destination.execute("DELETE FROM #{quoted_table}")
         else
           destination.truncate(table)
@@ -256,9 +256,11 @@ module PgSync
         when "random_time"
           "NOW() - (RANDOM() * 100000000)::int * INTERVAL '1 second'"
         when "random_ip"
+          # casting double to int rounds
           "(1 + RANDOM() * 254)::int::text || '.0.0.1'"
         when "random_letter"
-          "chr(65 + (RANDOM() * 26)::int)"
+          # casting double to int rounds
+          "chr(65 + (RANDOM() * 25)::int)"
         when "random_string"
           "RIGHT(MD5(RANDOM()::text), 10)"
         when "null", nil

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.6.7"
+  VERSION = "0.7.1"
 end

data/lib/pgsync.rb

@@ -5,12 +5,12 @@ require "slop"
 require "tty-spinner"
 
 # stdlib
+require "open3"
 require "set"
 require "shellwords"
 require "tempfile"
 require "uri"
 require "yaml"
-require "open3"
 
 # modules
 require "pgsync/utils"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.6.7
+  version: 0.7.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-04-26 00:00:00.000000000 Z
+date: 2022-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description:
-email: andrew@chartkick.com
+email: andrew@ankane.org
 executables:
 - pgsync
 extensions: []
@@ -103,14 +103,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.2'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Sync Postgres data between databases