pgsync 0.3.4 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c8bd6a2e877c863a693c0447ac9dc8a621a4c2c5
-  data.tar.gz: a5322f143b1daaa5cf60b55dbd664b22d31f7b7c
+  metadata.gz: aba5139d6101c73512853da24762c38308f83b00
+  data.tar.gz: b9dda9275f5faf6a70cafebfba723d1eb96f6b09
 SHA512:
-  metadata.gz: 9370577fcad8039401eb4d19dcad7ae5d24af1c0a1dfb0904e940ff8ab802a33218390dbceda4e214ea0f2591c507b14099a09819f1b9df75463ea64b16dc2b7
-  data.tar.gz: cd474765176aea15d99175ae302d47eae50bb3e507ab7cfe7c8d605cf22699fce49c4fbdf8793116e7a7b629b15292ee732c804c67090d06ded178bf73d78933
+  metadata.gz: 08ae766aff0f46b73579f00d3ca3a72c7a27a2c25ce6d89f8ad271c1d4558a195d71eb367a3b9096b3d6896510e321bd017a81f3da7821818d6849b7e871e463
+  data.tar.gz: 0ad9116bd57ffbf6cc594146db0aa9d4ea414351331e0fc7cffb66e9e8eb934db266c9e266cafc2bdb0da1c5fb87ff3dcd737ee72e7622e0e321a552ada2c1a8

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.3.5
+
+- Support schemas other than public
+
 # 0.3.4
 
 - Added `--in-batches` mode for production transfers with `--batch-size` and `--sleep`

data/README.md

@@ -70,11 +70,12 @@ exclude:
   - table2
 ```
 
-For Rails, you probably want to exclude schema migrations.
+For Rails, you probably want to exclude schema migrations and ActiveRecord metadata.
 
 ```yml
 exclude:
   - schema_migrations
+  - ar_internal_metadata
 ```
 
 ### Groups
@@ -181,6 +182,30 @@ To keep you from accidentally overwriting production, the destination is limited
 
 To use another host, add `to_safe: true` to your `.pgsync.yml`.
 
+## Large Tables
+
+For extremely large tables, sync in batches.
+
+```sh
+pgsync large_table --in-batches
+```
+
+The script will resume where it left off when run again, making it great for backfills.
+
+## Reference
+
+Help
+
+```sh
+pgsync --help
+```
+
+Version
+
+```sh
+pgsync --version
+```
+
 ## Setup Scripts
 
 Use groups when possible to take advantage of parallelism.
@@ -212,10 +237,6 @@ gem specific_install ankane/pgsync
 
 Inspired by [heroku-pg-transfer](https://github.com/ddollar/heroku-pg-transfer).
 
-## TODO
-
-- Support for schemas other than `public`
-
 ## Contributing
 
 Everyone is encouraged to help improve this project. Here are a few ways you can help:

data/config.yml

@@ -11,6 +11,7 @@ to: postgres://localhost:5432/myapp_development
 # exclude tables
 # exclude:
 #   - schema_migrations
+#   - ar_internal_metadata
 
 # define groups
 # groups:

data/lib/pgsync.rb

@@ -8,6 +8,7 @@ require "parallel"
 require "multiprocessing"
 require "fileutils"
 require "tempfile"
+require "cgi"
 
 module URI
   class POSTGRESQL < Generic
@@ -73,13 +74,17 @@ module PgSync
       if opts[:setup]
         setup(db_config_file(args[0]) || config_file || ".pgsync.yml")
       else
+        if args.size > 2
+          abort "Usage:\n    pgsync [options]"
+        end
+
         source = parse_source(opts[:from])
         abort "No source" unless source
-        source_uri = parse_uri(source)
+        source_uri, from_schema = parse_uri(source)
 
         destination = parse_source(opts[:to])
         abort "No destination" unless destination
-        destination_uri = parse_uri(destination)
+        destination_uri, to_schema = parse_uri(destination)
         abort "Danger! Add `to_safe: true` to `.pgsync.yml` if the destination is not localhost or 127.0.0.1" unless %(localhost 127.0.0.1).include?(destination_uri.host) || opts[:to_safe]
 
         print_uri("From", source_uri)
@@ -88,7 +93,7 @@ module PgSync
         from_uri = source_uri
         to_uri = destination_uri
 
-        tables = table_list(args, opts, from_uri)
+        tables = table_list(args, opts, from_uri, from_schema)
 
         if opts[:schema_only]
           log "* Dumping schema"
@@ -103,7 +108,7 @@ module PgSync
         else
           with_connection(to_uri, timeout: 3) do |conn|
             tables.keys.each do |table|
-              unless table_exists?(conn, table, "public")
+              unless table_exists?(conn, table, to_schema)
                 abort "Table does not exist in destination: #{table}"
               end
             end
@@ -117,7 +122,7 @@ module PgSync
             end
           else
             in_parallel(tables) do |table, table_opts|
-              sync_table(table, opts.merge(table_opts), from_uri, to_uri)
+              sync_table(table, opts.merge(table_opts), from_uri, to_uri, from_schema, to_schema)
             end
 
             log_completed(start_time)
@@ -129,15 +134,15 @@ module PgSync
 
     protected
 
-    def sync_table(table, opts, from_uri, to_uri)
+    def sync_table(table, opts, from_uri, to_uri, from_schema, to_schema)
       time =
         benchmark do
           with_connection(from_uri) do |from_connection|
             with_connection(to_uri) do |to_connection|
               bad_fields = opts[:no_rules] ? [] : config["data_rules"]
 
-              from_fields = columns(from_connection, table, "public")
-              to_fields = columns(to_connection, table, "public")
+              from_fields = columns(from_connection, table, from_schema)
+              to_fields = columns(to_connection, table, to_schema)
               shared_fields = to_fields & from_fields
               extra_fields = to_fields - from_fields
               missing_fields = from_fields - to_fields
@@ -177,7 +182,7 @@ module PgSync
 
                 copy_to_command = "COPY (SELECT #{copy_fields} FROM #{table}#{sql_clause}) TO STDOUT"
                 if opts[:in_batches]
-                  primary_key = self.primary_key(from_connection, table, "public")
+                  primary_key = self.primary_key(from_connection, table, from_schema)
                   abort "No primary key" unless primary_key
 
                   from_max_id = max_id(from_connection, table, primary_key, sql_clause)
@@ -218,7 +223,7 @@ module PgSync
                     end
                   end
                 elsif !opts[:truncate] && (opts[:overwrite] || opts[:preserve] || !sql_clause.empty?)
-                  primary_key = self.primary_key(from_connection, table, "public")
+                  primary_key = self.primary_key(to_connection, table, to_schema)
                   abort "No primary key" unless primary_key
 
                   temp_table = "pgsync_#{rand(1_000_000_000)}"
@@ -497,7 +502,8 @@ Options:}
       uri.host ||= "localhost"
       uri.port ||= 5432
       uri.path = "/#{uri.path}" if uri.path && uri.path[0] != "/"
-      uri
+      schema = ((uri.query && CGI::parse(uri.query)["schema"]) || ["public"])[0]
+      [uri, schema]
     end
 
     def print_uri(prefix, uri)
@@ -562,22 +568,22 @@ Options:}
       end
     end
 
-    def add_tables(tables, t, id, boom, from_uri)
+    def add_tables(tables, t, id, boom, from_uri, from_schema)
       t.each do |table|
         sql = nil
         if table.is_a?(Array)
           table, sql = table
         end
-        add_table(tables, table, id, boom || sql, from_uri)
+        add_table(tables, table, id, boom || sql, from_uri, from_schema)
       end
     end
 
-    def add_table(tables, table, id, boom, from_uri, wildcard = false)
+    def add_table(tables, table, id, boom, from_uri, from_schema, wildcard = false)
       if table.include?("*") && !wildcard
         regex = Regexp.new('\A' + Regexp.escape(table).gsub('\*','[^\.]*') + '\z')
-        t2 = with_connection(from_uri) { |conn| self.tables(conn, "public") }.select { |t| regex.match(t) }
+        t2 = with_connection(from_uri) { |conn| self.tables(conn, from_schema) }.select { |t| regex.match(t) }
         t2.each do |table|
-          add_table(tables, table, id, boom, from_uri, true)
+          add_table(tables, table, id, boom, from_uri, from_schema, true)
         end
       else
         tables[table] = {}
@@ -585,7 +591,7 @@ Options:}
       end
     end
 
-    def table_list(args, opts, from_uri)
+    def table_list(args, opts, from_uri, from_schema)
       tables = nil
 
       if opts[:groups]
@@ -594,7 +600,7 @@ Options:}
         specified_groups.map do |tag|
           group, id = tag.split(":", 2)
           if (t = (config["groups"] || {})[group])
-            add_tables(tables, t, id, args[1], from_uri)
+            add_tables(tables, t, id, args[1], from_uri, from_schema)
           else
             abort "Group not found: #{group}"
           end
@@ -605,7 +611,7 @@ Options:}
         tables ||= Hash.new { |hash, key| hash[key] = {} }
         to_arr(opts[:tables]).each do |tag|
           table, id = tag.split(":", 2)
-          add_table(tables, table, id, args[1], from_uri)
+          add_table(tables, table, id, args[1], from_uri, from_schema)
         end
       end
 
@@ -616,18 +622,18 @@ Options:}
         specified_groups.map do |tag|
           group, id = tag.split(":", 2)
           if (t = (config["groups"] || {})[group])
-            add_tables(tables, t, id, args[1], from_uri)
+            add_tables(tables, t, id, args[1], from_uri, from_schema)
           else
-            add_table(tables, group, id, args[1], from_uri)
+            add_table(tables, group, id, args[1], from_uri, from_schema)
           end
         end
       end
 
       with_connection(from_uri, timeout: 3) do |conn|
-        tables ||= Hash[(self.tables(conn, "public") - to_arr(opts[:exclude])).map { |k| [k, {}] }]
+        tables ||= Hash[(self.tables(conn, from_schema) - to_arr(opts[:exclude])).map { |k| [k, {}] }]
 
         tables.keys.each do |table|
-          unless table_exists?(conn, table, "public")
+          unless table_exists?(conn, table, from_schema)
             abort "Table does not exist in source: #{table}"
           end
         end

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.3.4"
+  VERSION = "0.3.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-04-29 00:00:00.000000000 Z
+date: 2016-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop