pgsync 0.5.1 → 0.6.0

data/lib/pgsync/task_resolver.rb

@@ -0,0 +1,235 @@
+module PgSync
+  class TaskResolver
+    include Utils
+
+    attr_reader :args, :opts, :source, :destination, :config, :first_schema, :notes
+
+    def initialize(args:, opts:, source:, destination:, config:, first_schema:)
+      @args = args
+      @opts = opts
+      @source = source
+      @destination = destination
+      @config = config
+      @groups = config["groups"] || {}
+      @first_schema = first_schema
+      @notes = []
+    end
+
+    def tasks
+      tasks = []
+
+      # get lists from args
+      groups, tables = process_args
+
+      # expand groups into tasks
+      groups.each do |group|
+        tasks.concat(group_to_tasks(group))
+      end
+
+      # expand tables into tasks
+      tables.each do |table|
+        tasks.concat(table_to_tasks(table))
+      end
+
+      # get default if none given
+      if !opts[:groups] && !opts[:tables] && args.size == 0
+        tasks.concat(default_tasks)
+      end
+
+      # resolve any tables that need it
+      tasks.each do |task|
+        task[:table] = fully_resolve(task[:table])
+      end
+
+      tasks
+    end
+
+    def group?(group)
+      @groups.key?(group)
+    end
+
+    private
+
+    def group_to_tasks(value)
+      group, param = value.split(":", 2)
+      raise Error, "Group not found: #{group}" unless group?(group)
+
+      @groups[group].map do |table|
+        table_sql = nil
+        if table.is_a?(Array)
+          table, table_sql = table
+        end
+
+        {
+          table: to_table(table),
+          sql: expand_sql(table_sql, param)
+        }
+      end
+    end
+
+    def table_to_tasks(value)
+      raise Error, "Cannot use parameters with tables" if value.include?(":")
+
+      tables =
+        if value.include?("*")
+          regex = Regexp.new('\A' + Regexp.escape(value).gsub('\*','[^\.]*') + '\z')
+          shared_tables.select { |t| regex.match(t.full_name) || regex.match(t.name) }
+        else
+          [to_table(value)]
+        end
+
+      tables.map do |table|
+        {
+          table: table,
+          sql: sql_arg # doesn't support params
+        }
+      end
+    end
+
+    # treats identifiers as if they were quoted (Users == "Users")
+    # this is different from Postgres (Users == "users")
+    #
+    # TODO add support for quoted identifiers like "my.schema"."my.table"
+    # so it's possible to specify identifiers with "." in them
+    def to_table(value)
+      parts = value.split(".")
+      case parts.size
+      when 1
+        # unknown schema
+        Table.new(nil, parts[0])
+      when 2
+        Table.new(*parts)
+      else
+        raise Error, "Cannot resolve table: #{value}"
+      end
+    end
+
+    def default_tasks
+      shared_tables.map do |table|
+        {
+          table: table
+        }
+      end
+    end
+
+    # tables that exists in both source and destination
+    # used when no tables specified, or a wildcard
+    # removes excluded tables and filters by schema
+    def shared_tables
+      tables = filter_tables(source.tables)
+
+      unless opts[:schema_only] || opts[:schema_first]
+        from_tables = tables
+        to_tables = filter_tables(destination.tables)
+
+        extra_tables = to_tables - from_tables
+        notes << "Extra tables: #{extra_tables.map { |t| friendly_name(t) }.join(", ")}" if extra_tables.any?
+
+        missing_tables = from_tables - to_tables
+        notes << "Missing tables: #{missing_tables.map { |t| friendly_name(t) }.join(", ")}" if missing_tables.any?
+
+        tables &= to_tables
+      end
+
+      tables
+    end
+
+    def filter_tables(tables)
+      tables = tables.dup
+
+      unless opts[:all_schemas]
+        # could support wildcard schemas as well
+        schemas = Set.new(opts[:schemas] ? to_arr(opts[:schemas]) : source.search_path)
+        tables.select! { |t| schemas.include?(t.schema) }
+      end
+
+      to_arr(opts[:exclude]).each do |value|
+        if value.include?("*")
+          regex = Regexp.new('\A' + Regexp.escape(value).gsub('\*','[^\.]*') + '\z')
+          tables.reject! { |t| regex.match(t.full_name) || regex.match(t.name) }
+        else
+          tables -= [fully_resolve(to_table(value))]
+        end
+      end
+
+      tables
+    end
+
+    def process_args
+      groups = to_arr(opts[:groups])
+      tables = to_arr(opts[:tables])
+      if args[0]
+        # could be a group, table, or mix
+        to_arr(args[0]).each do |value|
+          if group?(value.split(":", 2)[0])
+            groups << value
+          else
+            tables << value
+          end
+        end
+      end
+      [groups, tables]
+    end
+
+    def no_schema_tables
+      @no_schema_tables ||= begin
+        search_path_index = source.search_path.map.with_index.to_h
+        source.tables.group_by(&:name).map do |group, t2|
+          [group, t2.select { |t| search_path_index[t.schema] }.sort_by { |t| search_path_index[t.schema] }.first]
+        end.to_h
+      end
+    end
+
+    # for tables without a schema, find the table in the search path
+    def fully_resolve(table)
+      return table if table.schema
+      no_schema_tables[table.name] || (raise Error, "Table not found in source: #{table.name}")
+    end
+
+    # parse command line arguments and YAML
+    def to_arr(value)
+      if value.is_a?(Array)
+        value
+      else
+        # Split by commas, but don't use commas inside double quotes
+        # https://stackoverflow.com/questions/21105360/regex-find-comma-not-inside-quotes
+        value.to_s.split(/(?!\B"[^"]*),(?![^"]*"\B)/)
+      end
+    end
+
+    def sql_arg
+      args[1]
+    end
+
+    def expand_sql(sql, param)
+      # command line option takes precedence over group option
+      sql = sql_arg if sql_arg
+
+      return unless sql
+
+      # vars must match \w
+      missing_vars = sql.scan(/{\w+}/).map { |v| v[1..-2] }
+
+      vars = {}
+      if param
+        vars["id"] = cast(param)
+        vars["1"] = cast(param)
+      end
+
+      sql = sql.dup
+      vars.each do |k, v|
+        # only sub if in var list
+        sql.gsub!("{#{k}}", cast(v)) if missing_vars.delete(k)
+      end
+
+      raise Error, "Missing variables: #{missing_vars.uniq.join(", ")}" if missing_vars.any?
+
+      sql
+    end
+
+    # TODO quote vars in next major version
+    def cast(value)
+      value.to_s.gsub(/\A\"|\"\z/, '')
+    end
+  end
+end

data/lib/pgsync/utils.rb

@@ -0,0 +1,86 @@
+module PgSync
+  module Utils
+    COLOR_CODES = {
+      red: 31,
+      green: 32,
+      yellow: 33
+    }
+
+    def log(message = nil)
+      output.puts message
+    end
+
+    def colorize(message, color)
+      if output.tty?
+        "\e[#{COLOR_CODES[color]}m#{message}\e[0m"
+      else
+        message
+      end
+    end
+
+    def warning(message)
+      log colorize(message, :yellow)
+    end
+
+    def deprecated(message)
+      warning "[DEPRECATED] #{message}"
+    end
+
+    def output
+      $stderr
+    end
+
+    def db_config_file(db)
+      ".pgsync-#{db}.yml"
+    end
+
+    def confirm_tables_exist(data_source, tasks, description)
+      tasks.map(&:table).each do |table|
+        unless data_source.table_exists?(table)
+          raise Error, "Table not found in #{description}: #{table}"
+        end
+      end
+    end
+
+    def first_schema
+      @first_schema ||= source.search_path.find { |sp| sp != "pg_catalog" }
+    end
+
+    def task_name(task)
+      friendly_name(task.table)
+    end
+
+    def friendly_name(table)
+      if table.schema == first_schema
+        table.name
+      else
+        table.full_name
+      end
+    end
+
+    def quote_ident_full(ident)
+      if ident.is_a?(Table)
+        [quote_ident(ident.schema), quote_ident(ident.name)].join(".")
+      else # temp table names are strings
+        quote_ident(ident)
+      end
+    end
+
+    def quote_ident(value)
+      PG::Connection.quote_ident(value)
+    end
+
+    def escape(value)
+      if value.is_a?(String)
+        "'#{quote_string(value)}'"
+      else
+        value
+      end
+    end
+
+    # activerecord
+    def quote_string(s)
+      s.gsub(/\\/, '\&\&').gsub(/'/, "''")
+    end
+  end
+end

data/lib/pgsync/version.rb

@@ -1,3 +1,3 @@
 module PgSync
-  VERSION = "0.5.1"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pgsync
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-27 00:00:00.000000000 Z
+date: 2020-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: 4.8.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: 4.8.1
 - !ruby/object:Gem::Dependency
   name: tty-spinner
   requirement: !ruby/object:Gem::Requirement
@@ -123,8 +123,14 @@ files:
 - lib/pgsync.rb
 - lib/pgsync/client.rb
 - lib/pgsync/data_source.rb
-- lib/pgsync/table_list.rb
+- lib/pgsync/init.rb
+- lib/pgsync/schema_sync.rb
+- lib/pgsync/sync.rb
+- lib/pgsync/table.rb
 - lib/pgsync/table_sync.rb
+- lib/pgsync/task.rb
+- lib/pgsync/task_resolver.rb
+- lib/pgsync/utils.rb
 - lib/pgsync/version.rb
 homepage: https://github.com/ankane/pgsync
 licenses:

data/lib/pgsync/table_list.rb

@@ -1,107 +0,0 @@
-module PgSync
-  class TableList
-    attr_reader :args, :opts, :source, :config
-
-    def initialize(args, options, source, config)
-      @args = args
-      @opts = options
-      @source = source
-      @config = config
-    end
-
-    def tables
-      tables = nil
-
-      if opts[:groups]
-        tables ||= Hash.new { |hash, key| hash[key] = {} }
-        specified_groups = to_arr(opts[:groups])
-        specified_groups.map do |tag|
-          group, id = tag.split(":", 2)
-          if (t = (config["groups"] || {})[group])
-            add_tables(tables, t, id, args[1])
-          else
-            raise PgSync::Error, "Group not found: #{group}"
-          end
-        end
-      end
-
-      if opts[:tables]
-        tables ||= Hash.new { |hash, key| hash[key] = {} }
-        to_arr(opts[:tables]).each do |tag|
-          table, id = tag.split(":", 2)
-          raise PgSync::Error, "Cannot use parameters with tables" if id
-          add_table(tables, table, id, args[1])
-        end
-      end
-
-      if args[0]
-        # could be a group, table, or mix
-        tables ||= Hash.new { |hash, key| hash[key] = {} }
-        specified_groups = to_arr(args[0])
-        specified_groups.map do |tag|
-          group, id = tag.split(":", 2)
-          if (t = (config["groups"] || {})[group])
-            add_tables(tables, t, id, args[1])
-          else
-            raise PgSync::Error, "Cannot use parameters with tables" if id
-            add_table(tables, group, id, args[1])
-          end
-        end
-      end
-
-      tables ||= begin
-        exclude = to_arr(opts[:exclude])
-        exclude = source.fully_resolve_tables(exclude).keys if exclude.any?
-
-        tabs = source.tables
-        unless opts[:all_schemas]
-          schemas = Set.new(opts[:schemas] ? to_arr(opts[:schemas]) : source.search_path)
-          tabs.select! { |t| schemas.include?(t.split(".", 2)[0]) }
-        end
-
-        Hash[(tabs - exclude).map { |k| [k, {}] }]
-      end
-
-      source.fully_resolve_tables(tables)
-    end
-
-    private
-
-    def to_arr(value)
-      if value.is_a?(Array)
-        value
-      else
-        # Split by commas, but don't use commas inside double quotes
-        # https://stackoverflow.com/questions/21105360/regex-find-comma-not-inside-quotes
-        value.to_s.split(/(?!\B"[^"]*),(?![^"]*"\B)/)
-      end
-    end
-
-    def add_tables(tables, t, id, boom)
-      t.each do |table|
-        sql = nil
-        if table.is_a?(Array)
-          table, sql = table
-        end
-        add_table(tables, table, id, boom || sql)
-      end
-    end
-
-    def add_table(tables, table, id, boom, wildcard = false)
-      if table.include?("*") && !wildcard
-        regex = Regexp.new('\A' + Regexp.escape(table).gsub('\*','[^\.]*') + '\z')
-        t2 = source.tables.select { |t| regex.match(t) }
-        t2.each do |tab|
-          add_table(tables, tab, id, boom, true)
-        end
-      else
-        tables[table] = {}
-        tables[table][:sql] = boom.gsub("{id}", cast(id)).gsub("{1}", cast(id)) if boom
-      end
-    end
-
-    def cast(value)
-      value.to_s.gsub(/\A\"|\"\z/, '')
-    end
-  end
-end