table_saw 2.2.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0fe94c6f653348bda1f6a069925e802d271e251d09be82b0e817d758296e5bf6
-  data.tar.gz: 9a6ea705f4cde2aa3c3750c28cc66ccf7a28358e113bf930151c9877e9cacb35
+  metadata.gz: 858ba4566ea024a132f1f674faaabecafd97938d930329f552a2bd86530e46c7
+  data.tar.gz: 5cada2de36a7790563ec3347a7af8b52d3e70722387b13258b11c78b33798015
 SHA512:
-  metadata.gz: d01a1f66e7c418e8ab3f01c1ab65e6cd4f3439e00edaa9f0e39cfa144e5cbe2d8d1a7e32cf359ab228bc40bef722e5d218619dbc8d462f1115bd0e4922ba430f
-  data.tar.gz: f02f30fb42c96cf3a4b87a155c6a727ce703d9788e78df6aeba01f20e524309c18741c3a87ed83c3ef42983b0f3bfa7211357c934f9edbc0b64cacbf2fd715ad
+  metadata.gz: '036185f2584a299a98439068e1cef0dcbd5229b0fb81768029e92beb3f4e724865379dea00a84a169eeedddcbf99262ef7bb1fafaf0646310f8424669a79a4bd'
+  data.tar.gz: db613c022e74e739c9109b3c5d25ba3e12682083446df5aac6c84cdf2f4216298e55edbd618e95b5c541c614a558497a09433af6afe4db87f632d5ea5c374241

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    table_saw (2.2.0)
+    table_saw (2.3.0)
       activerecord (>= 5.2)
       pg
       thor

data/exe/table-saw

@@ -24,10 +24,11 @@ class CLI < Thor
   method_option :password, default: ENV['PGPASSWORD']
   method_option :manifest, aliases: '-m', required: true
   method_option :output, aliases: '-o', default: 'output.dump'
+  method_option :format, type: :hash, default: { 'type' => 'copy' }
   def dump
     TableSaw.configure(options.to_hash)
     records = TableSaw::DependencyGraph::Build.new(TableSaw::Manifest.instance).call
-    TableSaw::CreateDumpFile.new(records, options[:output]).call
+    TableSaw::CreateDumpFile.new(records, output: options[:output], format: options[:format]).call
   end
 
   desc 'version', 'Print version of table-saw'

data/lib/table_saw.rb

@@ -6,6 +6,7 @@ require 'table_saw/dependency_graph'
 require 'table_saw/information_schema'
 require 'table_saw/manifest'
 require 'table_saw/queries'
+require 'table_saw/formats'
 
 module TableSaw
   def self.configuration
@@ -25,4 +26,8 @@ module TableSaw
   def self.information_schema
     @information_schema ||= TableSaw::InformationSchema.new
   end
+
+  def self.schema_cache
+    TableSaw::Connection.adapter.schema_cache
+  end
 end

data/lib/table_saw/configuration.rb

@@ -2,7 +2,7 @@
 
 module TableSaw
   class Configuration
-    attr_accessor :dbname, :host, :port, :user, :password, :manifest, :output
+    attr_accessor :dbname, :host, :port, :user, :password, :manifest, :output, :format
 
     def connection
       { dbname: dbname, host: host, port: port, user: user, password: password }

data/lib/table_saw/create_dump_file.rb

@@ -2,11 +2,17 @@
 
 module TableSaw
   class CreateDumpFile
-    attr_reader :records, :file
+    attr_reader :records, :file, :format
 
-    def initialize(records, file = 'psql.dump')
+    FORMATS = {
+      'copy' => TableSaw::Formats::Copy,
+      'insert' => TableSaw::Formats::Insert
+    }.freeze
+
+    def initialize(records, output:, format:)
       @records = records
-      @file = file
+      @file = output
+      @format = format
     end
 
     # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
@@ -38,27 +44,27 @@ module TableSaw
 
         COMMENT
 
-        write_to_file <<~SQL
-          COPY #{name} (#{quoted_columns(name)}) FROM STDIN;
-        SQL
+        formatter = FORMATS.fetch(format.fetch('type', 'copy'), TableSaw::Formats::Copy).new(name, options: format)
+
+        Array.wrap(formatter.header).each { |line| write_to_file(line) }
 
         TableSaw::Connection.with do |conn|
-          conn.copy_data "COPY (#{table.copy_statement}) TO STDOUT" do
+          conn.copy_data "COPY (#{table.copy_statement}) TO STDOUT", formatter.coder do
             while (row = conn.get_copy_data)
-              write_to_file row
+              write_to_file formatter.dump_row(row)
             end
           end
         end
 
-        write_to_file '\.'
-        write_to_file "\n"
+        Array.wrap(formatter.footer).each { |line| write_to_file(line) }
       end
 
+      write_to_file 'COMMIT;'
+      write_to_file "\n"
+
       refresh_materialized_views
       restart_sequences
 
-      write_to_file 'COMMIT;'
-
       alter_constraints_deferrability keyword: 'NOT DEFERRABLE'
     end
     # rubocop:enable Metrics/MethodLength,Metrics/AbcSize
@@ -114,9 +120,5 @@ module TableSaw
     def write_to_file(data)
       File.open(file, 'ab') { |f| f.puts(data) }
     end
-
-    def quoted_columns(table)
-      TableSaw::Queries::TableColumns.new(table).call.map { |c| "\"#{c}\"" }.join(', ')
-    end
   end
 end

data/lib/table_saw/dependency_graph/add_directive.rb

@@ -24,7 +24,7 @@ module TableSaw
       end
 
       def primary_key
-        TableSaw.information_schema.primary_keys[table_name]
+        TableSaw.schema_cache.primary_keys(table_name)
       end
     end
   end

data/lib/table_saw/dependency_graph/build.rb

@@ -41,7 +41,7 @@ module TableSaw
       def select_ids(table)
         return [] unless table.partial?
 
-        TableSaw::Connection.exec(table.query).map { |row| row[TableSaw.information_schema.primary_keys[table.name]] }
+        TableSaw::Connection.exec(table.query).map { |row| row[TableSaw.schema_cache.primary_keys(table.name)] }
       end
     end
   end

data/lib/table_saw/dependency_graph/dump_table.rb

@@ -39,7 +39,7 @@ module TableSaw
       end
 
       def primary_key
-        TableSaw.information_schema.primary_keys[name]
+        TableSaw.schema_cache.primary_keys(name)
       end
     end
   end

data/lib/table_saw/dependency_graph/has_many_directives.rb

@@ -14,7 +14,7 @@ module TableSaw
         valid_associations.map do |table, column|
           TableSaw::DependencyGraph::AddDirective.new(
             table,
-            ids: query_result(table, column).map { |r| r[TableSaw.information_schema.primary_keys[table]] },
+            ids: query_result(table, column).map { |r| r[TableSaw.schema_cache.primary_keys(table)] },
             partial: directive.partial?
           )
         end
@@ -29,7 +29,7 @@ module TableSaw
       # rubocop:disable Metrics/AbcSize
       def valid_associations
         associations.select do |table, _column|
-          next false if directive.partial? && !TableSaw.information_schema.primary_keys.key?(table)
+          next false if directive.partial? && TableSaw.schema_cache.primary_keys(table).nil?
           next true if directive.has_many.include?(table)
 
           manifest.has_many.fetch(directive.table_name, []).include?(table)
@@ -43,7 +43,7 @@ module TableSaw
         TableSaw::Connection.exec(
           format(
             'select %{primary_key} from %{table} where %{clause}',
-            primary_key: TableSaw.information_schema.primary_keys[table], table: table,
+            primary_key: TableSaw.schema_cache.primary_keys(table), table: table,
             clause: TableSaw::Queries::SerializeSqlInClause.new(table, column, directive.ids).call
           )
         )

data/lib/table_saw/formats.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require 'table_saw/formats/base'
+require 'table_saw/formats/copy'
+require 'table_saw/formats/insert'

data/lib/table_saw/formats/base.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module TableSaw
+  module Formats
+    class Base
+      attr_reader :table_name, :options
+
+      def initialize(table_name, options: {})
+        @table_name = table_name
+        @options = options
+      end
+
+      def header
+        raise NotImplementedError
+      end
+
+      def footer
+        raise NotImplementedError
+      end
+
+      def dump_row(_row)
+        raise NotImplementedError
+      end
+
+      def coder
+      end
+    end
+  end
+end

data/lib/table_saw/formats/copy.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module TableSaw
+  module Formats
+    class Copy < TableSaw::Formats::Base
+      def header
+        "COPY #{table_name} (#{quoted_columns}) FROM STDIN;"
+      end
+
+      def footer
+        ['\.', "\n"]
+      end
+
+      def dump_row(row)
+        row
+      end
+
+      private
+
+      def quoted_columns
+        TableSaw.schema_cache.columns_hash(table_name)
+          .each_key
+          .map { |name| connection.quote_column_name(name) }
+          .join(', ')
+      end
+
+      def connection
+        TableSaw.schema_cache.connection
+      end
+    end
+  end
+end

data/lib/table_saw/formats/insert.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module TableSaw
+  module Formats
+    class Insert < TableSaw::Formats::Base
+      def header
+        prepared_statement.sql
+      end
+
+      def footer
+        "\n"
+      end
+
+      def dump_row(row)
+        TableSaw::Queries::ExecuteInsertStatement.new(prepared_statement, row).call
+      end
+
+      def coder
+        PG::TextDecoder::CopyRow.new
+      end
+
+      private
+
+      def prepared_statement
+        @prepared_statement ||= TableSaw::Queries::PreparedInsertStatement.new(table_name, options: options).call
+      end
+    end
+  end
+end

data/lib/table_saw/information_schema.rb

@@ -8,10 +8,6 @@ module TableSaw
       end
     end
 
-    def primary_keys
-      @primary_keys ||= TableSaw::Queries::PrimaryKeys.new.call
-    end
-
     private
 
     def foreign_key_relationships

data/lib/table_saw/queries.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
+require 'table_saw/queries/execute_insert_statement'
 require 'table_saw/queries/foreign_key_relationships'
 require 'table_saw/queries/materialized_views'
-require 'table_saw/queries/primary_keys'
+require 'table_saw/queries/prepared_insert_statement'
 require 'table_saw/queries/serial_sequences'
 require 'table_saw/queries/serialize_sql_in_clause'
-require 'table_saw/queries/table_columns'

data/lib/table_saw/queries/execute_insert_statement.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module TableSaw
+  module Queries
+    class ExecuteInsertStatement
+      attr_reader :statement, :row
+
+      def initialize(statement, row)
+        @statement = statement
+        @row = row
+      end
+
+      def call
+        "EXECUTE #{statement.name}(#{values});"
+      end
+
+      private
+
+      def values
+        TableSaw.schema_cache.columns(statement.table_name).zip(row)
+          .map { |column, value| connection.quote(connection.type_cast_from_column(column, value)) }
+          .join(', ')
+      end
+
+      def schema_cache
+        TableSaw.schema_cache
+      end
+
+      def connection
+        schema_cache.connection
+      end
+    end
+  end
+end

data/lib/table_saw/queries/prepared_insert_statement.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module TableSaw
+  module Queries
+    class PreparedInsertStatement
+      attr_reader :table_name, :options
+
+      Statement = Struct.new(:name, :table_name, :sql)
+
+      def initialize(table_name, options: {})
+        @table_name = table_name
+        @options = options
+      end
+
+      def call
+        Statement.new(name, table_name, sql)
+      end
+
+      private
+
+      def name
+        "#{table_name}_insert_plan"
+      end
+
+      def sql
+        [prepare_statement, conflict_statement].compact.join(' ') + ';'
+      end
+
+      def column_types
+        TableSaw.schema_cache.columns(table_name).map(&:sql_type).join(', ')
+      end
+
+      def values_clause
+        1.upto(TableSaw.schema_cache.columns(table_name).size).map { |i| "$#{i}" }.join(', ')
+      end
+
+      def prepare_statement
+        <<~SQL.squish
+          PREPARE #{name} (#{column_types}) AS INSERT INTO #{table_name} VALUES (#{values_clause})
+        SQL
+      end
+
+      def conflict_statement
+        return unless options['ignore_conflict'] == 'true'
+
+        'ON CONFLICT DO NOTHING'
+      end
+    end
+  end
+end

data/lib/table_saw/queries/serialize_sql_in_clause.rb

@@ -18,7 +18,7 @@ module TableSaw
       private
 
       def db_column
-        @db_column ||= connection.columns(table_name).find { |c| c.name == column }
+        TableSaw.schema_cache.columns_hash(table_name)[column]
       end
 
       def serialized_values

data/lib/table_saw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TableSaw
-  VERSION = '2.2.0'
+  VERSION = '2.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: table_saw
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.0
 platform: ruby
 authors:
 - Hamed Asghari
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-07 00:00:00.000000000 Z
+date: 2019-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -211,15 +211,19 @@ files:
 - lib/table_saw/dependency_graph/build.rb
 - lib/table_saw/dependency_graph/dump_table.rb
 - lib/table_saw/dependency_graph/has_many_directives.rb
+- lib/table_saw/formats.rb
+- lib/table_saw/formats/base.rb
+- lib/table_saw/formats/copy.rb
+- lib/table_saw/formats/insert.rb
 - lib/table_saw/information_schema.rb
 - lib/table_saw/manifest.rb
 - lib/table_saw/queries.rb
+- lib/table_saw/queries/execute_insert_statement.rb
 - lib/table_saw/queries/foreign_key_relationships.rb
 - lib/table_saw/queries/materialized_views.rb
-- lib/table_saw/queries/primary_keys.rb
+- lib/table_saw/queries/prepared_insert_statement.rb
 - lib/table_saw/queries/serial_sequences.rb
 - lib/table_saw/queries/serialize_sql_in_clause.rb
-- lib/table_saw/queries/table_columns.rb
 - lib/table_saw/version.rb
 - table_saw.gemspec
 homepage: https://github.com/hasghari/table_saw

data/lib/table_saw/queries/primary_keys.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module TableSaw
-  module Queries
-    class PrimaryKeys
-      QUERY = <<~SQL
-        select tc.table_name, kcu.column_name
-        from information_schema.table_constraints tc
-                 join information_schema.key_column_usage kcu using (constraint_schema, constraint_name)
-        where tc.constraint_type = 'PRIMARY KEY';
-      SQL
-
-      def call
-        TableSaw::Connection.exec(QUERY).each_with_object({}) do |row, memo|
-          memo[row['table_name']] = row['column_name']
-        end
-      end
-    end
-  end
-end

data/lib/table_saw/queries/table_columns.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-module TableSaw
-  module Queries
-    class TableColumns
-      QUERY = <<~SQL
-        select attname as colname
-        from pg_catalog.pg_attribute
-        where
-                attrelid = $1::regclass
-          and attnum > 0
-          and attisdropped = false
-        order by attnum
-      SQL
-
-      attr_reader :table
-
-      def initialize(table)
-        @table = table
-      end
-
-      def call
-        TableSaw::Connection.with { |conn| conn.exec_params(QUERY, [table]) }.map { |r| r['colname'] }
-      end
-    end
-  end
-end