table_saw 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce7f117d5db32cc1d9617f4f09126a60636519a1b8445ce2c65575120deac9db
-  data.tar.gz: 212a3f010918b397329ed3510c831898370865a60ee8949948b60f70f05f6a6d
+  metadata.gz: 1339fbe2ad8b61d13d84595da082e4bd8886f9ecb56e1c8ce55ba9b5d9d33304
+  data.tar.gz: 79e448b1ee7d8f77a019b491ed5cbe11e4b5ca353721643211543813a779b856
 SHA512:
-  metadata.gz: e11664a36c6c5a649d4f79ed4018b24b9681d68053a0187b5927582d24dbf1f62fa706511bd493fb19ff583316083f1fbac17173efdfec6feb13f8432188cd54
-  data.tar.gz: 6375f829abd5d21a7aee6f36891d1e0a1f2778dbf59eab9c42cde7817d22a34f0831cf733a9053d9a429149338cae782ee64029288197ec0414afc084450339a
+  metadata.gz: 49bf13e6be2584987b29755ca760fced988ff01f7f8ad362a2cb35122f83da8de7b1de37c0f02d32bee9dee733b0d9c517d515dc5134cb00a4e08c26f57af9c1
+  data.tar.gz: 290706ba80434334c9d4af235e84972018bad3d2ef009eb85ee288ce6e7739fb669e1ba428e8060e6c1509910a0cdb0bffe05f26fdeee562772b69d41c1fe687

data/.rubocop.yml

@@ -3,6 +3,9 @@ require: rubocop-rspec
 AllCops:
   TargetRubyVersion: 2.6
 
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented_relative_to_receiver
+
 Metrics/LineLength:
   Max: 120
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    table_saw (0.3.0)
+    table_saw (0.4.0)
       connection_pool
       pg
       thor
@@ -36,6 +36,7 @@ GEM
     arel (9.0.0)
     ast (2.4.0)
     builder (3.2.3)
+    coderay (1.1.2)
     combustion (1.1.0)
       activesupport (>= 3.0.0)
       railties (>= 3.0.0)
@@ -63,6 +64,9 @@ GEM
     parser (2.6.3.0)
       ast (~> 2.4.0)
     pg (1.1.4)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
     rack (2.0.7)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
@@ -124,6 +128,7 @@ DEPENDENCIES
   bundler (~> 2.0)
   combustion (~> 1.1)
   database_cleaner (~> 1.7)
+  pry
   rake (~> 10.0)
   rspec (~> 3.0)
   rubocop-rspec (~> 1.33)

data/exe/table-saw

@@ -5,9 +5,7 @@ lib = File.expand_path('../lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'table_saw'
-require 'table_saw/build_dependency_graph'
 require 'table_saw/create_dump_file'
-require 'table_saw/manifest'
 require 'thor'
 
 class CLI < Thor
@@ -27,7 +25,7 @@ class CLI < Thor
   method_option :output, aliases: '-o', default: 'output.dump'
   def dump
     TableSaw.configure(options.to_hash)
-    records = TableSaw::BuildDependencyGraph.new(TableSaw::Manifest.instance).call
+    records = TableSaw::DependencyGraph::Build.new(TableSaw::Manifest.instance).call
     TableSaw::CreateDumpFile.new(records, options[:output]).call
   end
 end

data/lib/table_saw/create_dump_file.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require 'table_saw/queries/table_columns'
-require 'table_saw/queries/materialized_views'
-
 module TableSaw
   class CreateDumpFile
     attr_reader :records, :file
@@ -28,7 +25,7 @@ module TableSaw
         SET search_path = public, pg_catalog;
       SQL
 
-      records.each do |name, ids|
+      records.each do |name, table|
         write_to_file <<~COMMENT
           --
           -- Data for Name: #{name}; Type: TABLE DATA
@@ -41,7 +38,7 @@ module TableSaw
         SQL
 
         TableSaw::Connection.with do |conn|
-          conn.copy_data "COPY (select * from #{name} where id in (#{ids.join(',')})) TO STDOUT" do
+          conn.copy_data "COPY (#{table.copy_statement}) TO STDOUT" do
             while (row = conn.get_copy_data)
               write_to_file row
             end
@@ -53,6 +50,7 @@ module TableSaw
       end
 
       refresh_materialized_views
+      restart_sequences
 
       write_to_file 'COMMIT;'
     end
@@ -62,8 +60,20 @@ module TableSaw
 
     def refresh_materialized_views
       TableSaw::Queries::MaterializedViews.new.call.each do |view|
-        write_to_file "REFRESH MATERIALIZED VIEW #{view};"
+        write_to_file "refresh materialized view #{view};"
+      end
+
+      write_to_file "\n"
+    end
+
+    def restart_sequences
+      records.each_key do |table|
+        write_to_file <<~SQL
+          select setval(pg_get_serial_sequence('#{table}', 'id'), (select max(id) from #{table}), true);
+        SQL
       end
+
+      write_to_file "\n"
     end
 
     def write_to_file(data)

data/lib/table_saw/dependency_graph/add_directive.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module TableSaw
+  module DependencyGraph
+    class AddDirective
+      attr_reader :table_name, :partial
+      attr_accessor :ids
+
+      def initialize(table_name, ids: [], partial: true)
+        @table_name = table_name
+        @ids = ids
+        @partial = partial
+      end
+
+      alias partial? partial
+
+      def selectable?
+        partial? && Array(ids).size.positive?
+      end
+
+      def queryable?
+        !partial || selectable?
+      end
+    end
+  end
+end

data/lib/table_saw/dependency_graph/belongs_to_directives.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module TableSaw
+  module DependencyGraph
+    class BelongsToDirectives
+      attr_reader :context, :directive
+
+      def initialize(context, directive)
+        @context = context
+        @directive = directive
+      end
+
+      def call
+        associations.map do |from_column, to_table|
+          TableSaw::DependencyGraph::AddDirective.new(to_table, ids: ids[from_column].to_a, partial: directive.partial?)
+        end
+      end
+
+      private
+
+      def associations
+        context.belongs_to.fetch(directive.table_name, {})
+      end
+
+      def ids
+        @ids ||= query_result.each_with_object(Hash.new { |h, k| h[k] = Set.new }) do |row, memo|
+          associations.each_key { |key| memo[key].add row[key] unless row[key].nil? }
+        end
+      end
+
+      def query_result
+        return [] unless directive.selectable?
+
+        context.perform_query(
+          format('select %{columns} from %{table_name} where id in (%{ids})',
+                 columns: associations.keys.join(','), table_name: directive.table_name, ids: directive.ids.join(','))
+        )
+      end
+    end
+  end
+end

data/lib/table_saw/dependency_graph/build.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module TableSaw
+  module DependencyGraph
+    class Build
+      attr_reader :manifest, :records
+
+      def initialize(manifest)
+        @manifest = manifest
+        @records = {}
+      end
+
+      def call
+        manifest.tables.values.sort_by { |t| t.partial? ? 1 : 0 }.each do |table|
+          add TableSaw::DependencyGraph::AddDirective.new(table.name, ids: select_ids(table), partial: table.partial?)
+        end
+
+        records
+      end
+
+      private
+
+      def add(directive)
+        return [] unless directive.queryable?
+
+        directives(directive).select(&:queryable?).each(&method(:add))
+      end
+
+      def directives(dir)
+        record = records[dir.table_name]
+
+        if record
+          dir.partial? ? record.fetch_associations(dir) : []
+        else
+          TableSaw::DependencyGraph::DumpTable.new(context: context, name: dir.table_name, partial: dir.partial?)
+            .tap { |table| records[dir.table_name] = table }.fetch_associations(dir)
+        end
+      end
+
+      def context
+        @context ||= TableSaw::DependencyGraph::Context.new(manifest)
+      end
+
+      def select_ids(table)
+        return [] unless table.partial?
+
+        context.perform_query(table.query).map { |row| row['id'] }
+      end
+    end
+  end
+end

data/lib/table_saw/dependency_graph/context.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module TableSaw
+  module DependencyGraph
+    class Context
+      attr_reader :manifest
+
+      def initialize(manifest)
+        @manifest = manifest
+      end
+
+      def belongs_to
+        foreign_key_relationships.belongs_to
+      end
+
+      # rubocop:disable Naming/PredicateName
+      def has_many
+        foreign_key_relationships.has_many
+      end
+
+      def has_many_mapping
+        @has_many_mapping ||= manifest.tables.transform_values(&:has_many)
+      end
+
+      # rubocop:enable Naming/PredicateName
+
+      def foreign_key_relationships
+        @foreign_key_relationships ||= TableSaw::Queries::ForeignKeyRelationships.new
+      end
+
+      def tables_with_no_ids
+        @tables_with_no_ids ||= TableSaw::Queries::NoIdTables.new.call
+      end
+
+      def perform_query(sql)
+        TableSaw::Connection.with do |conn|
+          conn.exec(sql)
+        end
+      end
+    end
+  end
+end

data/lib/table_saw/dependency_graph/dump_table.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module TableSaw
+  module DependencyGraph
+    class DumpTable
+      attr_reader :context, :name, :partial, :ids
+
+      def initialize(context:, name:, partial: true)
+        @context = context
+        @name = name
+        @partial = partial
+        @ids = Set.new
+      end
+
+      def copy_statement
+        if partial
+          "select * from #{name} where id in (#{ids.to_a.join(',')})"
+        else
+          "select * from #{name}"
+        end
+      end
+
+      def fetch_associations(directive)
+        directive.ids = directive.ids - ids.to_a
+        ids.merge(directive.ids)
+        fetch_belongs_to(directive) + fetch_has_many(directive)
+      end
+
+      private
+
+      def fetch_belongs_to(directive)
+        TableSaw::DependencyGraph::BelongsToDirectives.new(context, directive).call
+      end
+
+      def fetch_has_many(directive)
+        TableSaw::DependencyGraph::HasManyDirectives.new(context, directive).call
+      end
+    end
+  end
+end

data/lib/table_saw/dependency_graph/has_many_directives.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module TableSaw
+  module DependencyGraph
+    class HasManyDirectives
+      attr_reader :context, :directive
+
+      def initialize(context, directive)
+        @context = context
+        @directive = directive
+      end
+
+      def call
+        valid_associations.map do |table, column|
+          TableSaw::DependencyGraph::AddDirective.new(
+            table, ids: query_result(table, column).map { |r| r['id'] }, partial: directive.partial?
+          )
+        end
+      end
+
+      private
+
+      def associations
+        context.has_many.fetch(directive.table_name, [])
+      end
+
+      def valid_associations
+        associations.select do |table, _column|
+          next false if directive.partial? && context.tables_with_no_ids.include?(table)
+
+          context.has_many_mapping.fetch(directive.table_name, []).include?(table)
+        end
+      end
+
+      def query_result(table, column)
+        return [] unless directive.selectable?
+
+        context.perform_query(
+          format('select id from %{table} where %{column} in (%{ids})',
+                 table: table, column: column, ids: directive.ids.join(','))
+        )
+      end
+    end
+  end
+end

data/lib/table_saw/dependency_graph.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'table_saw/dependency_graph/add_directive'
+require 'table_saw/dependency_graph/belongs_to_directives'
+require 'table_saw/dependency_graph/build'
+require 'table_saw/dependency_graph/context'
+require 'table_saw/dependency_graph/dump_table'
+require 'table_saw/dependency_graph/has_many_directives'

data/lib/table_saw/manifest.rb

@@ -31,6 +31,10 @@ module TableSaw
         config.fetch('has_many', [])
       end
       # rubocop:enable Naming/PredicateName
+
+      def partial?
+        config.key?('query')
+      end
     end
 
     def self.instance
@@ -46,7 +50,7 @@ module TableSaw
     end
 
     def variables
-      config['variables']
+      config.fetch('variables', {})
     end
 
     def tables

data/lib/table_saw/queries/foreign_key_relationships.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'table_saw/connection'
-
 module TableSaw
   module Queries
     class ForeignKeyRelationships

data/lib/table_saw/queries/materialized_views.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'table_saw/connection'
-
 module TableSaw
   module Queries
     class MaterializedViews

data/lib/table_saw/queries/table_columns.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'table_saw/connection'
-
 module TableSaw
   module Queries
     class TableColumns

data/lib/table_saw/queries.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require 'table_saw/queries/foreign_key_relationships'
+require 'table_saw/queries/materialized_views'
+require 'table_saw/queries/no_id_tables'
+require 'table_saw/queries/table_columns'

data/lib/table_saw/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TableSaw
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

data/lib/table_saw.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
-require 'table_saw/version'
 require 'table_saw/configuration'
+require 'table_saw/connection'
+require 'table_saw/dependency_graph'
+require 'table_saw/manifest'
+require 'table_saw/queries'
 
 module TableSaw
   def self.configuration

data/table_saw.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'combustion', '~> 1.1'
   spec.add_development_dependency 'database_cleaner', '~> 1.7'
+  spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop-rspec', '~> 1.33'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: table_saw
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Hamed Asghari
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-11 00:00:00.000000000 Z
+date: 2019-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -202,11 +216,18 @@ files:
 - bin/setup
 - exe/table-saw
 - lib/table_saw.rb
-- lib/table_saw/build_dependency_graph.rb
 - lib/table_saw/configuration.rb
 - lib/table_saw/connection.rb
 - lib/table_saw/create_dump_file.rb
+- lib/table_saw/dependency_graph.rb
+- lib/table_saw/dependency_graph/add_directive.rb
+- lib/table_saw/dependency_graph/belongs_to_directives.rb
+- lib/table_saw/dependency_graph/build.rb
+- lib/table_saw/dependency_graph/context.rb
+- lib/table_saw/dependency_graph/dump_table.rb
+- lib/table_saw/dependency_graph/has_many_directives.rb
 - lib/table_saw/manifest.rb
+- lib/table_saw/queries.rb
 - lib/table_saw/queries/foreign_key_relationships.rb
 - lib/table_saw/queries/materialized_views.rb
 - lib/table_saw/queries/no_id_tables.rb

data/lib/table_saw/build_dependency_graph.rb

@@ -1,97 +0,0 @@
-# frozen_string_literal: true
-
-require 'table_saw/connection'
-require 'table_saw/queries/foreign_key_relationships'
-require 'table_saw/queries/no_id_tables'
-
-module TableSaw
-  class BuildDependencyGraph
-    attr_reader :manifest, :records
-
-    def initialize(manifest)
-      @manifest = manifest
-      @records = Hash.new { |h, k| h[k] = [] }
-    end
-
-    def call
-      manifest.tables.each_value do |table|
-        add(table.name, perform_query(table.query).map { |row| row['id'] })
-      end
-
-      records
-    end
-
-    def add(table_name, ids)
-      return if ids.empty?
-
-      ids_to_add = ids - records[table_name]
-      return if ids_to_add.empty?
-
-      records[table_name].concat ids_to_add
-      fetch_belongs_to_associations(table_name, ids_to_add)
-      fetch_has_many_associations(table_name, ids_to_add)
-    end
-
-    # rubocop:disable Metrics/AbcSize
-    def fetch_belongs_to_associations(table_name, ids)
-      associations = belongs_to[table_name]
-      return if associations.empty?
-
-      rows = perform_query(
-        format('select %{columns} from %{table_name} where id in (%{ids})',
-               columns: associations.keys.join(','), table_name: table_name, ids: ids.join(','))
-      )
-
-      values = rows.each_with_object(Hash.new { |h, k| h[k] = Set.new }) do |row, memo|
-        associations.each_key { |key| memo[key].add row[key] unless row[key].nil? }
-      end
-
-      associations.each { |from_column, to_table| add to_table, values[from_column].to_a }
-    end
-    # rubocop:enable Metrics/AbcSize
-
-    def fetch_has_many_associations(table_name, ids)
-      has_many.fetch(table_name, []).each do |table, column|
-        next if tables_with_no_ids.include?(table)
-        next unless has_many_mapping.fetch(table_name, []).include?(table)
-
-        rows = perform_query(
-          format('select id from %{table} where %{column} in (%{ids})',
-                 table: table, column: column, ids: ids.join(','))
-        )
-
-        add(table, rows.map { |row| row['id'] })
-      end
-    end
-
-    private
-
-    def perform_query(sql)
-      TableSaw::Connection.with do |conn|
-        conn.exec(sql)
-      end
-    end
-
-    def belongs_to
-      foreign_key_relationships.belongs_to
-    end
-
-    # rubocop:disable Naming/PredicateName
-    def has_many
-      foreign_key_relationships.has_many
-    end
-
-    def has_many_mapping
-      @has_many_mapping ||= manifest.tables.transform_values(&:has_many)
-    end
-    # rubocop:enable Naming/PredicateName
-
-    def foreign_key_relationships
-      @foreign_key_relationships ||= TableSaw::Queries::ForeignKeyRelationships.new
-    end
-
-    def tables_with_no_ids
-      @tables_with_no_ids ||= TableSaw::Queries::NoIdTables.new.call
-    end
-  end
-end