toku 0.1.0.4.3 → 0.1.0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1bd1fc5c406f8607ac17212e37ccca92a608b5b7
-  data.tar.gz: 6f75492ed9f5f4e78b813993c74a9264cc8fad3e
+  metadata.gz: d14fc10f4732aaa04b7d3fab0da39e55eb35c0d9
+  data.tar.gz: 72a5299d37764a11eb1556f3125b8fae80c8518e
 SHA512:
-  metadata.gz: 370d647af3b547f00da42f68d6d8816d5195180821048aae7d876afa5eef98515e4c92c107f1f561d618000107e79456ac3eaa1f5bf3bb84b3d8d0ae45cc9e8b
-  data.tar.gz: b0edd338a317f66fbdab1eb15f16950a1b17e72557b10725b29eb2598c4d9b577bb5416fdb5c073b2d86ed1922b478e2f7f63c7ed11898935a7b1cdc32c6294a
+  metadata.gz: b132102fd9d06dfcb2eda58fa76801bbf26a0f7990e0b4df2a5a035aa8b276d9e278b56d22a0e844ad4ee4da3b2af10481bce5cb7e0673a7c14d00b42bfb9dd6
+  data.tar.gz: f78d11cfa53c5502e818f9c6e03c1f324a489df3da064bda01b7df58e3e6feabce933cc03743e5fdaa7013c3d6772805e4941c56761a06c8d1111ea99f02d389

data/lib/tasks/generate_config.rake

@@ -1,18 +1,25 @@
 require 'yaml'
 # Run only in a Rails environment
-
+# Run this task to generate a base config file for your Rails application
+# after run it's up to you to whitelist specific arguments in the list
 namespace :toku do
   desc 'Generate base config file from your base database schema in Rails'
   task genesis_rails: :environment do
-    File.open(Rails.root.join('tmp').join('config.yml'), 'w') do |f|
+    path =  Rails.root.join('tmp').join('config.yml')
+    File.open(path, 'w') do |f|
       hash = {}
       ActiveRecord::Base.connection.tables.each do |table|
         hash[table] = { 'columns' => {}, 'rows' => [] }
         ActiveRecord::Base.connection.columns(table).map(&:name).each do |column|
-          hash[table]['columns'][column] = ['drop']
+          if column.ends_with?("_at") || column.ends_with?("id") || column == 'id'
+            hash[table]['columns'][column] = ['none']
+          else
+            hash[table]['columns'][column] = ['obfuscate']
+          end
         end
       end
       f.write(hash.to_yaml)
     end
+    puts "Pickup your config file in #{path}"
   end
 end

data/lib/toku.rb

@@ -1,5 +1,8 @@
 require "toku/version"
 require "uri"
+require 'sequel'
+require 'csv'
+require 'concurrent'
 
 Dir[File.dirname(__FILE__) + "/toku/**/*.rb"].each { |file| require file }
 
@@ -13,7 +16,9 @@ module Toku
       none: Toku::ColumnFilter::Passthrough,
       faker_last_name: Toku::ColumnFilter::FakerLastName,
       faker_first_name: Toku::ColumnFilter::FakerFirstName,
-      faker_email: Toku::ColumnFilter::FakerEmail
+      faker_email: Toku::ColumnFilter::FakerEmail,
+      obfuscate: Toku::ColumnFilter::Obfuscate,
+      nullify: Toku::ColumnFilter::Nullify
     }
 
     # A few default row filters mappings
@@ -21,11 +26,14 @@ module Toku
       drop: Toku::RowFilter::Drop
     }
 
-    SCHEMA_DUMP_PATH = "/tmp/toku_source_schema_dump.sql"
+    THREADPOOL_SIZE = (Concurrent.processor_count * 2).freeze
+
+    SCHEMA_DUMP_PATH = "tmp/toku_source_schema_dump.sql"
 
     # @param [String] config_file_path path of config file
     def initialize(config_file_path, column_filters = {}, row_filters = {})
-      @config = YAML.load(File.read(config_file_path))
+      @config = YAML.load(ERB.new(File.read(config_file_path)).result)
+      @threadpool = Concurrent::FixedThreadPool.new(THREADPOOL_SIZE)
       self.column_filters = column_filters.merge(COLUMN_FILTER_MAP)
       self.row_filters = row_filters.merge(ROW_FILTER_MAP)
       Sequel::Database.extension(:pg_streaming)
@@ -35,76 +43,99 @@ module Toku
     # @param uri_db_destination [String] URI of the destination DB
     # @return [void]
     def run(uri_db_source, uri_db_destination)
+      begin_time_stamp = Time.now
+      @global_count = 0
       source_db = Sequel.connect(uri_db_source)
-      dump_schema(URI(uri_db_source).path.tr("/", ""))
+      dump_schema(uri_db_source)
       parsed_destination_uri = URI(uri_db_destination)
       destination_db_name = parsed_destination_uri.path.tr("/", "")
-      destination_host = Sequel.connect("postgres://#{parsed_destination_uri.user}@#{parsed_destination_uri.host}:#{parsed_destination_uri.port}/template1")
-      destination_host.run("DROP DATABASE IF EXISTS #{destination_db_name}")
-      destination_host.run("CREATE DATABASE #{destination_db_name}")
+      destination_connection =
+        Sequel.connect("postgres://#{parsed_destination_uri.user}:#{parsed_destination_uri.password}@#{parsed_destination_uri.host}:#{parsed_destination_uri.port || 5432}/template1")
+      destination_connection.run("DROP DATABASE IF EXISTS #{destination_db_name}")
+      destination_connection.run("CREATE DATABASE #{destination_db_name}")
+      destination_connection.disconnect
       destination_db = Sequel.connect(uri_db_destination)
       destination_db.run(File.read(SCHEMA_DUMP_PATH))
+      destination_pool = Sequel::ThreadedConnectionPool.new(destination_db)
+      source_pool = Sequel::ThreadedConnectionPool.new(source_db)
 
-      raise Toku::SchemaMismatchError unless source_schema_included?(source_db, destination_db)
-
-      source_db.tables.each do |table|
-        if !row_filters?(table) && @config[table.to_s]['columns'].count < source_db.from(table).columns.count
+      source_db.tables.each do |t|
+        if !row_filters?(t) && @config[t.to_s]['columns'].count < source_db.from(t).columns.count
           raise Toku::ColumnFilterMissingError
         end
-        row_enumerator = source_db[table].stream.lazy
-
-        @config[table.to_s]['rows'].each do |f|
-          if f.is_a? String
-            row_filter = self.row_filters[f.to_sym].new({})
-          elsif f.is_a? Hash
-            row_filter = self.row_filters[f.keys.first.to_sym].new(f.values.first)
+        @threadpool.post do
+          destination_pool.hold do |destination_connection|
+            source_pool.hold do |source_connection|
+              process_table(t, source_connection.instance_variable_get(:@db), destination_connection.instance_variable_get(:@db))
+            end
           end
-
-          row_enumerator = row_filter.call(row_enumerator)
         end
-
-        row_enumerator = row_enumerator.map { |row| transform(row, table) }
-        destination_db.copy_into(table, data: row_enumerator, format: :csv)
-        count = destination_db[table].count
-        puts "Toku: copied #{count} objects into #{table} #{count != 0 ? ':)' : ':|'}"
       end
 
+      @threadpool.shutdown
+      @threadpool.wait_for_termination
       source_db.disconnect
       destination_db.disconnect
       FileUtils.rm(SCHEMA_DUMP_PATH)
+      puts "Toku: copied #{@global_count} elements in total and that took #{(Time.now - begin_time_stamp).round(2)} seconds with #{THREADPOOL_SIZE} green threads"
       nil
     end
 
     # @param name [Symbol]
     # @param row [Hash]
     # @return [String]
-    def transform(row, name)
-      row.map do |k, v|
-        filter_params = @config[name.to_s]['columns'][k.to_s].first
-        if filter_params.is_a? Hash
-          column_filter = self.column_filters[filter_params.keys.first.to_sym].new(
-            v,
-            filter_params.values.first
-          )
-        elsif filter_params.is_a? String
-          column_filter = self.column_filters[filter_params.to_sym].new(v, {})
+    def transform(row, table_name)
+      row.map do |row_key, row_value|
+        @config[table_name.to_s]['columns'][row_key.to_s].inject(row_value) do |result, filter|
+          if filter.is_a? Hash
+            filter_class(column_filters, filter.keys.first.to_sym).new(
+              result,
+              filter.values.first
+            ).call
+          elsif filter.is_a? String
+            filter_class(column_filters, filter.to_sym).new(result, {}).call
+          end
         end
-        column_filter.call
-      end.join(",") + "\n"
+      end.to_csv
     end
 
-    # Is the source database schema a subset of the destination database schema?
-    # @param source_db [String] URI of source database
-    # @param destination_db [String] URI of destination database
-    # @return [Boolean]
-    def source_schema_included?(source_db, destination_db)
-      source_db.tables.all? do |table|
-        source_db.schema(table) == destination_db.schema(table)
+    def process_table(table, source_connection, destination_connection)
+      row_enumerator = source_connection[table].stream.lazy
+      @config[table.to_s]['rows'].each do |f|
+        row_filter = if f.is_a? String
+          self.row_filters[f.to_sym].new({})
+        elsif f.is_a? Hash
+          self.row_filters[f.keys.first.to_sym].new(f.values.first)
+        end
+        row_enumerator = row_filter.call(row_enumerator)
       end
+
+      destination_connection.run("ALTER TABLE #{table} DISABLE TRIGGER ALL;")
+      destination_connection.copy_into(table, data: row_enumerator.map { |row| transform(row, table) }, format: :csv)
+      destination_connection.run("ALTER TABLE #{table} ENABLE TRIGGER ALL;")
+      count = destination_connection[table].count
+      @global_count += count
+      puts "Toku: copied #{count} objects into #{table} #{count != 0 ? ':)' : ':|'}"
+    end
+
+    # @param uri [String]
+    # @return [void]
+    def dump_schema(uri)
+      FileUtils::mkdir_p 'tmp'
+      host = URI(uri).host
+      password = URI(uri).password || ENV["PGPASSWORD"]
+      user = URI(uri).user
+      password = URI(uri).password
+      port = URI(uri).port || 5432
+      db_name = URI(uri).path.tr("/", "")
+      raise "pg_dump schema dump failed" unless system(
+        "PGPASSWORD=#{password} pg_dump -s -h #{host} -p #{port} -U #{user}  #{db_name} > #{SCHEMA_DUMP_PATH}"
+      )
     end
 
-    def dump_schema(db_name)
-      raise "Dump failed" unless system("pg_dump", "-s", "-x", "-O", "-f", SCHEMA_DUMP_PATH, "#{db_name}")
+    def filter_class(type, symbol)
+      raise "Please provide a filter for #{symbol}" if type[symbol].nil?
+      type[symbol]
     end
 
     # Are there row filters specified for this table?

data/lib/toku/filters/column/nullify.rb

@@ -0,0 +1,9 @@
+module Toku
+  class ColumnFilter
+    class Nullify < Toku::ColumnFilter
+      def initialize(value, options)
+        @value =  nil
+      end
+    end
+  end
+end

data/lib/toku/filters/column/obfuscate.rb

@@ -0,0 +1,9 @@
+module Toku
+  class ColumnFilter
+    class Obfuscate < Toku::ColumnFilter
+      def initialize(value, options)
+        @value =  SecureRandom.hex(10)
+      end
+    end
+  end
+end

data/lib/toku/version.rb

@@ -1,3 +1,3 @@
 module Toku
-  VERSION = "0.1.0.4.3"
+  VERSION = "0.1.0.4.4"
 end

data/toku.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'sequel'
   spec.add_dependency 'sequel_pg'
   spec.add_dependency 'faker'
+  spec.add_dependency 'concurrent-ruby'
 
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: toku
 version: !ruby/object:Gem::Version
-  version: 0.1.0.4.3
+  version: 0.1.0.4.4
 platform: ruby
 authors:
 - PSKL
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-02-20 00:00:00.000000000 Z
+date: 2018-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -53,6 +53,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -106,6 +120,8 @@ files:
 - lib/toku/filters/column/faker_email.rb
 - lib/toku/filters/column/faker_first_name.rb
 - lib/toku/filters/column/faker_last_name.rb
+- lib/toku/filters/column/nullify.rb
+- lib/toku/filters/column/obfuscate.rb
 - lib/toku/filters/column/passthrough.rb
 - lib/toku/filters/column_filter.rb
 - lib/toku/filters/row/drop.rb