db_subsetter 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8b69d7c3714c570b4a1f22b2ccff5e6e74c8b670
-  data.tar.gz: dafe34226a0da60c16d2823cc6d76e12b2f9837a
+  metadata.gz: 0513f9d6c747b0a4b59915680c0f753f585c7176
+  data.tar.gz: dd987be888e834ffa1e2f832eed209206eb791e0
 SHA512:
-  metadata.gz: 6a7deca37ab7b6267f9b9f0fbabd7dd221278e93366623df347a46d24aa9a6980c71ec0f9f0c3a3d41b4aaf3b44032ee185092671f1c6517fd9c70e449265c25
-  data.tar.gz: d50010b3ee53ce2b2a13c95dcae3f00b08681a0f802251d85c6e6764375b4b499c9db817087a2b62c011bed578a161464b9c36abfeaf119ad772e7a2f12cda88
+  metadata.gz: 4c1bf06867d598238213377d5c8e8c1f47aa9736b8136038e4e49a8f34fa6f113fae065df8b798914824c0329b21baf4763f8e1b09437dafd8e7889af84dea75
+  data.tar.gz: faa183feff6d80f767310170a9dce02612baaa9816cb07225869da55585c9ab05d75d2af17047c9b7ba8e298ac80d4df5c56be7497ff9718b498a450027ebdee

data/README.md

@@ -35,6 +35,17 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+## TODO
+
+* Improve the dialect handling
+* Better example docs on usage and filtering examples
+* Implement a scrubber API to allow sanitizing or correcting data at export time.  This allows us to keep sensitive/personal data out of the export and also allows correction of broken data that won't re-insert.
+* Add an executable and/or rake task to perform export and import rather than requiring the API to used directly.  Will need a config file to specific custom plugins
+* Add pre-flight check on import to make sure all tables smell like they will load the data (right columns, at minimum)
+* Finish building and test checks to make sure foreign keys are valid after import
+* Have verify_exportability return all failures together rather than one at a time
+* Add a verbose mode to display more detailed stats while running an export or import (what table we're on, records exported, time taken
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/lostapathy/db_subsetter.

data/lib/db_subsetter/dialect/generic.rb

@@ -0,0 +1,14 @@
+module DbSubsetter
+  module Dialect
+    class Generic
+      def self.import
+        yield
+      end
+
+      def self.integrity_problems
+        []
+      end
+    end
+  end
+end
+

data/lib/db_subsetter/dialect/ms_sql.rb

@@ -0,0 +1,24 @@
+module DbSubsetter
+  module Dialect
+    class MSSQL < Generic
+      def self.import
+        ActiveRecord::Base.connection.execute('EXEC sp_msforeachtable "ALTER TABLE ? NOCHECK CONSTRAINT all"')
+        ActiveRecord::Base.connection.execute('EXEC sp_msforeachtable "ALTER TABLE ? DISABLE TRIGGER all"')
+        ActiveRecord::Base.connection.execute("select 'ALTER INDEX ' + I.name + ' ON ' + T.name + ' DISABLE'
+            from sys.indexes I
+            inner join sys.tables T on I.object_id = T.object_id
+            where I.type_desc = 'NONCLUSTERED'
+            and I.name is not null")
+
+        yield
+        ActiveRecord::Base.connection.execute('EXEC sp_msforeachtable "ALTER TABLE ? ENABLE TRIGGER all"')
+        ActiveRecord::Base.connection.execute('EXEC sp_msforeachtable "ALTER TABLE ? WITH CHECK CHECK CONSTRAINT all"')
+      end
+
+      def self.integrity_problems
+        ActiveRecord::Base.connection.execute('EXEC sp_msforeachtable "DBCC CHECKCONSTRAINTS WITH ALL_CONSTRAINTS"')
+      end
+    end
+  end
+end
+

data/lib/db_subsetter/dialect/my_sql.rb

@@ -0,0 +1,16 @@
+module DbSubsetter
+  module Dialect
+    class MySQL < Generic
+      def self.import
+        ActiveRecord::Base.connection.execute("SET FOREIGN_KEY_CHECKS=0;")
+        yield
+        ActiveRecord::Base.connection.execute("SET FOREIGN_KEY_CHECKS=1;")
+      end
+
+      def self.integrity_problems
+        raise NotImplementedError.new("integrity_problems not implemented for MySQL")
+      end
+    end
+  end
+end
+

data/lib/db_subsetter/exporter.rb

@@ -36,7 +36,7 @@ module DbSubsetter
       verify_exportability
 
       @output = SQLite3::Database.new(filename)
-      @output.execute("CREATE TABLE tables (name TEXT, columns TEXT)")
+      @output.execute("CREATE TABLE tables (name TEXT, records_exported INTEGER, columns TEXT)")
       tables.each do |table|
         export_table(table)
       end
@@ -64,8 +64,8 @@ module DbSubsetter
     end
 
     def filtered_row_count(table)
-      query = Arel::Table.new(table, ActiveRecord::Base).project( Arel.sql("count(1)") )
-      query = filter.filter(table, query)
+      query = Arel::Table.new(table, ActiveRecord::Base)
+      query = filter.filter(table, query).project( Arel.sql("count(1)") )
       ActiveRecord::Base.connection.select_one(query.to_sql).values.first
     end
 
@@ -96,22 +96,24 @@ module DbSubsetter
 
     def export_table(table)
       columns = ActiveRecord::Base.connection.columns(table).map{ |table| table.name }
-      @output.execute("INSERT INTO tables VALUES (?, ?)", [table, columns.to_json])
+      rows_exported = 0
       @output.execute("CREATE TABLE #{table.underscore} ( data TEXT )")
       for i in 0..pages(table)
-        query = Arel::Table.new(table, ActiveRecord::Base)
+        arel_table = query = Arel::Table.new(table, ActiveRecord::Base)
+        query = filter.filter(table, query)
         # Need to extend this to take more than the first batch_size records
-        query = query.order(query[order_by(table)]) if order_by(table)
+        query = query.order(arel_table[order_by(table)]) if order_by(table)
 
         sql = query.skip(i * select_batch_size).take(select_batch_size).project( Arel.sql('*') ).to_sql
 
         records = ActiveRecord::Base.connection.select_rows( sql )
         records.each_slice(insert_batch_size) do |rows|
           @output.execute("INSERT INTO #{table.underscore} (data) VALUES #{ Array.new(rows.size){"(?)"}.join(",")}", rows.map{|x| cleanup_types(x)}.map(&:to_json) )
+          rows_exported += rows.size
         end
       end
+      @output.execute("INSERT INTO tables VALUES (?, ?, ?)", [table, rows_exported, columns.to_json])
     end
-
   end
 end
 

data/lib/db_subsetter/filter.rb

@@ -11,7 +11,7 @@ module DbSubsetter
     end
 
     def filter(table, query)
-      filter_method = "filter_#{table}"
+      filter_method = "filter_#{table.downcase}"
       if self.respond_to? filter_method
         self.send(filter_method, query)
       else

data/lib/db_subsetter/importer.rb

@@ -3,10 +3,11 @@ require 'sqlite3'
 module DbSubsetter
   class Importer
 
-    def initialize(filename)
+    def initialize(filename, dialect = DbSubsetter::Dialect::Generic)
       raise ArgumentError.new("invalid input file") unless File.exists?(filename)
 
       @data = SQLite3::Database.new(filename)
+      @dialect = dialect
     end
 
     def tables
@@ -18,25 +19,40 @@ module DbSubsetter
     end
 
     def import
-     ActiveRecord::Base.connection.execute("SET FOREIGN_KEY_CHECKS=0;")
-     tables.each do |table|
-        import_table(table)
+      @dialect.import do
+        tables.each do |table|
+          import_table(table)
+        end
       end
-     ActiveRecord::Base.connection.execute("SET FOREIGN_KEY_CHECKS=1;")
+    end
+
+    def insert_batch_size
+      100 # more like 500 for mysql
     end
 
     private
     def import_table(table)
-      ActiveRecord::Base.connection.truncate(table)
-      @data.execute("SELECT data FROM #{table.underscore}") do |row|
-        insert_sql = "INSERT INTO #{quoted_table_name(table)} (#{quoted_column_names(table).join(",")}) VALUES (#{quoted_values(row).join(",")})"
+      begin
+        ActiveRecord::Base.connection.truncate(table)
+      rescue NotImplementedError
+        ActiveRecord::Base.connection.execute("DELETE FROM #{quoted_table_name(table)}")
+      end
+
+      ActiveRecord::Base.connection.begin_db_transaction
+
+      all_rows = @data.execute("SELECT data FROM #{table.underscore}")
+      all_rows.each_slice(insert_batch_size) do |rows|
+        quoted_rows = rows.map{ |row| "(" + quoted_values(row).join(",") + ")" }.join(",")
+        insert_sql = "INSERT INTO #{quoted_table_name(table)} (#{quoted_column_names(table).join(",")}) VALUES #{quoted_rows}"
         ActiveRecord::Base.connection.execute(insert_sql)
       end
+
+       ActiveRecord::Base.connection.commit_db_transaction
     end
 
     def quoted_values(row)
       out = JSON.parse(row[0])
-      out = out.map{|x| ActiveRecord::Base.connection.type_cast(x, nil) } #.first, x.last) }
+      out = out.map{|x| ActiveRecord::Base.connection.type_cast(x, nil) }
       out = out.map{|x| ActiveRecord::Base.connection.quote(x) }
       out
     end

data/lib/db_subsetter/version.rb

@@ -1,3 +1,3 @@
 module DbSubsetter
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/db_subsetter.rb

@@ -2,6 +2,10 @@ require "db_subsetter/version"
 require "db_subsetter/filter"
 require "db_subsetter/exporter"
 require "db_subsetter/importer"
+require "db_subsetter/dialect/generic"
+require "db_subsetter/dialect/my_sql"
+require "db_subsetter/dialect/ms_sql"
+
 
 module DbSubsetter
   # Your code goes here...

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: db_subsetter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Joe Francis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-02 00:00:00.000000000 Z
+date: 2016-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -97,6 +97,9 @@ files:
 - bin/setup
 - db_subsetter.gemspec
 - lib/db_subsetter.rb
+- lib/db_subsetter/dialect/generic.rb
+- lib/db_subsetter/dialect/ms_sql.rb
+- lib/db_subsetter/dialect/my_sql.rb
 - lib/db_subsetter/exporter.rb
 - lib/db_subsetter/filter.rb
 - lib/db_subsetter/importer.rb