db_subsetter 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0513f9d6c747b0a4b59915680c0f753f585c7176
-  data.tar.gz: dd987be888e834ffa1e2f832eed209206eb791e0
+  metadata.gz: 99f915db184481361de79ea28709bf1a13737c89
+  data.tar.gz: c916a1d6f148331a4565d4d16c030a26739384c6
 SHA512:
-  metadata.gz: 4c1bf06867d598238213377d5c8e8c1f47aa9736b8136038e4e49a8f34fa6f113fae065df8b798914824c0329b21baf4763f8e1b09437dafd8e7889af84dea75
-  data.tar.gz: faa183feff6d80f767310170a9dce02612baaa9816cb07225869da55585c9ab05d75d2af17047c9b7ba8e298ac80d4df5c56be7497ff9718b498a450027ebdee
+  metadata.gz: 594062ff2d049b7142d441cb6a208744fc27282bcc1ff912973f692cf4654614baf7245021001b72dd3cebeb1ad281cd63a0e75db09dc4d48a48f5a59989b1b4
+  data.tar.gz: 58a8e4bc3cbf6cdf19a4d9fade8d7cd645d7008025d7e47754afb0df27cae8c9e29f15c45fbc56955fecd53755faaf9447d1d8f6f60db932d432fa794fe5dd05

data/lib/db_subsetter/exporter.rb

@@ -1,8 +1,9 @@
 require 'sqlite3'
+require 'active_record'
 
 module DbSubsetter
   class Exporter
-    attr_writer :filter
+    attr_writer :filter, :max_unfiltered_rows, :max_filtered_rows
 
     def all_tables
       ActiveRecord::Base.connection.tables
@@ -26,15 +27,21 @@ module DbSubsetter
       end
     end
 
-    def verify_exportability
-      tables.each do |table|
-        verify_table_exportability(table)
+    def verify_exportability(verbose = true)
+      puts "Verifying table exportability ...\n\n" if verbose
+      errors = tables.map{|x| verify_table_exportability(x) }.flatten.compact
+      if errors.count > 0
+        puts errors.join("\n")
+        raise ArgumentError.new "Some tables are not exportable"
       end
+      puts "\n\n" if verbose
     end
 
-    def export(filename)
-      verify_exportability
+    def export(filename, verbose = true)
+      @verbose = verbose
+      verify_exportability(verbose)
 
+      puts "Exporting data...\n\n" if @verbose
       @output = SQLite3::Database.new(filename)
       @output.execute("CREATE TABLE tables (name TEXT, records_exported INTEGER, columns TEXT)")
       tables.each do |table|
@@ -45,8 +52,12 @@ module DbSubsetter
 
 
     private
-    def max_rows
-      10000000
+    def max_unfiltered_rows
+      @max_unfiltered_rows || 1000
+    end
+
+    def max_filtered_rows
+      @max_filtered_rows || 2000
     end
 
     def insert_batch_size
@@ -80,8 +91,11 @@ module DbSubsetter
     end
 
     def verify_table_exportability(table)
-      raise "ERROR: Multiple pages but no primary key on: #{table}" if pages(table) > 1 && order_by(table).blank?
-      raise "ERROR: Too many rows in: #{table} (#{filtered_row_count(table)})" if( filtered_row_count(table) > max_rows )
+      puts "Verifying: #{table}" if @verbose
+      errors = []
+      errors << "ERROR: Multiple pages but no primary key on: #{table}" if pages(table) > 1 && order_by(table).blank?
+      errors << "ERROR: Too many rows in: #{table} (#{filtered_row_count(table)})" if( filtered_row_count(table) > max_filtered_rows )
+      errors
     end
 
     def cleanup_types(row)
@@ -95,10 +109,12 @@ module DbSubsetter
     end
 
     def export_table(table)
+      print "Exporting: #{table} (#{pages(table)} pages)" if @verbose
+      $stdout.flush if @verbose
       columns = ActiveRecord::Base.connection.columns(table).map{ |table| table.name }
       rows_exported = 0
       @output.execute("CREATE TABLE #{table.underscore} ( data TEXT )")
-      for i in 0..pages(table)
+      for i in 0..(pages(table) - 1)
         arel_table = query = Arel::Table.new(table, ActiveRecord::Base)
         query = filter.filter(table, query)
         # Need to extend this to take more than the first batch_size records
@@ -111,7 +127,10 @@ module DbSubsetter
           @output.execute("INSERT INTO #{table.underscore} (data) VALUES #{ Array.new(rows.size){"(?)"}.join(",")}", rows.map{|x| cleanup_types(x)}.map(&:to_json) )
           rows_exported += rows.size
         end
+        print "." if @verbose
+        $stdout.flush if @verbose
       end
+      puts "" if @verbose
       @output.execute("INSERT INTO tables VALUES (?, ?, ?)", [table, rows_exported, columns.to_json])
     end
   end

data/lib/db_subsetter/filter.rb

@@ -1,3 +1,5 @@
+require 'active_record'
+
 module DbSubsetter
   class Filter
     attr_writer :exporter

data/lib/db_subsetter/version.rb

@@ -1,3 +1,3 @@
 module DbSubsetter
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: db_subsetter
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Joe Francis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-15 00:00:00.000000000 Z
+date: 2016-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler