data_sampler 0.0.5 → 0.0.6

data/lib/data_sampler/dependency.rb

@@ -16,7 +16,7 @@ module DataSampler
     end
 
     def to_s
-      "#{keys} in table `#{table_name}` (referred from `#{referring_table_name}`)"
+      "row with keys #{keys} in table `#{table_name}` (referred from `#{referring_table_name}`)"
     end
 
   end

data/lib/data_sampler/sample.rb

@@ -23,9 +23,10 @@ module DataSampler
       begin
         new_dependencies = 0
         @table_samples.values.each do |table_sample|
-          if table_sample.ensure_referential_integrity(@table_samples)
-            new_dependencies += 1 
-            warn "  Found new dependents for table `#{table_sample.table_name}`"
+          newly_added = table_sample.ensure_referential_integrity(@table_samples)
+          if newly_added > 0
+            new_dependencies += newly_added
+            warn "  Found #{newly_added} new dependents for table `#{table_sample.table_name}`"
           end
         end
         warn " Discovered #{new_dependencies} new dependencies" if new_dependencies > 0

data/lib/data_sampler/table_sample.rb

@@ -23,11 +23,11 @@ module DataSampler
     end
 
     def fulfil(dependency)
-      return if fulfilled?(dependency)
+      return 0 if fulfilled?(dependency)
       where = dependency.keys.collect { |col, val| "#{@connection.quote_column_name col} = #{@connection.quote val}" } * ' AND '
       sql = "SELECT * FROM #{@connection.quote_table_name @table_name} WHERE " + where
       row = @connection.select_one(sql)
-      raise "Could not find dependent row: #{dependency} (using SQL: #{sql})" if row.nil?
+      raise "Could not find #{dependency}" if row.nil?
       add row
     end
 
@@ -44,43 +44,47 @@ module DataSampler
     end
 
     def add(row)
-      return false unless @sample.add? row
+      return 0 unless @sample.add? row
       @sampled_ids.add row['id'] if row['id']
-      any_new = false
+      newly_added = 0
       dependencies_for(row).each do |dep|
-        any_new = true if @pending_dependencies.add?(dep)
+        newly_added += 1 if @pending_dependencies.add?(dep)
       end
-      any_new
+      newly_added
     rescue ActiveRecord::StatementInvalid => e
       # Don't choke on unknown table engines, such as Sphinx
     end
 
     def ensure_referential_integrity(table_samples)
-      any_new = false
+      newly_added = 0
       deps_in_progress = @pending_dependencies
       @pending_dependencies = Set.new
       deps_in_progress.each do |dependency|
         raise "Table sample for #{dependency.table_name} not found" unless table_samples[dependency.table_name]
-        any_new = true if table_samples[dependency.table_name].fulfil(dependency)
+        newly_added += table_samples[dependency.table_name].fulfil(dependency)
       end
-      any_new
+      newly_added
     end
 
     def to_sql
-      ret = ["-- #{@table_name}: #{@sample.count} rows"]
+      ret = "-- #{@table_name}: #{@sample.count} rows\n"
       unless @sample.empty?
         quoted_cols = @sample.first.keys.collect { |col| @connection.quote_column_name col }
-        sql = "INSERT INTO #{@connection.quote_table_name @table_name} (#{quoted_cols * ','})"
-        @sample.each do |row|
-          quoted_vals = []
-          row.each_pair do |field,val|
-            val.gsub! /./, '*' if field.downcase == 'password'
-            quoted_vals << @connection.quote(val)
-          end
-          ret << sql + " VALUES (#{quoted_vals * ','});"
+        # INSERT in batches of 1000
+        @sample.each_slice(1000) do |rows|
+          values = rows.collect { |row|
+            quoted_vals = []
+            row.each_pair do |field,val|
+              # HACK: Brute attempt at not revealing sensitive data
+              val.gsub! /./, '*' if field.downcase == 'password'
+              quoted_vals << @connection.quote(val)
+            end
+            quoted_vals * ','
+          } * '),('
+          ret << "INSERT INTO #{@connection.quote_table_name @table_name} (#{quoted_cols * ','}) VALUES (#{values});\n"
         end
       end
-      ret * "\n"
+      ret
     end
 
     protected

data/lib/data_sampler/version.rb

@@ -1,3 +1,3 @@
 module DataSampler
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_sampler
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors: