db_subsetter 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99f915db184481361de79ea28709bf1a13737c89
-  data.tar.gz: c916a1d6f148331a4565d4d16c030a26739384c6
+  metadata.gz: 3cbd626cd24132c2e8db14c89ecb66c300350886
+  data.tar.gz: f7171ef20aded756e45b11a04c5072ab0303e1ec
 SHA512:
-  metadata.gz: 594062ff2d049b7142d441cb6a208744fc27282bcc1ff912973f692cf4654614baf7245021001b72dd3cebeb1ad281cd63a0e75db09dc4d48a48f5a59989b1b4
-  data.tar.gz: 58a8e4bc3cbf6cdf19a4d9fade8d7cd645d7008025d7e47754afb0df27cae8c9e29f15c45fbc56955fecd53755faaf9447d1d8f6f60db932d432fa794fe5dd05
+  metadata.gz: dd3d89c74237f6a7c942bfb72561e3eaef426e34c3919f1c0b6651287e76650bebec068eb1b5b81c1dcf2c0d7959c43e1b5682be51ba6f21b62234afab1d4107
+  data.tar.gz: d9d8f8a4c5483e12fb9fe10cfee561c656fa67205c3968cc2c2cac6ca281ab7d76d9c8787b13534445434a0939f6477102ff7b2318c9e5e72effc282936280c1

data/db_subsetter.gemspec

@@ -27,5 +27,6 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "activerecord", "~> 4.2"
   spec.add_dependency "sqlite3", "~> 1.3"
+  spec.add_dependency "random-word"
 end
 

data/lib/db_subsetter/exporter.rb

@@ -49,7 +49,13 @@ module DbSubsetter
       end
     end
 
+    def add_scrambler(scrambler)
+      @scramblers << scrambler
+    end
 
+    def initialize
+      @scramblers = []
+    end
 
     private
     def max_unfiltered_rows
@@ -60,6 +66,7 @@ module DbSubsetter
       @max_filtered_rows || 2000
     end
 
+    # this is the batch size we insert into sqlite, which seems to be a reasonable balance of speed and memory usage
     def insert_batch_size
       250
     end
@@ -108,6 +115,13 @@ module DbSubsetter
       end
     end
 
+    def scramble_data(table, data)
+      @scramblers.each do |scrambler|
+        data = scrambler.scramble(table, data)
+      end
+      data
+    end
+
     def export_table(table)
       print "Exporting: #{table} (#{pages(table)} pages)" if @verbose
       $stdout.flush if @verbose
@@ -124,7 +138,7 @@ module DbSubsetter
 
         records = ActiveRecord::Base.connection.select_rows( sql )
         records.each_slice(insert_batch_size) do |rows|
-          @output.execute("INSERT INTO #{table.underscore} (data) VALUES #{ Array.new(rows.size){"(?)"}.join(",")}", rows.map{|x| cleanup_types(x)}.map(&:to_json) )
+          @output.execute("INSERT INTO #{table.underscore} (data) VALUES #{ Array.new(rows.size){"(?)"}.join(",")}", rows.map{|x| scramble_data(table, cleanup_types(x))}.map(&:to_json) )
           rows_exported += rows.size
         end
         print "." if @verbose

data/lib/db_subsetter/scrambler.rb

@@ -0,0 +1,28 @@
+require 'random-word'
+
+module DbSubsetter
+  class Scrambler
+    def scramble(table, row)
+      scramble_method = "scramble_#{table.downcase}"
+      if self.respond_to? scramble_method
+        self.send(scramble_method, row)
+      else
+        row
+      end
+    end
+
+    def initialize
+      @column_index_cache = {}
+    end
+
+    protected
+    def scramble_column(table, column, row_data, value)
+      row_data[column_index(table, column)] = value
+    end
+
+    private
+    def column_index(table, column)
+      @column_index_cache["#{table}##{column}"] ||= ActiveRecord::Base.connection.columns(table).map{|table| table.name}.index(column.to_s)
+    end
+  end
+end

data/lib/db_subsetter/version.rb

@@ -1,3 +1,3 @@
 module DbSubsetter
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/db_subsetter.rb

@@ -2,6 +2,7 @@ require "db_subsetter/version"
 require "db_subsetter/filter"
 require "db_subsetter/exporter"
 require "db_subsetter/importer"
+require "db_subsetter/scrambler"
 require "db_subsetter/dialect/generic"
 require "db_subsetter/dialect/my_sql"
 require "db_subsetter/dialect/ms_sql"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: db_subsetter
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Joe Francis
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: random-word
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - joe@lostapathy.com
@@ -103,6 +117,7 @@ files:
 - lib/db_subsetter/exporter.rb
 - lib/db_subsetter/filter.rb
 - lib/db_subsetter/importer.rb
+- lib/db_subsetter/scrambler.rb
 - lib/db_subsetter/version.rb
 homepage: https://github.com/lostapathy/db_subsetter
 licenses: