my_obfuscate 0.5.3 → 0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 8e7f50293f88ff7eea7bdce3905110af4602229c
-  data.tar.gz: 4008d4d1bb153fde2125dabcf2afdcbef334fbdc
+SHA256:
+  metadata.gz: bf957dc89a1e6833eb09616f54f2da012d24e9bec472a72cd861e646a4e85abf
+  data.tar.gz: cc7ce4fe94578d843bf383dbe685aefd5ea3fcba3d026631bf1ab5be231fbfad
 SHA512:
-  metadata.gz: 77a9e4325cabd6434e89928d6c1939f914cad6bfe0289618c4a91f67a4c6d237c27a81e98ae43a5b4913b75b1504a1723cebd0f9f1829a938bcea249dd36270c
-  data.tar.gz: b55e7c9b10453a22b77c8231a2f9665393aff73bc1fbcda9bf48959ad3a1846042943dc27d7d4a96cf8c9cb991589139d5caa4323ae753a44cc1163b0b47ed3e
+  metadata.gz: 8a3040cda7709223f02815f67c431664d4ae2564c33f3f8c6d00124fe22af3b6cb89edac68fdf1100b930d093d4fe60d5471dd8bb753448467063913d4dd8190
+  data.tar.gz: 10440115a506cf1265db0cbc76fb19b11292578ac58f2c55e976ef895a7d4be24a44946e870f388e93d889a9f7023c73e9e861a894b595a9ba48ae21b718e6dc

data/.ruby-version

@@ -1 +1 @@
-ruby-2.1.2
+ruby-2.4.0

data/README.rdoc

@@ -48,6 +48,7 @@ Make an obfuscator.rb script:
   })
   obfuscator.fail_on_unspecified_columns = true # if you want it to require every column in the table to be in the above definition
   obfuscator.globally_kept_columns = %w[id created_at updated_at] # if you set fail_on_unspecified_columns, you may want this as well
+  # If you'd like to also validate against your schema.rb file to make sure all fields and tables are present, see https://gist.github.com/cantino/5376e73b0ad806dc4da4
   obfuscator.obfuscate(STDIN, STDOUT)
 
 And to get an obfuscated dump:
@@ -72,9 +73,34 @@ If using Postgres, use pg_dump to get a dump:
 
 == Types
 
-Available types include: email, string, lorem, name, first_name, last_name, address, street_address, city, state,
+Available types include: email, string, lorem, name, first_name, last_name, address, street_address, secondary_address, city, state,
 zip_code, phone, company, ipv4, ipv6, url, integer, fixed, null, and keep.
 
+== Helping with creation of the "obfuscator.rb" script
+
+If you don't want to type all those table names and column names into your obfuscator.rb script,
+you can use my_obfuscate to do some of that work for you. It can consume your database dump file and create a "scaffold" for the script.
+To run my_obfuscate in this mode, start with an "empty" scaffolder.rb script as follows:
+
+  #!/usr/bin/env ruby
+  require "rubygems"
+  require "my_obfuscate"
+
+  obfuscator = MyObfuscate.new({})
+  obfuscator.scaffold(STDIN, STDOUT)
+
+Then feed in your database dump:
+  mysqldump -c  --hex-blob -u user -ppassword database | ruby scaffolder.rb > obfuscator_scaffold.rb_snippet
+  pg_dump database | ruby scaffolder.rb > obfuscator_scaffold.rb_snippet
+
+The output will be a series of configuration statements of the form:
+    :table_name => {
+      :column1_name     => :keep   # scaffold
+      :column2_name     => :keep   # scaffold
+  	... etc.
+    
+Scaffolding also works if you have a partial configuration.  If your configuration is missing some tables or some columns, a call to 'scaffold' will pass through the configuration that exists and augment it with scaffolding for the missing tables or columns. 
+
 == Changes
 
 * Support for Postgres.  Thanks @samuelreh!

data/Rakefile

@@ -6,3 +6,4 @@ RSpec::Core::RakeTask.new(:spec) do |spec|
 end
 
 task :default => :spec
+# comment

data/lib/my_obfuscate/config_applicator.rb

@@ -34,7 +34,7 @@ class MyObfuscate
           when :string
             random_string(definition[:length] || 30, definition[:chars] || SENSIBLE_CHARS)
           when :lorem
-            clean_bad_whitespace(clean_quotes(Faker::Lorem.sentences(definition[:number] || 1).join(".  ")))
+            clean_bad_whitespace(clean_quotes(Faker::Lorem.sentences(number: definition[:number] || 1).join(" ")))
           when :like_english
             clean_quotes random_english_sentences(definition[:number] || 1)
           when :name
@@ -44,15 +44,17 @@ class MyObfuscate
           when :last_name
             clean_quotes(Faker::Name.last_name)
           when :address
-            clean_quotes("#{Faker::AddressUS.street_address}\\n#{Faker::AddressUS.city}, #{Faker::AddressUS.state_abbr} #{Faker::AddressUS.zip_code}")
+            clean_quotes(Faker::Address.full_address)
           when :street_address
-            clean_bad_whitespace(clean_quotes(Faker::AddressUS.street_address))
+            clean_bad_whitespace(clean_quotes(Faker::Address.street_address))
+          when :secondary_address
+            clean_bad_whitespace(clean_quotes(Faker::Address.secondary_address))
           when :city
-            clean_quotes(Faker::AddressUS.city)
+            clean_quotes(Faker::Address.city)
           when :state
-            clean_quotes Faker::AddressUS.state_abbr
+            clean_quotes Faker::Address.state_abbr
           when :zip_code
-            Faker::AddressUS.zip_code
+            Faker::Address.zip_code
           when :phone
             clean_quotes Faker::PhoneNumber.phone_number
           when :company
@@ -60,12 +62,9 @@ class MyObfuscate
           when :ipv4
             Faker::Internet.ip_v4_address
           when :ipv6
-            # Inlined from Faker because ffaker doesn't have ipv6.
-            @@ip_v6_space ||= (0..65535).to_a
-            container = (1..8).map{ |_| @@ip_v6_space.sample }
-            container.map{ |n| n.to_s(16) }.join(':')
+            Faker::Internet.ip_v6_address
           when :url
-            clean_bad_whitespace(Faker::Internet.http_url)
+            clean_bad_whitespace(Faker::Internet.url)
           when :integer
             random_integer(definition[:between] || (0..1000)).to_s
           when :fixed
@@ -106,7 +105,7 @@ class MyObfuscate
     end
 
     def self.random_string(length_or_range, chars)
-      length_or_range = (length_or_range..length_or_range) if length_or_range.is_a?(Fixnum)
+      length_or_range = (length_or_range..length_or_range) if length_or_range.is_a?(Integer)
       times = random_integer(length_or_range)
       out = ""
       times.times { out << chars[rand * chars.length] }

data/lib/my_obfuscate/config_scaffold_generator.rb

@@ -0,0 +1,85 @@
+class MyObfuscate
+  module ConfigScaffoldGenerator
+
+    def generate_config(obfuscator, config, input_io, output_io)
+      input_io.each do |line|
+        if obfuscator.database_type == :postgres
+          table_data = parse_copy_statement(line)
+        else
+          table_data = parse_insert_statement(line)
+        end
+        next unless table_data
+
+        table_name = table_data[:table_name]
+        next if obfuscator.scaffolded_tables[table_name]    # only process each table_name once
+
+        columns = table_data[:column_names]
+        table_config = config[table_name]
+        next if table_config == :truncate || table_config == :keep
+
+        missing_columns = obfuscator.missing_column_list(table_name, columns)
+        extra_columns = obfuscator.extra_column_list(table_name, columns)
+
+        if missing_columns.count == 0 && extra_columns.count == 0
+          # all columns are accounted for
+          output_io.puts "\n# All columns in the config for #{table_name.upcase} are present and accounted for."
+        else
+          # there are columns missing (or perhaps the whole table is missing); show a scaffold
+          emit_scaffold(table_name, table_config, extra_columns, missing_columns, output_io)
+        end
+
+        # Now that this table_name has been processed, remember it so we don't scaffold it again
+        obfuscator.scaffolded_tables[table_name] = 1
+      end
+    end
+
+    def config_table_open(table_name)
+      "\n  :#{table_name} => {"
+    end
+
+    def config_table_close(table_name)
+      "  },"
+    end
+
+    def emit_scaffold(table_name, existing_config, extra_columns, columns_to_scaffold, output_io)
+
+      # header block: contains table name and any existing config
+      if existing_config
+        output_io.puts config_table_open(table_name)
+        existing_config.each do |column, definition|
+          unless extra_columns.include?(column)
+            output_io.puts formatted_line(column, definition)
+          end
+        end
+      end
+
+      extra_columns.each do |column|
+        output_string = formatted_line(column, existing_config[column], "# unreferenced config")
+        output_io.puts "#  #{output_string}"
+      end
+
+      # scaffold block: contains any config that's not already present
+      output_io.puts config_table_open(table_name) unless existing_config
+
+      scaffold = columns_to_scaffold.map do |column|
+        formatted_line(column, "keep", "# scaffold")
+      end.join("\n").chomp(',')
+      output_io.puts scaffold
+      output_io.puts config_table_close(table_name)
+    end
+
+    def formatted_line(column, definition, comment = nil)
+      colon_string = if (definition.to_s[0]=="{" || definition.to_s[0]==":") then definition.to_s else ":#{definition}" end
+
+      if column.length < 40
+        "    :#{'%-40.40s' % column}  => #{colon_string},   #{comment}"
+      else
+        "    :#{column} => #{definition},  #{comment}"
+      end
+
+    end
+
+  end
+end
+
+

data/lib/my_obfuscate/copy_statement_parser.rb

@@ -29,12 +29,13 @@ class MyObfuscate
           end
 
           output_io.write line
-        elsif line.match /\S*\.\n/
+        elsif line.match /^\\\.$/
           inside_copy_statement = false
 
           output_io.write line
         elsif inside_copy_statement
-          output_io.puts obfuscator.obfuscate_bulk_insert_line(line, current_table_name, current_columns)
+          obfuscated_line = obfuscator.obfuscate_bulk_insert_line(line, current_table_name, current_columns)
+          output_io.puts obfuscated_line unless obfuscated_line.empty?
         else
           output_io.write line
         end

data/lib/my_obfuscate/insert_statement_parser.rb

@@ -6,8 +6,9 @@ class MyObfuscate
         if table_data = parse_insert_statement(line)
           table_name = table_data[:table_name]
           columns = table_data[:column_names]
+          ignore = table_data[:ignore]
           if config[table_name]
-            output_io.puts obfuscator.obfuscate_bulk_insert_line(line, table_name, columns)
+            output_io.puts obfuscator.obfuscate_bulk_insert_line(line, table_name, columns, ignore)
           else
             $stderr.puts "Deprecated: #{table_name} was not specified in the config.  A future release will cause this to be an error.  Please specify the table definition or set it to :keep."
             output_io.write line
@@ -20,3 +21,5 @@ class MyObfuscate
 
   end
 end
+
+

data/lib/my_obfuscate/mysql.rb

@@ -1,31 +1,88 @@
+#encoding: UTF-8
+require 'stringio'
+require 'strscan'
+
 class MyObfuscate
   class Mysql
     include MyObfuscate::InsertStatementParser
+    include MyObfuscate::ConfigScaffoldGenerator
+
+    LPAREN = /\(/
+    RPAREN = /\)/
+    NULL_LITERAL = /NULL/
+    STRING_LITERAL = /'(\\\\|\\'|.)*?'/ # Matching "\\" followed by "\'" followed by . ensures proper escape handling
+    OTHER_LITERAL = /[^,\)]+/           # All other literals are terminated by separator or right paren
+    WHITESPACE = /[\s,;]+/              # We treat the "," separator and ";" terminator as whitespace
 
     def parse_insert_statement(line)
       if regex_match = insert_regex.match(line)
         {
-            :table_name => regex_match[1].to_sym,
-            :column_names => regex_match[2].split(/`\s*,\s*`/).map { |col| col.gsub('`', "").to_sym }
+            :ignore     => !regex_match[1].nil?,
+            :table_name => regex_match[2].to_sym,
+            :column_names => regex_match[3].split(/`\s*,\s*`/).map { |col| col.gsub('`', "").to_sym }
         }
       end
     end
 
-    def make_insert_statement(table_name, column_names, values)
-      values_strings = values.collect do |values|
-        "(" + values.join(",") + ")"
-      end.join(",")
+    def make_insert_statement(table_name, column_names, rows, ignore = nil)
+      buffer = StringIO.new
+      buffer.write "INSERT #{ignore ? 'IGNORE ' : '' }INTO `#{table_name}` (`#{column_names.join('`, `')}`) VALUES "
+      write_rows(buffer, rows)
+      buffer.write ";"
+      buffer.string
+    end
+
+    def write_rows(buffer, rows)
+      rows.each_with_index do |row_values, i|
+        buffer.write("(")
+        write_row_values(buffer, row_values)
+        buffer.write(")")
+        buffer.write(",") if i < rows.length - 1
+      end
+    end
 
-      "INSERT INTO `#{table_name}` (`#{column_names.join('`, `')}`) VALUES #{values_strings};"
+    def write_row_values(buffer, row_values)
+      row_values.each_with_index do |value, j|
+        buffer.write value
+        buffer.write(",") if j < row_values.length - 1
+      end
     end
 
     def insert_regex
-      /^\s*INSERT INTO `(.*?)` \((.*?)\) VALUES\s*/i
+      /^\s*INSERT\s*(IGNORE )?\s*INTO `(.*?)` \((.*?)\) VALUES\s*/i
     end
 
     def rows_to_be_inserted(line)
-      line = line.gsub(insert_regex, '').gsub(/\s*;\s*$/, '')
-      context_aware_mysql_string_split(line)
+      scanner = StringScanner.new line
+      scanner.scan insert_regex
+
+      rows = []
+      row_values = []
+      until scanner.eos?
+        if scanner.scan(LPAREN)
+          # Left paren indicates the start of a row of (val1, val2, ..., valn)
+          row_values = []
+        elsif scanner.scan(RPAREN)
+          # Right paren indicates the end of a row of (val1, val2, ..., valn)
+          rows << row_values
+        elsif scanner.scan(NULL_LITERAL)
+          row_values << nil
+        elsif match = scanner.scan(STRING_LITERAL)
+          # We drop the leading and trailing quotes to extract the string
+          row_values << match.slice(1, match.length - 2)
+        elsif match = scanner.scan(OTHER_LITERAL)
+          # All other literals.  We match these up to the "," separator or ")" closing paren.
+          # Hence we rstrip to drop any whitespace between the literal and the "," or ")".
+          row_values << match.rstrip
+        else
+          # This is minimal validation.  We're assuming valid input generated by mysqldump.
+          raise "Parse error: unexpected token begginning at #{scanner.peek 80}"
+        end
+        # Ignore whitespace/separator after any token
+        scanner.scan(WHITESPACE)
+      end
+
+      rows
     end
 
     def make_valid_value_string(value)
@@ -37,61 +94,5 @@ class MyObfuscate
         "'" + value + "'"
       end
     end
-
-    # Be aware, strings must be quoted in single quotes!
-    def context_aware_mysql_string_split(string)
-      in_sub_insert = false
-      in_quoted_string = false
-      escaped = false
-      current_field = nil
-      length = string.length
-      fields = []
-      output = []
-
-      string.each_char do |i|
-        if escaped
-          escaped = false
-          current_field ||= ""
-          current_field << i
-        else
-          if i == "\\"
-            escaped = true
-            current_field ||= ""
-            current_field << i
-          elsif i == "(" && !in_quoted_string && !in_sub_insert
-            in_sub_insert = true
-          elsif i == ")" && !in_quoted_string && in_sub_insert
-            fields << current_field unless current_field.nil?
-            output << fields unless fields.length == 0
-            in_sub_insert = false
-            fields = []
-            current_field = nil
-          elsif i == "'" && !in_quoted_string
-            fields << current_field unless current_field.nil?
-            current_field = ''
-            in_quoted_string = true
-          elsif i == "'" && in_quoted_string
-            fields << current_field unless current_field.nil?
-            current_field = nil
-            in_quoted_string = false
-          elsif i == "," && !in_quoted_string && in_sub_insert
-            fields << current_field unless current_field.nil?
-            current_field = nil
-          elsif i == "L" && !in_quoted_string && in_sub_insert && current_field == "NUL"
-            current_field = nil
-            fields << current_field
-          elsif (i == " " || i == "\t") && !in_quoted_string
-            # Don't add whitespace not in a string
-          elsif in_sub_insert
-            current_field ||= ""
-            current_field << i
-          end
-        end
-      end
-
-      fields << current_field unless current_field.nil?
-      output << fields unless fields.length == 0
-      output
-    end
   end
 end

data/lib/my_obfuscate/postgres.rb

@@ -1,8 +1,9 @@
 class MyObfuscate
   class Postgres
     include MyObfuscate::CopyStatementParser
+    include MyObfuscate::ConfigScaffoldGenerator
 
-    # Copy statements contain the column values tab seperated like so:
+    # Copy statements contain the column values tab separated like so:
     #   blah	blah	blah	blah
     # which we want to turn into:
     #   [['blah','blah','blah','blah']]
@@ -33,7 +34,7 @@ class MyObfuscate
       end
     end
 
-    def make_insert_statement(table_name, column_names, values)
+    def make_insert_statement(table_name, column_names, values, ignore = nil)
       values.join("\t")
     end
 

data/lib/my_obfuscate/sql_server.rb

@@ -1,6 +1,7 @@
 class MyObfuscate
   class SqlServer
     include MyObfuscate::InsertStatementParser
+    include MyObfuscate::ConfigScaffoldGenerator
 
     def parse_insert_statement(line)
       if regex_match = insert_regex.match(line)
@@ -26,7 +27,7 @@ class MyObfuscate
       end
     end
 
-    def make_insert_statement(table_name, column_names, values)
+    def make_insert_statement(table_name, column_names, values, ignore = nil)
       values_strings = values.collect do |values|
         "(" + values.join(",") + ")"
       end.join(",")

data/lib/my_obfuscate/version.rb

@@ -1,3 +1,3 @@
 class MyObfuscate
-  VERSION = "0.5.3"
+  VERSION = "0.6"
 end

data/lib/my_obfuscate.rb

@@ -1,12 +1,12 @@
 require 'jcode' if RUBY_VERSION < '1.9'
 require 'digest/md5'
-require 'ffaker'
+require 'faker'
 require 'walker_method'
 
 # Class for obfuscating MySQL dumps. This can parse mysqldump outputs when using the -c option, which includes
 # column names in the insert statements.
 class MyObfuscate
-  attr_accessor :config, :globally_kept_columns, :fail_on_unspecified_columns, :database_type
+  attr_accessor :config, :globally_kept_columns, :fail_on_unspecified_columns, :database_type, :scaffolded_tables
 
   NUMBER_CHARS = "1234567890"
   USERNAME_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_" + NUMBER_CHARS
@@ -16,6 +16,7 @@ class MyObfuscate
   # performed.  See the README.rdoc file for more information.
   def initialize(configuration = {})
     @config = configuration
+    @scaffolded_tables = {}
   end
 
   def fail_on_unspecified_columns?
@@ -42,19 +43,29 @@ class MyObfuscate
     database_helper.parse(self, config, input_io, output_io)
   end
 
-  def reassembling_each_insert(line, table_name, columns)
+  # Read an input stream and dump out a config file scaffold.  These streams could be StringIO objects, Files,
+  # or STDIN and STDOUT.
+  def scaffold(input_io, output_io)
+    database_helper.generate_config(self, config, input_io, output_io)
+  end
+
+  def reassembling_each_insert(line, table_name, columns, ignore = nil)
     output = database_helper.rows_to_be_inserted(line).map do |sub_insert|
       result = yield(sub_insert)
       result = result.map do |i|
         database_helper.make_valid_value_string(i)
       end
     end
-    database_helper.make_insert_statement(table_name, columns, output)
+    database_helper.make_insert_statement(table_name, columns, output, ignore)
+  end
+
+  def extra_column_list(table_name, columns)
+    config_columns = (config[table_name] || {}).keys
+    config_columns - columns
   end
 
   def check_for_defined_columns_not_in_table(table_name, columns)
-    return unless config[table_name]
-    missing_columns = config[table_name].keys - columns
+    missing_columns = extra_column_list(table_name, columns)
     unless missing_columns.length == 0
       error_message = missing_columns.map do |missing_column|
         "Column '#{missing_column}' could not be found in table '#{table_name}', please fix your obfuscator config."
@@ -63,8 +74,13 @@ class MyObfuscate
     end
   end
 
+  def missing_column_list(table_name, columns)
+    config_columns = (config[table_name] || {}).keys
+    columns - (config_columns + (globally_kept_columns || []).map {|i| i.to_sym}).uniq
+  end
+
   def check_for_table_columns_not_in_definition(table_name, columns)
-    missing_columns = columns - (config[table_name].keys + (globally_kept_columns || []).map {|i| i.to_sym}).uniq
+    missing_columns = missing_column_list(table_name, columns)
     unless missing_columns.length == 0
       error_message = missing_columns.map do |missing_column|
         "Column '#{missing_column}' defined in table '#{table_name}', but not found in table definition, please fix your obfuscator config."
@@ -73,7 +89,7 @@ class MyObfuscate
     end
   end
 
-  def obfuscate_bulk_insert_line(line, table_name, columns)
+  def obfuscate_bulk_insert_line(line, table_name, columns, ignore = nil)
     table_config = config[table_name]
     if table_config == :truncate
       ""
@@ -83,7 +99,7 @@ class MyObfuscate
       check_for_defined_columns_not_in_table(table_name, columns)
       check_for_table_columns_not_in_definition(table_name, columns) if fail_on_unspecified_columns?
       # Note: Remember to SQL escape strings in what you pass back.
-      reassembling_each_insert(line, table_name, columns) do |row|
+      reassembling_each_insert(line, table_name, columns, ignore) do |row|
         ConfigApplicator.apply_table_config(row, table_config, columns)
       end
     end
@@ -93,6 +109,7 @@ end
 
 require 'my_obfuscate/copy_statement_parser'
 require 'my_obfuscate/insert_statement_parser'
+require 'my_obfuscate/config_scaffold_generator'
 require 'my_obfuscate/mysql'
 require 'my_obfuscate/sql_server'
 require 'my_obfuscate/postgres'

data/mise.toml

@@ -0,0 +1,2 @@
+[tools]
+ruby = "3.4.7"

data/my_obfuscate.gemspec

@@ -5,19 +5,19 @@ require "my_obfuscate/version"
 Gem::Specification.new do |s|
   s.name = %q{my_obfuscate}
   s.version = MyObfuscate::VERSION
+  s.required_ruby_version = ">= 3"
+  s.license = "MIT"
 
   s.authors = ["Andrew Cantino", "Dave Willett", "Mike Grafton", "Mason Glaves", "Greg Bell", "Mavenlink"]
   s.description = %q{Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.}
-  s.email = %q{andrew@mavenlink.com}
-  s.homepage = %q{http://github.com/mavenlink/my_obfuscate}
+  s.homepage = %q{https://github.com/mavenlink/my_obfuscate}
   s.summary = %q{Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.}
 
-  s.add_dependency "ffaker"
+  s.add_dependency "faker"
   s.add_dependency "walker_method"
   s.add_development_dependency "rspec"
 
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.files = `git ls-files -z`.split("\x0")
+  s.executables = `git ls-files -z -- bin/*`.split("\x0").map { |f| File.basename(f) }
   s.require_paths = ["lib"]
 end

data/spec/my_obfuscate/config_applicator_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'uri'
 
 describe MyObfuscate::ConfigApplicator do
 
@@ -7,7 +8,7 @@ describe MyObfuscate::ConfigApplicator do
       100.times do
         new_row = MyObfuscate::ConfigApplicator.apply_table_config(["blah", "something_else"], {:a => {:type => :email}}, [:a, :b])
         expect(new_row.length).to eq(2)
-        expect(new_row.first).to match(/^[\w\.]+\@(\w+\.){2,3}[a-f0-9]{5}\.example\.com$/)
+        expect(new_row.first).to match(URI::MailTo::EMAIL_REGEXP)
       end
     end
 

data/spec/my_obfuscate/mysql_spec.rb

@@ -73,7 +73,14 @@ describe MyObfuscate::Mysql do
 
     it "should return a hash of table name, column names for MySQL insert statements" do
       hash = subject.parse_insert_statement("INSERT INTO `some_table` (`email`, `name`, `something`, `age`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54);")
-      expect(hash).to eq({:table_name => :some_table, :column_names => [:email, :name, :something, :age]})
+      expect(hash).to eq({:ignore => false, :table_name => :some_table, :column_names => [:email, :name, :something, :age]})
+    end
+  end
+
+  describe "#parse_insert_ignore_statement" do
+    it "should return a hash of IGNORE, table name, column names for MySQL insert statements" do
+      hash = subject.parse_insert_statement("INSERT IGNORE INTO `some_table` (`email`, `name`, `something`, `age`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54);")
+      expect(hash).to eq({:ignore => true, :table_name => :some_table, :column_names => [:email, :name, :something, :age]})
     end
   end
 

data/spec/my_obfuscate_spec.rb

@@ -73,6 +73,31 @@ COPY some_table_to_keep (a, b) FROM stdin;
         output.read
       end
 
+      let(:scaffolder) do
+        MyObfuscate.new({
+            :some_other_table => {
+                :email => {:type => :email, :skip_regexes => [/^[\w\.\_]+@honk\.com$/i, /^dontmurderme@direwolf.com$/]},
+                :name => {:type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS},
+                :age => {:type => :integer, :between => 10...80, :unless => :nil },
+            },
+            :single_column_table => {
+                :id => {:type => :integer, :between => 2..9, :unless => :nil}
+            },
+            :another_table => :truncate,
+            :some_table_to_keep => :keep
+        }).tap do |scaffolder|
+          scaffolder.database_type = :postgres
+          scaffolder.globally_kept_columns = %w[age]
+        end
+      end
+
+      let(:scaffold_output_string) do
+        output = StringIO.new
+        scaffolder.scaffold(dump, output)
+        output.rewind
+        output.read
+      end
+
       it "is able to obfuscate single column tables" do
         expect(output_string).not_to include("1\n2\n")
         expect(output_string).to match(/\d\n\d\n/)
@@ -106,7 +131,30 @@ COPY some_table_to_keep (a, b) FROM stdin;
           expect { output_string }.to raise_error RuntimeError
         end
       end
+
+      it "when there is no existing config, should scaffold all the columns that are not globally kept" do
+        expect(scaffold_output_string).to match(/:email\s+=>\s+:keep.+scaffold/)
+        expect(scaffold_output_string).to match(/:name\s+=>\s+:keep.+scaffold/)
+      end
+
+      it "should not scaffold a columns that is globally kept" do
+        expect(scaffold_output_string).not_to match(/:age\s+=>\s+:keep.+scaffold/)
+      end
+
+      context "when dump contains a '.' at the end of the line" do
+        let(:dump) do
+          StringIO.new(<<-SQL)
+  COPY another_table (a, b, c, d) FROM stdin;
+  1	2	3	4
+  1	2	3	.
+  \.
+          SQL
+      end
+      it "should not fail if a insert statement ends in a '.''" do
+        expect(output_string).not_to match(/1\t2\t3\t\./)
+      end
     end
+  end
 
     describe "when using MySQL" do
       context "when there is nothing to obfuscate" do
@@ -143,6 +191,7 @@ COPY some_table_to_keep (a, b) FROM stdin;
         end
       end
 
+
       context "when there is something to obfuscate" do
         before do
           @database_dump = StringIO.new(<<-SQL)
@@ -164,7 +213,7 @@ COPY some_table_to_keep (a, b) FROM stdin;
                                      :one_more_table => {
                                          # Note: fixed strings must be pre-SQL escaped!
                                          :password => {:type => :fixed, :string => "monkey"},
-                                         :c => {:type => :null}
+                                         :c => {:type => :null},
                                      }
                                  })
           @output = StringIO.new
@@ -237,6 +286,189 @@ COPY some_table_to_keep (a, b) FROM stdin;
           }.not_to raise_error
         end
       end
+
+      context "when there is an existing config to scaffold" do
+        before do
+          @database_dump = StringIO.new(<<-SQL)
+          INSERT IGNORE INTO `some_table` (`email`, `name`, `something`, `age`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54);
+          SQL
+          @ddo = MyObfuscate.new({
+                                     :some_table => {
+                                         :email => {:type => :email, :honk_email_skip => true},
+                                         :name => {:type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS}
+                                     },
+                                     :another_table => :truncate
+                                 })
+          @ddo.globally_kept_columns = %w[something]
+          @output = StringIO.new
+          @ddo.scaffold(@database_dump, @output)
+          @output.rewind
+          @output_string = @output.read
+        end
+
+        it "should scaffold missing columns" do
+          expect(@output_string).to match(/:age\s+=>\s+:keep.+scaffold/)
+        end
+
+        it "should not scaffold globally_kept_columns" do
+          expect(@output_string).not_to match(/:something\s+=>\s+:keep.+scaffold/)
+        end
+
+        it "should pass through correct columns" do
+          expect(@output_string).not_to match(/:email\s+=>\s+:keep.+scaffold/)
+          expect(@output_string).to match(/:email\s+=>/)
+          expect(@output_string).not_to match(/\#\s*:email/)
+        end
+      end
+
+      context "when using :secondary_address" do
+        before do
+          @database_dump = StringIO.new(<<-SQL)
+          INSERT INTO `some_table` (`email`, `name`, `something`, `age`, `address1`, `address2`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25, '221B Baker St', 'Suite 100'),('joe@joe.com','joe', 'somethingelse2', 54, '1300 Pennsylvania Ave', '2nd floor');
+          SQL
+          @ddo = MyObfuscate.new({
+                                     :some_table => {
+                                         :email => {:type => :email, :honk_email_skip => true},
+                                         :name => {:type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS},
+                                         :something => :keep,
+                                         :age => :keep,
+                                         :address1 => :street_address,
+                                         :address2 => :secondary_address
+                                     }})
+          @output = StringIO.new
+          @ddo.obfuscate(@database_dump, @output)
+          @output.rewind
+          @output_string = @output.read
+        end
+
+        it "should obfuscate address1" do
+          expect(@output_string).to include("address1")
+          expect(@output_string).not_to include("Baker St")
+        end
+
+        it "should obfuscate address2" do
+          expect(@output_string).to include("address2")
+          expect(@output_string).not_to include("Suite 100")
+        end
+      end
+
+      context "when there is an existing config to scaffold" do
+        before do
+          @database_dump = StringIO.new(<<-SQL)
+          INSERT INTO `some_table` (`email`, `name`, `something`, `age`, `address1`, `address2`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25, '221B Baker St', 'Suite 100'),('joe@joe.com','joe', 'somethingelse2', 54, '1300 Pennsylvania Ave', '2nd floor');
+          SQL
+          @ddo = MyObfuscate.new({
+                                     :some_table => {
+                                         :email => {:type => :email, :honk_email_skip => true},
+                                         :name => {:type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS},
+                                         :something => :keep,
+                                         :age => :keep,
+                                         :gender => {:type => :fixed, :string => "m"},
+                                         :address1 => :street_address,
+                                         :address2 => :secondary_address
+                                     }})
+          @output = StringIO.new
+          @ddo.scaffold(@database_dump, @output)
+          @output.rewind
+          @output_string = @output.read
+        end
+
+        it "should enumerate extra columns" do
+          expect(@output_string).to match(/\#\s*:gender\s+=>\s+\{type:\s+:fixed,\s*string:.*#\s*unreferenced/)
+        end
+
+        it "should pass through existing columns" do
+          expect(@output_string).to match(/:age\s+=>\s+:keep\s*,/)
+          expect(@output_string).to match(/:address2\s+=>\s*:secondary_address/)
+        end
+      end
+
+      context "when there is an existing config to scaffold with both missing and extra columns" do
+        before do
+          @database_dump = StringIO.new(<<-SQL)
+          INSERT IGNORE INTO `some_table` (`email`, `name`, `something`, `age`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54);
+          SQL
+          @ddo = MyObfuscate.new({
+                                     :some_table => {
+                                         :email => {:type => :email, :honk_email_skip => true},
+                                         :name => {:type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS},
+                                         :gender => {:type => :fixed, :string => "m"}
+                                     }})
+          @output = StringIO.new
+          @ddo.scaffold(@database_dump, @output)
+          @output.rewind
+          @output_string = @output.read
+        end
+
+        it "should scaffold missing columns" do
+          expect(@output_string).to match(/:age\s+=>\s+:keep.+scaffold/)
+          expect(@output_string).to match(/:something\s+=>\s+:keep.+scaffold/)
+        end
+
+        it "should enumerate extra columns" do
+          expect(@output_string).to match(/\#\s*:gender/)
+        end
+      end
+
+      context "when there is an existing config to scaffold and it is just right" do
+        before do
+          @database_dump = StringIO.new(<<-SQL)
+          INSERT INTO `some_table` (`email`, `name`, `something`, `age`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54);
+          SQL
+          @ddo = MyObfuscate.new({
+                                     :some_table => {
+                                         :email => {:type => :email, :honk_email_skip => true},
+                                         :name => {:type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS},
+                                         :something => :keep,
+                                         :age => :keep
+                                     }})
+          @output = StringIO.new
+          @ddo.scaffold(@database_dump, @output)
+          @output.rewind
+          @output_string = @output.read
+        end
+
+        it "should say that everything is present and accounted for" do
+          expect(@output_string).to match(/^\s*\#.*account/)
+          expect(@output_string).not_to include("scaffold")
+          expect(@output_string).not_to include(":some_table")
+        end
+      end
+
+      context "when scaffolding a table with no existing config" do
+        before do
+          @database_dump = StringIO.new(<<-SQL)
+          INSERT INTO `some_table` (`email`, `name`, `something`, `age_of_the_individual_who_is_specified_by_this_row_of_the_table`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54);
+          SQL
+          @ddo = MyObfuscate.new({
+                                     :some_other_table => {
+                                         :email => {:type => :email, :honk_email_skip => true},
+                                         :name => {:type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS},
+                                         :something => :keep,
+                                         :age_of_the_individual_who_is_specified_by_this_row_of_the_table => :keep
+                                     }})
+          @ddo.globally_kept_columns = %w[name]
+
+          @output = StringIO.new
+          @ddo.scaffold(@database_dump, @output)
+          @output.rewind
+          @output_string = @output.read
+        end
+
+        it "should scaffold all the columns that are not globally kept" do
+          expect(@output_string).to match(/:email\s+=>\s+:keep.+scaffold/)
+          expect(@output_string).to match(/:something\s+=>\s+:keep.+scaffold/)
+        end
+
+        it "should not scaffold globally kept columns" do
+          expect(@output_string).not_to match(/:name\s+=>\s+:keep.+scaffold/)
+        end
+
+        it "should preserve long column names" do
+          expect(@output_string).to match(/:age_of_the_individual_who_is_specified_by_this_row_of_the_table/)
+        end
+
+      end
     end
 
     describe "when using MS SQL Server" do
@@ -389,6 +621,148 @@ COPY some_table_to_keep (a, b) FROM stdin;
           }.not_to raise_error
         end
       end
+
+      context "when there is an existing config to scaffold and it is missing columns" do
+        before do
+          @database_dump = StringIO.new(<<-SQL)
+          INSERT [dbo].[some_table] ([email], [name], [something], [age]) VALUES ('bob@honk.com','bob', 'some''thin,ge())lse1', 25);
+          SQL
+          @ddo = MyObfuscate.new({
+                                     :some_table => {
+                                         :email => {:type => :email, :honk_email_skip => true},
+                                         :name => {:type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS}
+                                     }})
+          @ddo.database_type = :sql_server
+          @ddo.globally_kept_columns = %w[something]
+          @output = StringIO.new
+          @ddo.scaffold(@database_dump, @output)
+          @output.rewind
+          @output_string = @output.read
+        end
+
+        it "should scaffold columns that can't be found" do
+          expect(@output_string).to match(/:age\s+=>\s+:keep.+scaffold/)
+        end
+
+        it "should not scaffold globally_kept_columns" do
+          expect(@output_string).not_to match(/:something\s+=>\s+:keep.+scaffold/)
+        end
+      end
+
+      context "when there is an existing config to scaffold and it has extra columns" do
+        before do
+          @database_dump = StringIO.new(<<-SQL)
+          INSERT [dbo].[some_table] ([email], [name], [something], [age]) VALUES ('bob@honk.com','bob', 'some''thin,ge())lse1', 25);
+          SQL
+          @ddo = MyObfuscate.new({
+                                     :some_table => {
+                                         :email => {:type => :email, :honk_email_skip => true},
+                                         :name => {:type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS},
+                                         :something => :keep,
+                                         :age => :keep,
+                                         :gender => {:type => :fixed, :string => "m"}
+                                     }})
+          @ddo.database_type = :sql_server
+
+          @output = StringIO.new
+          @ddo.scaffold(@database_dump, @output)
+          @output.rewind
+          @output_string = @output.read
+        end
+
+        it "should enumerate extra columns" do
+          expect(@output_string).to match(/\#\s*:gender/)
+        end
+      end
+
+      context "when there is an existing config to scaffold and it has both missing and extra columns" do
+        before do
+          @database_dump = StringIO.new(<<-SQL)
+          INSERT [dbo].[some_table] ([email], [name], [something], [age]) VALUES ('bob@honk.com','bob', 'some''thin,ge())lse1', 25);
+          SQL
+          @ddo = MyObfuscate.new({
+                                     :some_table => {
+                                         :email => {:type => :email, :honk_email_skip => true},
+                                         :name => {:type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS},
+                                         :gender => {:type => :fixed, :string => "m"}
+                                     }})
+          @ddo.database_type = :sql_server
+
+          @output = StringIO.new
+          @ddo.scaffold(@database_dump, @output)
+          @output.rewind
+          @output_string = @output.read
+        end
+
+        it "should scaffold columns that can't be found" do
+          expect(@output_string).to match(/:age\s+=>\s+:keep.+scaffold/)
+          expect(@output_string).to match(/:something\s+=>\s+:keep.+scaffold/)
+        end
+
+        it "should enumerate extra columns" do
+          expect(@output_string).to match(/\#\s*:gender/)
+        end
+      end
+
+      context "when there is an existing config to scaffold and it is just right" do
+        before do
+          @database_dump = StringIO.new(<<-SQL)
+          INSERT [dbo].[some_table] ([email], [name], [something], [age]) VALUES ('bob@honk.com','bob', 'some''thin,ge())lse1', 25);
+          SQL
+          @ddo = MyObfuscate.new({
+                                     :some_table => {
+                                         :email => {:type => :email, :honk_email_skip => true},
+                                         :name => {:type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS},
+                                         :something => :keep,
+                                         :age => :keep
+                                     }})
+          @ddo.database_type = :sql_server
+
+          @output = StringIO.new
+          @ddo.scaffold(@database_dump, @output)
+          @output.rewind
+          @output_string = @output.read
+        end
+
+        it "should say that everything is present and accounted for" do
+          expect(@output_string).to match(/^\s*\#.*account/)
+          expect(@output_string).not_to include("scaffold")
+          expect(@output_string).not_to include(":some_table")
+        end
+      end
+
+      context "when scaffolding a table with no existing config" do
+        before do
+          @database_dump = StringIO.new(<<-SQL)
+          INSERT [dbo].[some_table] ([email], [name], [something], [age]) VALUES ('bob@honk.com','bob', 'some''thin,ge())lse1', 25);
+          SQL
+          @ddo = MyObfuscate.new({
+                                     :some_other_table => {
+                                         :email => {:type => :email, :honk_email_skip => true},
+                                         :name => {:type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS},
+                                         :something => :keep,
+                                         :age => :keep
+                                     }})
+          @ddo.database_type = :sql_server
+          @ddo.globally_kept_columns = %w[age]
+
+          @output = StringIO.new
+          @ddo.scaffold(@database_dump, @output)
+          @output.rewind
+          @output_string = @output.read
+        end
+
+        it "should scaffold all the columns that are not globally kept" do
+          expect(@output_string).to match(/:email\s+=>\s+:keep.+scaffold/)
+          expect(@output_string).to match(/:name\s+=>\s+:keep.+scaffold/)
+          expect(@output_string).to match(/:something\s+=>\s+:keep.+scaffold/)
+        end
+
+        it "should not scaffold globally kept columns" do
+          expect(@output_string).not_to match(/:age\s+=>\s+:keep.+scaffold/)
+        end
+      end
+
     end
   end
 

data/spec/spec_helper.rb

@@ -6,5 +6,5 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'my_obfuscate'
 
 RSpec.configure do |config|
-#  config.mock_with :rr
+ config.mock_with :rspec
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: my_obfuscate
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: '0.6'
 platform: ruby
 authors:
 - Andrew Cantino
@@ -10,13 +10,12 @@ authors:
 - Mason Glaves
 - Greg Bell
 - Mavenlink
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-24 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: ffaker
+  name: faker
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -59,7 +58,6 @@ dependencies:
         version: '0'
 description: Standalone Ruby code for the selective rewriting of MySQL dumps in order
   to protect user privacy.
-email: andrew@mavenlink.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -75,6 +73,7 @@ files:
 - Rakefile
 - lib/my_obfuscate.rb
 - lib/my_obfuscate/config_applicator.rb
+- lib/my_obfuscate/config_scaffold_generator.rb
 - lib/my_obfuscate/copy_statement_parser.rb
 - lib/my_obfuscate/data/en_50K.txt
 - lib/my_obfuscate/insert_statement_parser.rb
@@ -82,6 +81,7 @@ files:
 - lib/my_obfuscate/postgres.rb
 - lib/my_obfuscate/sql_server.rb
 - lib/my_obfuscate/version.rb
+- mise.toml
 - my_obfuscate.gemspec
 - spec/my_obfuscate/config_applicator_spec.rb
 - spec/my_obfuscate/mysql_spec.rb
@@ -89,10 +89,10 @@ files:
 - spec/my_obfuscate/sql_server_spec.rb
 - spec/my_obfuscate_spec.rb
 - spec/spec_helper.rb
-homepage: http://github.com/mavenlink/my_obfuscate
-licenses: []
+homepage: https://github.com/mavenlink/my_obfuscate
+licenses:
+- MIT
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -100,23 +100,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Standalone Ruby code for the selective rewriting of MySQL dumps in order
   to protect user privacy.
-test_files:
-- spec/my_obfuscate/config_applicator_spec.rb
-- spec/my_obfuscate/mysql_spec.rb
-- spec/my_obfuscate/postgres_spec.rb
-- spec/my_obfuscate/sql_server_spec.rb
-- spec/my_obfuscate_spec.rb
-- spec/spec_helper.rb
+test_files: []