my_obfuscate 0.4.2 → 0.5.0

data/CHANGES

@@ -1 +1,2 @@
+7/17/2013 - Switch Postgres to use COPY statements and refactor internals.  Thanks @samuelreh!
 3/4/2013 - Switch to the ffaker gem for speed.  Add WalkerMethod and an English language frequency dictionary for generating random texts.

data/Gemfile

@@ -2,3 +2,5 @@ source "http://rubygems.org"
 
 # Specify your gem's dependencies in my_obfuscate.gemspec
 gemspec
+
+gem 'rake'

data/README.rdoc

@@ -1,5 +1,7 @@
 = MyObfuscate
 
+{<img src="https://travis-ci.org/mavenlink/my_obfuscate.png">}[https://travis-ci.org/mavenlink/my_obfuscate]
+
 You want to develop against real production data, but you don't want to violate your users' privacy.  Enter MyObfuscate: standalone Ruby code for the selective rewriting of SQL dumps in order to protect user privacy.  It supports MySQL, Postgres, and SQL Server.
 
 = Install
@@ -64,6 +66,10 @@ builtin SQL Server support by specifying:
     obfuscator.database_type = :sql_server
     obfuscator.database_type = :postgres
 
+If using Postgres, use pg_dump to get a dump:
+
+    pg_dump database | ruby obfuscator.rb > obfuscated_dump.sql
+
 == Types
 
 Available types include: email, string, lorem, name, first_name, last_name, address, street_address, city, state,

data/lib/my_obfuscate/config_applicator.rb

@@ -0,0 +1,146 @@
+class MyObfuscate
+  class ConfigApplicator
+
+    def self.apply_table_config(row, table_config, columns)
+      return row unless table_config.is_a?(Hash)
+      row_hash = row_as_hash(row, columns)
+
+      table_config.each do |column, definition|
+        index = columns.index(column)
+
+        definition = { :type => definition } if definition.is_a?(Symbol)
+
+        if definition.has_key?(:unless)
+          unless_check = make_conditional_method(definition[:unless], index, row)
+
+          next if unless_check.call(row_hash)
+        end
+
+
+        if definition.has_key?(:if)
+          if_check = make_conditional_method(definition[:if], index, row)
+
+          next unless if_check.call(row_hash)
+        end
+
+        if definition[:skip_regexes]
+          next if definition[:skip_regexes].any? {|regex| row[index] =~ regex}
+        end
+
+        row[index.to_i] = case definition[:type]
+          when :email
+            md5 = Digest::MD5.hexdigest(rand.to_s)[0...5]
+            clean_quotes("#{Faker::Internet.email}.#{md5}.example.com")
+          when :string
+            random_string(definition[:length] || 30, definition[:chars] || SENSIBLE_CHARS)
+          when :lorem
+            clean_bad_whitespace(clean_quotes(Faker::Lorem.sentences(definition[:number] || 1).join(".  ")))
+          when :like_english
+            clean_quotes random_english_sentences(definition[:number] || 1)
+          when :name
+            clean_quotes(Faker::Name.name)
+          when :first_name
+            clean_quotes(Faker::Name.first_name)
+          when :last_name
+            clean_quotes(Faker::Name.last_name)
+          when :address
+            clean_quotes("#{Faker::AddressUS.street_address}\\n#{Faker::AddressUS.city}, #{Faker::AddressUS.state_abbr} #{Faker::AddressUS.zip_code}")
+          when :street_address
+            clean_bad_whitespace(clean_quotes(Faker::AddressUS.street_address))
+          when :city
+            clean_quotes(Faker::AddressUS.city)
+          when :state
+            clean_quotes Faker::AddressUS.state_abbr
+          when :zip_code
+            Faker::AddressUS.zip_code
+          when :phone
+            clean_quotes Faker::PhoneNumber.phone_number
+          when :company
+            clean_bad_whitespace(clean_quotes(Faker::Company.name))
+          when :ipv4
+            Faker::Internet.ip_v4_address
+          when :ipv6
+            # Inlined from Faker because ffaker doesn't have ipv6.
+            @@ip_v6_space ||= (0..65535).to_a
+            container = (1..8).map{ |_| @@ip_v6_space.sample }
+            container.map{ |n| n.to_s(16) }.join(':')
+          when :url
+            clean_bad_whitespace(Faker::Internet.http_url)
+          when :integer
+            random_integer(definition[:between] || (0..1000)).to_s
+          when :fixed
+            if definition[:one_of]
+              definition[:one_of][(rand * definition[:one_of].length).to_i]
+            else
+              definition[:string].is_a?(Proc) ? definition[:string].call(row_hash) : definition[:string]
+            end
+          when :null
+            nil
+          when :keep
+            row[index]
+          else
+            $stderr.puts "Keeping a column value by providing an unknown type (#{definition[:type]}) is deprecated.  Use :keep instead."
+            row[index]
+        end
+      end
+      row
+    end
+
+    def self.row_as_hash(row, columns)
+      columns.zip(row).inject({}) {|m, (name, value)| m[name] = value; m}
+    end
+
+    def self.make_conditional_method(conditional_method, index, row)
+      if conditional_method.is_a?(Symbol)
+        if conditional_method == :blank
+          conditional_method = lambda { |row_hash| row[index].nil? || row[index] == '' }
+        elsif conditional_method == :nil
+          conditional_method = lambda { |row_hash| row[index].nil? }
+        end
+      end
+      conditional_method
+    end
+
+    def self.random_integer(between)
+      (between.min + (between.max - between.min) * rand).round
+    end
+
+    def self.random_string(length_or_range, chars)
+      length_or_range = (length_or_range..length_or_range) if length_or_range.is_a?(Fixnum)
+      times = random_integer(length_or_range)
+      out = ""
+      times.times { out << chars[rand * chars.length] }
+      out
+    end
+
+    def self.random_english_sentences(num)
+      @@walker_method ||= begin
+        words, counts = [], []
+        File.read(File.expand_path(File.join(File.dirname(__FILE__), 'my_obfuscate', 'data', 'en_50K.txt'))).each_line do |line|
+          word, count = line.split(/\s+/)
+          words << word
+          counts << count.to_i
+        end
+        WalkerMethod.new(words, counts)
+      end
+
+      sentences = []
+      num.times do
+        words = []
+        (3 + rand * 5).to_i.times { words << @@walker_method.random }
+        sentences << words.join(" ") + "."
+        sentences.last[0] = sentences.last[0].upcase
+      end
+      sentences.join(" ")
+    end
+
+    def self.clean_quotes(value)
+      value.gsub(/['"]/, '')
+    end
+
+    def self.clean_bad_whitespace(value)
+      value.gsub(/[\n\t\r]/, '')
+    end
+
+  end
+end

data/lib/my_obfuscate/copy_statement_parser.rb

@@ -0,0 +1,45 @@
+class MyObfuscate
+  module CopyStatementParser
+
+    # Postgres uses COPY statements instead of INSERT and look like:
+    #
+    #   COPY some_table (a, b, c, d) FROM stdin;
+    #   1	2	3	4
+    #   5	6	7	8
+    #   \.
+    #
+    # This requires the parse methods to persist data (table name and
+    # column names) across multiple lines.
+    #
+    def parse(obfuscator, config, input_io, output_io)
+      current_table_name, current_columns = ""
+      inside_copy_statement = false
+
+      input_io.each do |line|
+        if parse_insert_statement(line)
+          raise RuntimeError.new("Cannot obfuscate Postgres dumps containing INSERT statements. Please use COPY statments.")
+        elsif table_data = parse_copy_statement(line)
+          inside_copy_statement = true
+
+          current_table_name = table_data[:table_name]
+          current_columns = table_data[:column_names]
+
+          if !config[current_table_name]
+            $stderr.puts "Deprecated: #{current_table_name} was not specified in the config.  A future release will cause this to be an error.  Please specify the table definition or set it to :keep."
+          end
+
+          output_io.write line
+        elsif line.match /\S*\.\n/
+          inside_copy_statement = false
+
+          output_io.write line
+        elsif inside_copy_statement
+          output_io.puts obfuscator.obfuscate_bulk_insert_line(line, current_table_name, current_columns)
+        else
+          output_io.write line
+        end
+      end
+    end
+
+  end
+end

data/lib/my_obfuscate/insert_statement_parser.rb

@@ -0,0 +1,22 @@
+class MyObfuscate
+  module InsertStatementParser
+
+    def parse(obfuscator, config, input_io, output_io)
+      input_io.each do |line|
+        if table_data = parse_insert_statement(line)
+          table_name = table_data[:table_name]
+          columns = table_data[:column_names]
+          if config[table_name]
+            output_io.puts obfuscator.obfuscate_bulk_insert_line(line, table_name, columns)
+          else
+            $stderr.puts "Deprecated: #{table_name} was not specified in the config.  A future release will cause this to be an error.  Please specify the table definition or set it to :keep."
+            output_io.write line
+          end
+        else
+          output_io.write line
+        end
+      end
+    end
+
+  end
+end

data/lib/my_obfuscate/mysql.rb

@@ -1,6 +1,6 @@
 class MyObfuscate
   class Mysql
-    include MyObfuscate::DatabaseHelperShared
+    include MyObfuscate::InsertStatementParser
 
     def parse_insert_statement(line)
       if regex_match = insert_regex.match(line)
@@ -11,7 +11,11 @@ class MyObfuscate
       end
     end
 
-    def make_insert_statement(table_name, column_names, values_strings)
+    def make_insert_statement(table_name, column_names, values)
+      values_strings = values.collect do |values|
+        "(" + values.join(",") + ")"
+      end.join(",")
+
       "INSERT INTO `#{table_name}` (`#{column_names.join('`, `')}`) VALUES #{values_strings};"
     end
 
@@ -19,5 +23,75 @@ class MyObfuscate
       /^\s*INSERT INTO `(.*?)` \((.*?)\) VALUES\s*/i
     end
 
+    def rows_to_be_inserted(line)
+      line = line.gsub(insert_regex, '').gsub(/\s*;\s*$/, '')
+      context_aware_mysql_string_split(line)
+    end
+
+    def make_valid_value_string(value)
+      if value.nil?
+        "NULL"
+      elsif value =~ /^0x[0-9a-fA-F]+$/
+        value
+      else
+        "'" + value + "'"
+      end
+    end
+
+    # Be aware, strings must be quoted in single quotes!
+    def context_aware_mysql_string_split(string)
+      in_sub_insert = false
+      in_quoted_string = false
+      escaped = false
+      current_field = nil
+      length = string.length
+      fields = []
+      output = []
+
+      string.each_char do |i|
+        if escaped
+          escaped = false
+          current_field ||= ""
+          current_field << i
+        else
+          if i == "\\"
+            escaped = true
+            current_field ||= ""
+            current_field << i
+          elsif i == "(" && !in_quoted_string && !in_sub_insert
+            in_sub_insert = true
+          elsif i == ")" && !in_quoted_string && in_sub_insert
+            fields << current_field unless current_field.nil?
+            output << fields unless fields.length == 0
+            in_sub_insert = false
+            fields = []
+            current_field = nil
+          elsif i == "'" && !in_quoted_string
+            fields << current_field unless current_field.nil?
+            current_field = ''
+            in_quoted_string = true
+          elsif i == "'" && in_quoted_string
+            fields << current_field unless current_field.nil?
+            current_field = nil
+            in_quoted_string = false
+          elsif i == "," && !in_quoted_string && in_sub_insert
+            fields << current_field unless current_field.nil?
+            current_field = nil
+          elsif i == "L" && !in_quoted_string && in_sub_insert && current_field == "NUL"
+            current_field = nil
+            fields << current_field
+          elsif (i == " " || i == "\t") && !in_quoted_string
+            # Don't add whitespace not in a string
+          elsif in_sub_insert
+            current_field ||= ""
+            current_field << i
+          end
+        end
+      end
+
+      fields << current_field unless current_field.nil?
+      output << fields unless fields.length == 0
+      output
+    end
   end
 end

data/lib/my_obfuscate/postgres.rb

@@ -1,9 +1,31 @@
 class MyObfuscate
   class Postgres
-    include MyObfuscate::DatabaseHelperShared
+    include MyObfuscate::CopyStatementParser
 
-    def parse_insert_statement(line)
-      if regex_match = insert_regex.match(line)
+    # Copy statements contain the column values tab seperated like so:
+    #   blah	blah	blah	blah
+    # which we want to turn into:
+    #   [['blah','blah','blah','blah']]
+    #
+    # We wrap it in an array to keep it consistent with MySql bulk
+    # obfuscation (multiple rows per insert statement)
+    def rows_to_be_inserted(line)
+      line.gsub!(/\n$/,"")
+      row = line.split(/\t/)
+
+      row.collect! do |value|
+        if value == "\\N"
+          nil
+        else
+          value
+        end
+      end
+
+      [row]
+    end
+
+    def parse_copy_statement(line)
+      if regex_match = /^\s*COPY (.*?) \((.*?)\) FROM\s*/i.match(line)
         {
             :table_name => regex_match[1].to_sym,
             :column_names => regex_match[2].split(/\s*,\s*/).map(&:to_sym)
@@ -11,12 +33,20 @@ class MyObfuscate
       end
     end
 
-    def make_insert_statement(table_name, column_names, values_strings)
-      "INSERT INTO #{table_name} (#{column_names.join(', ')}) VALUES #{values_strings};"
+    def make_insert_statement(table_name, column_names, values)
+      values.join("\t")
     end
 
-    def insert_regex
-      /^\s*INSERT INTO (.*?) \((.*?)\) VALUES\s*/i
+    def make_valid_value_string(value)
+      if value.nil?
+        "\\N"
+      else
+        value
+      end
+    end
+
+    def parse_insert_statement(line)
+      /^\s*INSERT INTO/i.match(line)
     end
 
   end

data/lib/my_obfuscate/sql_server.rb

@@ -1,5 +1,6 @@
 class MyObfuscate
   class SqlServer
+    include MyObfuscate::InsertStatementParser
 
     def parse_insert_statement(line)
       if regex_match = insert_regex.match(line)
@@ -25,7 +26,11 @@ class MyObfuscate
       end
     end
 
-    def make_insert_statement(table_name, column_names, values_strings)
+    def make_insert_statement(table_name, column_names, values)
+      values_strings = values.collect do |values|
+        "(" + values.join(",") + ")"
+      end.join(",")
+
       "INSERT [dbo].[#{table_name}] ([#{column_names.join("], [")}]) VALUES #{values_strings};"
     end
 

data/lib/my_obfuscate/version.rb

@@ -1,3 +1,3 @@
 class MyObfuscate
-  VERSION = "0.4.2"
+  VERSION = "0.5.0"
 end

data/lib/my_obfuscate.rb

@@ -39,22 +39,7 @@ class MyObfuscate
   # Read an input stream and dump out an obfuscated output stream.  These streams could be StringIO objects, Files,
   # or STDIN and STDOUT.
   def obfuscate(input_io, output_io)
-
-    # We assume that every INSERT INTO line occupies one line in the file, with no internal linebreaks.
-    input_io.each do |line|
-      if table_data = database_helper.parse_insert_statement(line)
-        table_name = table_data[:table_name]
-        columns = table_data[:column_names]
-        if config[table_name]
-          output_io.puts obfuscate_bulk_insert_line(line, table_name, columns)
-        else
-          $stderr.puts "Deprecated: #{table_name} was not specified in the config.  A future release will cause this to be an error.  Please specify the table definition or set it to :keep."
-          output_io.write line
-        end
-      else
-        output_io.write line
-      end
-    end
+    database_helper.parse(self, config, input_io, output_io)
   end
 
   def reassembling_each_insert(line, table_name, columns)
@@ -63,146 +48,12 @@ class MyObfuscate
       result = result.map do |i|
         database_helper.make_valid_value_string(i)
       end
-      result = result.join(",")
-      "(" + result + ")"
-    end.join(",")
-    database_helper.make_insert_statement(table_name, columns, output)
-  end
-
-  def self.row_as_hash(row, columns)
-    columns.zip(row).inject({}) {|m, (name, value)| m[name] = value; m}
-  end
-
-  def self.make_conditional_method(conditional_method, index, row)
-    if conditional_method.is_a?(Symbol)
-      if conditional_method == :blank
-        conditional_method = lambda { |row_hash| row[index].nil? || row[index] == '' }
-      elsif conditional_method == :nil
-        conditional_method = lambda { |row_hash| row[index].nil? }
-      end
     end
-    conditional_method
-  end
-
-  def self.apply_table_config(row, table_config, columns)
-    return row unless table_config.is_a?(Hash)
-    row_hash = row_as_hash(row, columns)
-
-    table_config.each do |column, definition|
-      index = columns.index(column)
-
-      definition = { :type => definition } if definition.is_a?(Symbol)
-
-      if definition.has_key?(:unless)
-        unless_check = make_conditional_method(definition[:unless], index, row)
-
-        next if unless_check.call(row_hash)
-      end
-
-
-      if definition.has_key?(:if)
-        if_check = make_conditional_method(definition[:if], index, row)
-
-        next unless if_check.call(row_hash)
-      end
-
-      if definition[:skip_regexes]
-        next if definition[:skip_regexes].any? {|regex| row[index] =~ regex}
-      end
-
-      row[index.to_i] = case definition[:type]
-        when :email
-          md5 = Digest::MD5.hexdigest(rand.to_s)[0...5]
-          clean_quotes("#{Faker::Internet.email}.#{md5}.example.com")
-        when :string
-          random_string(definition[:length] || 30, definition[:chars] || SENSIBLE_CHARS)
-        when :lorem
-          clean_bad_whitespace(clean_quotes(Faker::Lorem.sentences(definition[:number] || 1).join(".  ")))
-        when :like_english
-          clean_quotes random_english_sentences(definition[:number] || 1)
-        when :name
-          clean_quotes(Faker::Name.name)
-        when :first_name
-          clean_quotes(Faker::Name.first_name)
-        when :last_name
-          clean_quotes(Faker::Name.last_name)
-        when :address
-          clean_quotes("#{Faker::AddressUS.street_address}\\n#{Faker::AddressUS.city}, #{Faker::AddressUS.state_abbr} #{Faker::AddressUS.zip_code}")
-        when :street_address
-          clean_bad_whitespace(clean_quotes(Faker::AddressUS.street_address))
-        when :city
-          clean_quotes(Faker::AddressUS.city)
-        when :state
-          clean_quotes Faker::AddressUS.state_abbr
-        when :zip_code
-          Faker::AddressUS.zip_code
-        when :phone
-          clean_quotes Faker::PhoneNumber.phone_number
-        when :company
-          clean_bad_whitespace(clean_quotes(Faker::Company.name))
-        when :ipv4
-          Faker::Internet.ip_v4_address
-        when :ipv6
-          # Inlined from Faker because ffaker doesn't have ipv6.
-          @@ip_v6_space ||= (0..65535).to_a
-          container = (1..8).map{ |_| @@ip_v6_space.sample }
-          container.map{ |n| n.to_s(16) }.join(':')
-        when :url
-          clean_bad_whitespace(Faker::Internet.http_url)
-        when :integer
-          random_integer(definition[:between] || (0..1000)).to_s
-        when :fixed
-          if definition[:one_of]
-            definition[:one_of][(rand * definition[:one_of].length).to_i]
-          else
-            definition[:string].is_a?(Proc) ? definition[:string].call(row_hash) : definition[:string]
-          end
-        when :null
-          nil
-        when :keep
-          row[index]
-        else
-          $stderr.puts "Keeping a column value by providing an unknown type (#{definition[:type]}) is deprecated.  Use :keep instead."
-          row[index]
-      end
-    end
-    row
-  end
-
-  def self.random_integer(between)
-    (between.min + (between.max - between.min) * rand).round
-  end
-
-  def self.random_string(length_or_range, chars)
-    length_or_range = (length_or_range..length_or_range) if length_or_range.is_a?(Fixnum)
-    times = random_integer(length_or_range)
-    out = ""
-    times.times { out << chars[rand * chars.length] }
-    out
-  end
-
-  def self.random_english_sentences(num)
-    @@walker_method ||= begin
-      words, counts = [], []
-      File.read(File.expand_path(File.join(File.dirname(__FILE__), 'my_obfuscate', 'data', 'en_50K.txt'))).each_line do |line|
-        word, count = line.split(/\s+/)
-        words << word
-        counts << count.to_i
-      end
-      WalkerMethod.new(words, counts)
-    end
-
-    sentences = []
-    num.times do
-      words = []
-      (5 + rand * 6).to_i.times { words << @@walker_method.random }
-      sentences << words.join(" ") + "."
-      sentences.last[0] = sentences.last[0].upcase
-    end
-    sentences.join(" ")
+    database_helper.make_insert_statement(table_name, columns, output)
   end
 
   def check_for_defined_columns_not_in_table(table_name, columns)
+    return unless config[table_name]
     missing_columns = config[table_name].keys - columns
     unless missing_columns.length == 0
       error_message = missing_columns.map do |missing_column|
@@ -233,23 +84,16 @@ class MyObfuscate
       check_for_table_columns_not_in_definition(table_name, columns) if fail_on_unspecified_columns?
       # Note: Remember to SQL escape strings in what you pass back.
       reassembling_each_insert(line, table_name, columns) do |row|
-        MyObfuscate.apply_table_config(row, table_config, columns)
+        ConfigApplicator.apply_table_config(row, table_config, columns)
       end
     end
   end
 
-  private
-
-  def self.clean_quotes(value)
-    value.gsub(/['"]/, '')
-  end
-
-  def self.clean_bad_whitespace(value)
-    value.gsub(/[\n\t\r]/, '')
-  end
 end
 
-require 'my_obfuscate/database_helper_shared'
+require 'my_obfuscate/copy_statement_parser'
+require 'my_obfuscate/insert_statement_parser'
 require 'my_obfuscate/mysql'
 require 'my_obfuscate/sql_server'
 require 'my_obfuscate/postgres'
+require 'my_obfuscate/config_applicator'