my_obfuscate 0.3.0 → 0.3.7

data/.gitignore

@@ -4,3 +4,7 @@ coverage
 rdoc
 pkg
 .idea
+.rvmrc
+Gemfile.lock
+*.deb
+*.gem

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in my_obfuscate.gemspec
+gemspec

data/README.rdoc

@@ -1,10 +1,10 @@
 = my_obfuscate
 
-Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.
+Standalone Ruby code for the selective rewriting of SQL dumps in order to protect user privacy.  Supports MySQL and SQL Server.
 
 = Install
 
-  sudo gem install my_obfuscate
+  (sudo) gem install my_obfuscate
 
 = Example Usage
 
@@ -17,34 +17,65 @@ Make an obfuscator.rb script:
   obfuscator = MyObfuscate.new({
     :people => {
       :email                     => { :type => :email, :skip_regexes => [/^[\w\.\_]+@my_company\.com$/i] },
-      :ethnicity                 => { :type => :null },
+      :ethnicity                 => :keep,
       :crypted_password          => { :type => :fixed, :string => "SOME_FIXED_PASSWORD_FOR_EASE_OF_DEBUGGING" },
       :salt                      => { :type => :fixed, :string => "SOME_THING" },
-      :remember_token            => { :type => :null },
-      :remember_token_expires_at => { :type => :null },
-      :photo_file_name           => { :type => :null },
-      :photo_content_type        => { :type => :null },
-      :photo_file_size           => { :type => :null },
-      :photo_updated_at          => { :type => :null },
+      :remember_token            => :null,
+      :remember_token_expires_at => :null,
+      :age                       => { :type => :null, :unless => lambda { |person| person[:email] == "hello@example.com" } },
+      :photo_file_name           => :null,
+      :photo_content_type        => :null,
+      :photo_file_size           => :null,
+      :photo_updated_at          => :null,
       :postal_code               => { :type => :fixed, :string => "94109", :unless => lambda {|person| person[:postal_code] == "12345"} },
-      :name                      => { :type => :fixed, :string => "Production User", :if => lambda {|person| person[:email] == "hello@example.com"} },
+      :name                      => :name,
+      :full_address              => :address,
+      :bio                       => { :type => :lorem, :number => 4 },
       :relationship_status       => { :type => :fixed, :one_of => ["Single", "Divorced", "Married", "Engaged", "In a Relationship"] },
       :has_children              => { :type => :integer, :between => 0..1 },
     },
 
     :invites                     => :truncate,
     :invite_requests             => :truncate,
+    :tags                        => :keep,
 
     :relationships => {
-      :account_id                => { :type => :string, :length => 8, :chars => MyObfuscate::NUMBER_CHARS },
+      :account_id                => :keep,
       :code                      => { :type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS }
     }
   })
+  obfuscator.fail_on_unspecified_columns = true # if you want it to require every column in the table to be in the above definition
+  obfuscator.globally_kept_columns = %w[id created_at updated_at] # if you set fail_on_unspecified_columns, you may want this as well
   obfuscator.obfuscate(STDIN, STDOUT)
 
 And to get an obfuscated dump:
-  mysqldump -c --add-drop-table -u user -ppassword database | ruby obfuscator.rb > obfuscated_dump.sql
-Note that the -c option on mysqldump is required to use my_obfuscator.
+
+    mysqldump -c --add-drop-table --hex-blob -u user -ppassword database | ruby obfuscator.rb > obfuscated_dump.sql
+
+Note that the -c option on mysqldump is required to use my_obfuscator.  Additionally, the default behavior of mysqldump 
+is to output special characters. This may cause trouble, so you can request hex-encoded blob content with --hex-blob.  
+If you get MySQL errors due to very long lines, try some combination of --max_allowed_packet=128M, --single-transaction, --skip-extended-insert, and --quick.
+
+== Database Server
+
+By default the database type is assumed to be MySQL, but you can use the
+builtin SQL Server support by specifying:
+
+    obfuscator.database_type = :sql_server
+
+== Types
+
+Available types include: email, string, lorem, name, first_name, last_name, address, street_address, city, state,
+zip_code, phone, company, ipv4, ipv6, url, integer, fixed, null, and keep.
+
+== Changes
+
+* Support for SQL Server
+* :unless and :if now support :nil as a shorthand for a Proc that checks for nil
+* :name, :lorem, and :address are all now supported types.  You can pass :number to :lorem to specify how many sentences to generate.  The default is one.
+* <tt>{ :type => :whatever }</tt> is now optional when no additional options are needed.  Just use <tt>:whatever</tt>.
+* Warnings are thrown when an unknown column type or table is encountered.  Use <tt>:keep</tt> in both cases.
+* <tt>{ :type => :fixed, :string => Proc { |row| ... } }</tt> is now available.
 
 == Note on Patches/Pull Requests
 
@@ -54,6 +85,10 @@ Note that the -c option on mysqldump is required to use my_obfuscator.
 * Commit, do not mess with rakefile, version, or history.  (If you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
 * Send me a pull request. Bonus points for topic branches.
 
+== Thanks
+
+Thanks to Mavenlink and Pivotal Labs for patches and updates!
+
 == Copyright
 
-Copyright (c) 2009 Honk. See LICENSE for details.
+Copyright (c) 2009 Honk. Now maintained by Iteration Labs, LLC. See LICENSE for details.

data/Rakefile

@@ -1,60 +1,8 @@
-require 'rubygems'
-require 'rake'
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
-
-# When updating:
-#  rake version:bump:minor
-#  rake gemspec
-#  rake build
-#  rake rubyforge:release
-# Then git checkin and commit
-
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "my_obfuscate"
-    gem.summary = %Q{Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.}
-    gem.description = %Q{Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.}
-    gem.email = "andrew@pivotallabs.com"
-    gem.homepage = "http://github.com/honkster/myobfuscate"
-    gem.authors = ["Andrew Cantino", "Dave Willett", "Mike Grafton", "Mason Glaves"]
-    gem.add_development_dependency "rspec"
-    gem.rubyforge_project = 'my-obfuscate'
-  end
-
-  Jeweler::RubyforgeTasks.new do |rubyforge|
-    rubyforge.doc_task = "rdoc"
-  end
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
 end
 
-require 'spec/rake/spectask'
-Spec::Rake::SpecTask.new(:spec) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.spec_files = FileList['spec/**/*_spec.rb']
-end
-
-Spec::Rake::SpecTask.new(:rcov) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rcov = true
-end
-
-task :spec => :check_dependencies
-
 task :default => :spec
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  if File.exist?('VERSION')
-    version = File.read('VERSION')
-  else
-    version = ""
-  end
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "my_obfuscate #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end

data/lib/my_obfuscate.rb

@@ -1,11 +1,11 @@
-require 'jcode'
+require 'jcode' if RUBY_VERSION < '1.9'
+require 'faker'
 
 # Class for obfuscating MySQL dumps. This can parse mysqldump outputs when using the -c option, which includes
 # column names in the insert statements.
 class MyObfuscate
-  attr_accessor :config
+  attr_accessor :config, :globally_kept_columns, :fail_on_unspecified_columns, :database_type
 
-  INSERT_REGEX = /^\s*INSERT INTO `(.*?)` \((.*?)\) VALUES\s*/i
   NUMBER_CHARS = "1234567890"
   USERNAME_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_" + NUMBER_CHARS
   SENSIBLE_CHARS = USERNAME_CHARS + '+-=[{]}/?|!@#$%^&*()`~'
@@ -16,17 +16,35 @@ class MyObfuscate
     @config = configuration
   end
 
+  def fail_on_unspecified_columns?
+    @fail_on_unspecified_columns
+  end
+
+  def database_helper
+    if @database_helper.nil?
+      if @database_type == :sql_server
+        @database_helper = SqlServer.new
+      else
+        @database_helper = Mysql.new
+      end
+    end
+
+    @database_helper
+  end
+
   # Read an input stream and dump out an obfuscated output stream.  These streams could be StringIO objects, Files,
   # or STDIN and STDOUT.
   def obfuscate(input_io, output_io)
+
     # We assume that every INSERT INTO line occupies one line in the file, with no internal linebreaks.
     input_io.each do |line|
-      if regex_result = INSERT_REGEX.match(line)
-        table_name = regex_result[1].to_sym
-        columns = regex_result[2].split(/`\s*,\s*`/).map { |col| col.gsub('`',"").to_sym }
+      if table_data = database_helper.parse_insert_statement(line)
+        table_name = table_data[:table_name]
+        columns = table_data[:column_names]
         if config[table_name]
           output_io.puts obfuscate_bulk_insert_line(line, table_name, columns)
         else
+          $stderr.puts "Deprecated: #{table_name} was not specified in the config.  A future release will cause this to be an error.  Please specify the table definition or set it to :keep."
           output_io.write line
         end
       else
@@ -35,93 +53,53 @@ class MyObfuscate
     end
   end
 
-  def self.reasembling_each_insert(line, table_name, columns)
-    line = line.gsub(INSERT_REGEX, '').gsub(/\s*;\s*$/, '')
-    output = context_aware_mysql_string_split(line).map do |sub_insert|
+  def reassembling_each_insert(line, table_name, columns)
+    output = database_helper.rows_to_be_inserted(line).map do |sub_insert|
       result = yield(sub_insert)
       result = result.map do |i|
-        if i.nil?
-          "NULL"
-        else
-          "'" + i + "'"
-        end
+        database_helper.make_valid_value_string(i)
       end
       result = result.join(",")
       "(" + result + ")"
     end.join(",")
-    "INSERT INTO `#{table_name}` (`#{columns.join('`, `')}`) VALUES #{output};"
-  end
-
-  # Be aware, strings must be quoted in single quotes!
-  def self.context_aware_mysql_string_split(string)
-    in_sub_insert = false
-    in_quoted_string = false
-    escaped = false
-    current_field = nil
-    length = string.length
-    index = 0
-    fields = []
-    output = []
-    string.each_char do |i|
-      if escaped
-        escaped = false
-        current_field ||= ""
-        current_field << i
-      else
-       if i == "\\"
-         escaped = true
-         current_field ||= ""
-         current_field << i
-       elsif i == "(" && !in_quoted_string && !in_sub_insert
-         in_sub_insert = true
-       elsif i == ")" && !in_quoted_string && in_sub_insert
-         fields << current_field unless current_field.nil?
-         output << fields unless fields.length == 0
-         in_sub_insert = false
-         fields = []
-         current_field = nil
-       elsif i == "'" && !in_quoted_string
-         fields << current_field unless current_field.nil?
-         current_field = ''
-         in_quoted_string = true
-       elsif i == "'" && in_quoted_string
-         fields << current_field unless current_field.nil?
-         current_field = nil
-         in_quoted_string = false
-       elsif i == "," && !in_quoted_string && in_sub_insert
-         fields << current_field unless current_field.nil?
-         current_field = nil
-       elsif i == "L" && !in_quoted_string && in_sub_insert && current_field == "NUL"
-         current_field = nil
-         fields << current_field
-       elsif (i == " " || i == "\t") && !in_quoted_string
-         # Don't add whitespace not in a string
-       elsif in_sub_insert
-         current_field ||= ""
-         current_field << i
-       end
-      end
-      index += 1
-    end
-    fields << current_field unless current_field.nil?
-    output << fields unless fields.length == 0
-    output
+    database_helper.make_insert_statement(table_name, columns, output)
   end
 
   def self.row_as_hash(row, columns)
     columns.zip(row).inject({}) {|m, (name, value)| m[name] = value; m}
   end
 
+  def self.make_conditional_method(conditional_method, index, row)
+    if conditional_method.is_a?(Symbol)
+      if conditional_method == :blank
+        conditional_method = lambda { |row_hash| row[index].nil? || row[index] == '' }
+      elsif conditional_method == :nil
+        conditional_method = lambda { |row_hash| row[index].nil? }
+      end
+    end
+    conditional_method
+  end
+
   def self.apply_table_config(row, table_config, columns)
     return row unless table_config.is_a?(Hash)
     row_hash = row_as_hash(row, columns)
 
     table_config.each do |column, definition|
       index = columns.index(column)
+      
+      definition = { :type => definition } if definition.is_a?(Symbol)
+
+      if definition.has_key?(:unless)
+        unless_check = make_conditional_method(definition[:unless], index, row)
 
-      next if definition[:unless] && definition[:unless].call(row_hash)
-      if definition[:if]
-        next unless definition[:if].call(row_hash)
+        next if unless_check.call(row_hash)
+      end
+
+
+      if definition.has_key?(:if)
+        if_check = make_conditional_method(definition[:if], index, row)
+
+        next unless if_check.call(row_hash)
       end
 
       if definition[:skip_regexes]
@@ -130,20 +108,51 @@ class MyObfuscate
 
       row[index.to_i] = case definition[:type]
         when :email
-          random_string(4..10, USERNAME_CHARS) + "@example.com"
+          clean_quotes(Faker::Internet.email)
         when :string
-          random_string(definition[:length], definition[:chars] || SENSIBLE_CHARS)
+          random_string(definition[:length] || 30, definition[:chars] || SENSIBLE_CHARS)
+        when :lorem
+          clean_bad_whitespace(clean_quotes(Faker::Lorem.sentences(definition[:number] || 1).join(".  ")))
+        when :name
+          clean_quotes(Faker::Name.name)
+        when :first_name
+          clean_quotes(Faker::Name.first_name)
+        when :last_name
+          clean_quotes(Faker::Name.last_name)
+        when :address
+          clean_quotes("#{Faker::Address.street_address}\\n#{Faker::Address.city}, #{Faker::Address.state_abbr} #{Faker::Address.zip_code}")
+        when :street_address
+          clean_bad_whitespace(clean_quotes(Faker::Address.street_address))
+        when :city
+          clean_quotes(Faker::Address.city)
+        when :state
+          Faker::Address.state_abbr
+        when :zip_code
+          Faker::Address.zip_code
+        when :phone
+          Faker::PhoneNumber.phone_number
+        when :company
+          clean_bad_whitespace(clean_quotes(Faker::Company.name))
+        when :ipv4
+          Faker::Internet.ip_v4_address
+        when :ipv6
+          Faker::Internet.ip_v6_address
+        when :url
+          clean_bad_whitespace(Faker::Internet.url)
         when :integer
           random_integer(definition[:between] || (0..1000)).to_s
         when :fixed
           if definition[:one_of]
             definition[:one_of][(rand * definition[:one_of].length).to_i]
           else
-            definition[:string]
+            definition[:string].is_a?(Proc) ? definition[:string].call(row_hash) : definition[:string]
           end
         when :null
           nil
+        when :keep
+          row[index]
         else
+          $stderr.puts "Keeping a column value by providing an unknown type (#{definition[:type]}) is deprecated.  Use :keep instead."
           row[index]
       end
     end
@@ -162,26 +171,52 @@ class MyObfuscate
     out
   end
 
-  def check_for_missing_columns(table_name, columns)
+  def check_for_defined_columns_not_in_table(table_name, columns)
     missing_columns = config[table_name].keys - columns
     unless missing_columns.length == 0
       error_message = missing_columns.map do |missing_column|
         "Column '#{missing_column}' could not be found in table '#{table_name}', please fix your obfuscator config."
       end.join("\n")
-       raise RuntimeError.new(error_message)
+      raise RuntimeError.new(error_message)
     end
   end
 
-  def obfuscate_bulk_insert_line (line, table_name, columns)
+  def check_for_table_columns_not_in_definition(table_name, columns)
+    missing_columns = columns - (config[table_name].keys + (globally_kept_columns || []).map {|i| i.to_sym}).uniq
+    unless missing_columns.length == 0
+      error_message = missing_columns.map do |missing_column|
+        "Column '#{missing_column}' defined in table '#{table_name}', but not found in table definition, please fix your obfuscator config."
+      end.join("\n")
+      raise RuntimeError.new(error_message)
+    end
+  end
+
+  def obfuscate_bulk_insert_line(line, table_name, columns)
     table_config = config[table_name]
     if table_config == :truncate
       ""
+    elsif table_config == :keep
+      line
     else
-      check_for_missing_columns(table_name, columns)
+      check_for_defined_columns_not_in_table(table_name, columns)
+      check_for_table_columns_not_in_definition(table_name, columns) if fail_on_unspecified_columns?
       # Note: Remember to SQL escape strings in what you pass back.
-      MyObfuscate.reasembling_each_insert(line, table_name, columns) do |row|
+      reassembling_each_insert(line, table_name, columns) do |row|
         MyObfuscate.apply_table_config(row, table_config, columns)
       end
     end
   end
+  
+  private
+  
+  def self.clean_quotes(value)
+    value.gsub(/['"]/, '')
+  end
+  
+  def self.clean_bad_whitespace(value)
+    value.gsub(/[\n\t\r]/, '')
+  end
 end
+
+require 'my_obfuscate/mysql'
+require 'my_obfuscate/sql_server'