my_obfuscate 0.3.7 → 0.4.2

data/lib/my_obfuscate/database_helper_shared.rb

@@ -0,0 +1,76 @@
+class MyObfuscate
+  module DatabaseHelperShared
+
+    def rows_to_be_inserted(line)
+      line = line.gsub(insert_regex, '').gsub(/\s*;\s*$/, '')
+      context_aware_mysql_string_split(line)
+    end
+
+    def make_valid_value_string(value)
+      if value.nil?
+        "NULL"
+      elsif value =~ /^0x[0-9a-fA-F]+$/
+        value
+      else
+        "'" + value + "'"
+      end
+    end
+
+    # Be aware, strings must be quoted in single quotes!
+    def context_aware_mysql_string_split(string)
+      in_sub_insert = false
+      in_quoted_string = false
+      escaped = false
+      current_field = nil
+      length = string.length
+      fields = []
+      output = []
+
+      string.each_char do |i|
+        if escaped
+          escaped = false
+          current_field ||= ""
+          current_field << i
+        else
+          if i == "\\"
+            escaped = true
+            current_field ||= ""
+            current_field << i
+          elsif i == "(" && !in_quoted_string && !in_sub_insert
+            in_sub_insert = true
+          elsif i == ")" && !in_quoted_string && in_sub_insert
+            fields << current_field unless current_field.nil?
+            output << fields unless fields.length == 0
+            in_sub_insert = false
+            fields = []
+            current_field = nil
+          elsif i == "'" && !in_quoted_string
+            fields << current_field unless current_field.nil?
+            current_field = ''
+            in_quoted_string = true
+          elsif i == "'" && in_quoted_string
+            fields << current_field unless current_field.nil?
+            current_field = nil
+            in_quoted_string = false
+          elsif i == "," && !in_quoted_string && in_sub_insert
+            fields << current_field unless current_field.nil?
+            current_field = nil
+          elsif i == "L" && !in_quoted_string && in_sub_insert && current_field == "NUL"
+            current_field = nil
+            fields << current_field
+          elsif (i == " " || i == "\t") && !in_quoted_string
+            # Don't add whitespace not in a string
+          elsif in_sub_insert
+            current_field ||= ""
+            current_field << i
+          end
+        end
+      end
+
+      fields << current_field unless current_field.nil?
+      output << fields unless fields.length == 0
+      output
+    end
+
+  end
+end

data/lib/my_obfuscate/mysql.rb

@@ -1,9 +1,9 @@
 class MyObfuscate
   class Mysql
-    INSERT_REGEX = /^\s*INSERT INTO `(.*?)` \((.*?)\) VALUES\s*/i
+    include MyObfuscate::DatabaseHelperShared
 
     def parse_insert_statement(line)
-      if regex_match = INSERT_REGEX.match(line)
+      if regex_match = insert_regex.match(line)
         {
             :table_name => regex_match[1].to_sym,
             :column_names => regex_match[2].split(/`\s*,\s*`/).map { |col| col.gsub('`', "").to_sym }
@@ -11,81 +11,13 @@ class MyObfuscate
       end
     end
 
-    def rows_to_be_inserted(line)
-      line = line.gsub(INSERT_REGEX, '').gsub(/\s*;\s*$/, '')
-      context_aware_mysql_string_split(line)
-    end
-
-    def make_valid_value_string(value)
-      if value.nil?
-        "NULL"
-      elsif value =~ /^0x[0-9a-fA-F]+$/
-        value
-      else
-        "'" + value + "'"
-      end
-    end
-
     def make_insert_statement(table_name, column_names, values_strings)
       "INSERT INTO `#{table_name}` (`#{column_names.join('`, `')}`) VALUES #{values_strings};"
     end
 
-    private
-
-    # Be aware, strings must be quoted in single quotes!
-    def context_aware_mysql_string_split(string)
-      in_sub_insert = false
-      in_quoted_string = false
-      escaped = false
-      current_field = nil
-      length = string.length
-      fields = []
-      output = []
-
-      string.each_char do |i|
-        if escaped
-          escaped = false
-          current_field ||= ""
-          current_field << i
-        else
-          if i == "\\"
-            escaped = true
-            current_field ||= ""
-            current_field << i
-          elsif i == "(" && !in_quoted_string && !in_sub_insert
-            in_sub_insert = true
-          elsif i == ")" && !in_quoted_string && in_sub_insert
-            fields << current_field unless current_field.nil?
-            output << fields unless fields.length == 0
-            in_sub_insert = false
-            fields = []
-            current_field = nil
-          elsif i == "'" && !in_quoted_string
-            fields << current_field unless current_field.nil?
-            current_field = ''
-            in_quoted_string = true
-          elsif i == "'" && in_quoted_string
-            fields << current_field unless current_field.nil?
-            current_field = nil
-            in_quoted_string = false
-          elsif i == "," && !in_quoted_string && in_sub_insert
-            fields << current_field unless current_field.nil?
-            current_field = nil
-          elsif i == "L" && !in_quoted_string && in_sub_insert && current_field == "NUL"
-            current_field = nil
-            fields << current_field
-          elsif (i == " " || i == "\t") && !in_quoted_string
-            # Don't add whitespace not in a string
-          elsif in_sub_insert
-            current_field ||= ""
-            current_field << i
-          end
-        end
-      end
-
-      fields << current_field unless current_field.nil?
-      output << fields unless fields.length == 0
-      output
+    def insert_regex
+      /^\s*INSERT INTO `(.*?)` \((.*?)\) VALUES\s*/i
     end
+
   end
 end

data/lib/my_obfuscate/postgres.rb

@@ -0,0 +1,23 @@
+class MyObfuscate
+  class Postgres
+    include MyObfuscate::DatabaseHelperShared
+
+    def parse_insert_statement(line)
+      if regex_match = insert_regex.match(line)
+        {
+            :table_name => regex_match[1].to_sym,
+            :column_names => regex_match[2].split(/\s*,\s*/).map(&:to_sym)
+        }
+      end
+    end
+
+    def make_insert_statement(table_name, column_names, values_strings)
+      "INSERT INTO #{table_name} (#{column_names.join(', ')}) VALUES #{values_strings};"
+    end
+
+    def insert_regex
+      /^\s*INSERT INTO (.*?) \((.*?)\) VALUES\s*/i
+    end
+
+  end
+end

data/lib/my_obfuscate/sql_server.rb

@@ -1,9 +1,8 @@
 class MyObfuscate
   class SqlServer
-    INSERT_REGEX = /^\s*INSERT (?:INTO )?\[dbo\]\.\[(.*?)\] \((.*?)\) VALUES\s*/i
 
     def parse_insert_statement(line)
-      if regex_match = INSERT_REGEX.match(line)
+      if regex_match = insert_regex.match(line)
         {
             :table_name => regex_match[1].to_sym,
             :column_names => regex_match[2].split(/\]\s*,\s*\[/).map { |col| col.gsub(/[\[\]]/, "").to_sym }
@@ -12,7 +11,7 @@ class MyObfuscate
     end
 
     def rows_to_be_inserted(line)
-      line = line.gsub(INSERT_REGEX, '').gsub(/\s*;?\s*$/, '').gsub(/^\(/, '').gsub(/\)$/, '')
+      line = line.gsub(insert_regex, '').gsub(/\s*;?\s*$/, '').gsub(/^\(/, '').gsub(/\)$/, '')
       context_aware_sql_server_string_split(line)
     end
 
@@ -32,6 +31,10 @@ class MyObfuscate
 
     private
 
+    def insert_regex
+      /^\s*INSERT (?:INTO )?\[dbo\]\.\[(.*?)\] \((.*?)\) VALUES\s*/i
+    end
+
     def context_aware_sql_server_string_split(string)
       in_quoted_string = false
       backslash_escape = false
@@ -78,4 +81,4 @@ class MyObfuscate
       [completed_fields]
     end
   end
-end
+end

data/lib/my_obfuscate/version.rb

@@ -1,3 +1,3 @@
 class MyObfuscate
-  VERSION = "0.3.7"
+  VERSION = "0.4.2"
 end

data/lib/my_obfuscate.rb

@@ -1,5 +1,7 @@
 require 'jcode' if RUBY_VERSION < '1.9'
-require 'faker'
+require 'digest/md5'
+require 'ffaker'
+require 'walker_method'
 
 # Class for obfuscating MySQL dumps. This can parse mysqldump outputs when using the -c option, which includes
 # column names in the insert statements.
@@ -24,6 +26,8 @@ class MyObfuscate
     if @database_helper.nil?
       if @database_type == :sql_server
         @database_helper = SqlServer.new
+      elsif @database_type == :postgres
+        @database_helper = Postgres.new
       else
         @database_helper = Mysql.new
       end
@@ -86,7 +90,7 @@ class MyObfuscate
 
     table_config.each do |column, definition|
       index = columns.index(column)
-      
+
       definition = { :type => definition } if definition.is_a?(Symbol)
 
       if definition.has_key?(:unless)
@@ -108,11 +112,14 @@ class MyObfuscate
 
       row[index.to_i] = case definition[:type]
         when :email
-          clean_quotes(Faker::Internet.email)
+          md5 = Digest::MD5.hexdigest(rand.to_s)[0...5]
+          clean_quotes("#{Faker::Internet.email}.#{md5}.example.com")
         when :string
           random_string(definition[:length] || 30, definition[:chars] || SENSIBLE_CHARS)
         when :lorem
           clean_bad_whitespace(clean_quotes(Faker::Lorem.sentences(definition[:number] || 1).join(".  ")))
+        when :like_english
+          clean_quotes random_english_sentences(definition[:number] || 1)
         when :name
           clean_quotes(Faker::Name.name)
         when :first_name
@@ -120,25 +127,28 @@ class MyObfuscate
         when :last_name
           clean_quotes(Faker::Name.last_name)
         when :address
-          clean_quotes("#{Faker::Address.street_address}\\n#{Faker::Address.city}, #{Faker::Address.state_abbr} #{Faker::Address.zip_code}")
+          clean_quotes("#{Faker::AddressUS.street_address}\\n#{Faker::AddressUS.city}, #{Faker::AddressUS.state_abbr} #{Faker::AddressUS.zip_code}")
         when :street_address
-          clean_bad_whitespace(clean_quotes(Faker::Address.street_address))
+          clean_bad_whitespace(clean_quotes(Faker::AddressUS.street_address))
         when :city
-          clean_quotes(Faker::Address.city)
+          clean_quotes(Faker::AddressUS.city)
         when :state
-          Faker::Address.state_abbr
+          clean_quotes Faker::AddressUS.state_abbr
         when :zip_code
-          Faker::Address.zip_code
+          Faker::AddressUS.zip_code
         when :phone
-          Faker::PhoneNumber.phone_number
+          clean_quotes Faker::PhoneNumber.phone_number
         when :company
           clean_bad_whitespace(clean_quotes(Faker::Company.name))
         when :ipv4
           Faker::Internet.ip_v4_address
         when :ipv6
-          Faker::Internet.ip_v6_address
+          # Inlined from Faker because ffaker doesn't have ipv6.
+          @@ip_v6_space ||= (0..65535).to_a
+          container = (1..8).map{ |_| @@ip_v6_space.sample }
+          container.map{ |n| n.to_s(16) }.join(':')
         when :url
-          clean_bad_whitespace(Faker::Internet.url)
+          clean_bad_whitespace(Faker::Internet.http_url)
         when :integer
           random_integer(definition[:between] || (0..1000)).to_s
         when :fixed
@@ -171,6 +181,27 @@ class MyObfuscate
     out
   end
 
+  def self.random_english_sentences(num)
+    @@walker_method ||= begin
+      words, counts = [], []
+      File.read(File.expand_path(File.join(File.dirname(__FILE__), 'my_obfuscate', 'data', 'en_50K.txt'))).each_line do |line|
+        word, count = line.split(/\s+/)
+        words << word
+        counts << count.to_i
+      end
+      WalkerMethod.new(words, counts)
+    end
+
+    sentences = []
+    num.times do
+      words = []
+      (5 + rand * 6).to_i.times { words << @@walker_method.random }
+      sentences << words.join(" ") + "."
+      sentences.last[0] = sentences.last[0].upcase
+    end
+    sentences.join(" ")
+  end
+
   def check_for_defined_columns_not_in_table(table_name, columns)
     missing_columns = config[table_name].keys - columns
     unless missing_columns.length == 0
@@ -206,17 +237,19 @@ class MyObfuscate
       end
     end
   end
-  
+
   private
-  
+
   def self.clean_quotes(value)
     value.gsub(/['"]/, '')
   end
-  
+
   def self.clean_bad_whitespace(value)
     value.gsub(/[\n\t\r]/, '')
   end
 end
 
+require 'my_obfuscate/database_helper_shared'
 require 'my_obfuscate/mysql'
 require 'my_obfuscate/sql_server'
+require 'my_obfuscate/postgres'

data/my_obfuscate.gemspec

@@ -8,12 +8,13 @@ Gem::Specification.new do |s|
 
   s.authors = ["Andrew Cantino", "Dave Willett", "Mike Grafton", "Mason Glaves", "Greg Bell", "Mavenlink"]
   s.description = %q{Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.}
-  s.email = %q{andrew@iterationlabs.com}
-  s.homepage = %q{http://github.com/iterationlabs/my_obfuscate}
+  s.email = %q{andrew@mavenlink.com}
+  s.homepage = %q{http://github.com/mavenlink/my_obfuscate}
   s.summary = %q{Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.}
 
+  s.add_dependency "ffaker"
+  s.add_dependency "walker_method"
   s.add_development_dependency "rspec"
-  s.add_dependency "faker", ">= 1.0.1"
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/spec/{mysql_spec.rb → my_obfuscate/database_helper_shared_examples.rb}

@@ -1,20 +1,4 @@
-require 'spec_helper'
-
-describe MyObfuscate::Mysql do
-  describe "#parse_insert_statement" do
-    it "should return nil for other SQL syntaxes (MS SQL Server)" do
-      subject.parse_insert_statement("INSERT [dbo].[TASKS] ([TaskID], [TaskName]) VALUES (61, N'Report Thing')").should be_nil
-    end
-
-    it "should return nil for MySQL non-insert statements" do
-      subject.parse_insert_statement("CREATE TABLE `some_table`;").should be_nil
-    end
-
-    it "should return a hash of table name, column names for MySQL insert statements" do
-      hash = subject.parse_insert_statement("INSERT INTO `some_table` (`email`, `name`, `something`, `age`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54);")
-      hash.should == {:table_name => :some_table, :column_names => [:email, :name, :something, :age]}
-    end
-  end
+shared_examples MyObfuscate::DatabaseHelperShared do
 
   describe "#rows_to_be_inserted" do
     it "should split a mysql string into fields" do
@@ -75,4 +59,5 @@ describe MyObfuscate::Mysql do
       subject.make_valid_value_string(value).should == "'hello world'"
     end
   end
+
 end

data/spec/my_obfuscate/mysql_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+require 'my_obfuscate/database_helper_shared_examples'
+
+describe MyObfuscate::Mysql do
+
+  it_behaves_like MyObfuscate::DatabaseHelperShared
+
+  describe "#parse_insert_statement" do
+    it "should return nil for other SQL syntaxes (MS SQL Server)" do
+      subject.parse_insert_statement("INSERT [dbo].[TASKS] ([TaskID], [TaskName]) VALUES (61, N'Report Thing')").should be_nil
+    end
+
+    it "should return nil for MySQL non-insert statements" do
+      subject.parse_insert_statement("CREATE TABLE `some_table`;").should be_nil
+    end
+
+    it "should return a hash of table name, column names for MySQL insert statements" do
+      hash = subject.parse_insert_statement("INSERT INTO `some_table` (`email`, `name`, `something`, `age`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54);")
+      hash.should == {:table_name => :some_table, :column_names => [:email, :name, :something, :age]}
+    end
+  end
+
+end

data/spec/my_obfuscate/postgres_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+require 'my_obfuscate/database_helper_shared_examples'
+
+describe MyObfuscate::Postgres do
+
+  it_behaves_like MyObfuscate::DatabaseHelperShared
+
+  describe "#parse_insert_statement" do
+    it "should return nil for other SQL syntaxes (MS SQL Server)" do
+      subject.parse_insert_statement("INSERT [dbo].[TASKS] ([TaskID], [TaskName]) VALUES (61, N'Report Thing')").should be_nil
+    end
+
+    it "should return nil for MySQL non-insert statements" do
+      subject.parse_insert_statement("CREATE TABLE `some_table`;").should be_nil
+    end
+
+    it "should return a hash of table name, column names for MySQL insert statements" do
+      hash = subject.parse_insert_statement("INSERT INTO some_table (email, name, something, age) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54);")
+      hash.should == {:table_name => :some_table, :column_names => [:email, :name, :something, :age]}
+    end
+  end
+
+end

data/spec/my_obfuscate_spec.rb

@@ -25,9 +25,11 @@ describe MyObfuscate do
 
   describe "MyObfuscate.apply_table_config" do
     it "should work on email addresses" do
-      new_row = MyObfuscate.apply_table_config(["blah", "something_else"], {:a => {:type => :email}}, [:a, :b])
-      new_row.length.should == 2
-      new_row.first.should =~ /^[\w\.]+\@\w+\.\w+$/
+      100.times do
+        new_row = MyObfuscate.apply_table_config(["blah", "something_else"], {:a => {:type => :email}}, [:a, :b])
+        new_row.length.should == 2
+        new_row.first.should =~ /^[\w\.]+\@(\w+\.){2,3}[a-f0-9]{5}\.example\.com$/
+      end
     end
 
     it "should work on strings" do
@@ -248,7 +250,7 @@ describe MyObfuscate do
       new_row[0].should_not == "blah"
       new_row[0].should =~ /\d+/
     end
-    
+
     describe "when faker generates values with quotes in them" do
       before do
         Faker::Address.stub(:city).and_return("O'ReillyTown")
@@ -257,7 +259,7 @@ describe MyObfuscate do
         Faker::Name.stub(:last_name).and_return("O'Reilly")
         Faker::Lorem.stub(:sentences).with(any_args).and_return(["Foo bar O'Thingy"])
       end
-      
+
       it "should remove single quotes from the value" do
         new_row = MyObfuscate.apply_table_config(["address", "city", "first", "last", "fullname", "some text"],
                   {:a => :address, :b => :city, :c => :first_name, :d => :last_name, :e => :name, :f => :lorem},
@@ -366,10 +368,11 @@ describe MyObfuscate do
           @output_string.should_not include("INSERT INTO `some_table` (`email`, `name`, `something`, `age`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54);")
         end
 
-        it "honors a special case: on the people table, rows with anything@honk.com in a slot marked with :honk_email_skip do not change this slot" do
+        it "honors a special case: on the people table, rows with skip_regexes that match are skipped" do
           @output_string.should include("('bob@honk.com',")
           @output_string.should include("('dontmurderme@direwolf.com',")
           @output_string.should_not include("joe@joe.com")
+          @output_string.should include("example.com")
         end
       end
 
@@ -556,4 +559,23 @@ describe MyObfuscate do
       end
     end
   end
+
+  describe "MyObfuscate.random_english_sentences" do
+    before do
+      File.should_receive(:read).once.and_return("hello 2")
+    end
+
+    after do
+      MyObfuscate.class_variable_set(:@@walker_method, nil)
+    end
+
+    it "should only load file data once" do
+      MyObfuscate.random_english_sentences(1)
+      MyObfuscate.random_english_sentences(1)
+    end
+
+    it "should make random sentences" do
+      MyObfuscate.random_english_sentences(2).should =~ /^(Hello( hello)+\.\s*){2}$/
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: my_obfuscate
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.4.2
   prerelease: 
 platform: ruby
 authors:
@@ -14,17 +14,17 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-27 00:00:00.000000000 Z
+date: 2013-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: ffaker
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -33,13 +33,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: faker
+  name: walker_method
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -47,29 +47,54 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.1
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Standalone Ruby code for the selective rewriting of MySQL dumps in order
   to protect user privacy.
-email: andrew@iterationlabs.com
+email: andrew@mavenlink.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .ruby-gemset
+- .ruby-version
+- .travis.yml
+- CHANGES
 - Gemfile
 - LICENSE
 - README.rdoc
 - Rakefile
 - lib/my_obfuscate.rb
+- lib/my_obfuscate/data/en_50K.txt
+- lib/my_obfuscate/database_helper_shared.rb
 - lib/my_obfuscate/mysql.rb
+- lib/my_obfuscate/postgres.rb
 - lib/my_obfuscate/sql_server.rb
 - lib/my_obfuscate/version.rb
 - my_obfuscate.gemspec
+- spec/my_obfuscate/database_helper_shared_examples.rb
+- spec/my_obfuscate/mysql_spec.rb
+- spec/my_obfuscate/postgres_spec.rb
+- spec/my_obfuscate/sql_server_spec.rb
 - spec/my_obfuscate_spec.rb
-- spec/mysql_spec.rb
 - spec/spec_helper.rb
-- spec/sql_server_spec.rb
-homepage: http://github.com/iterationlabs/my_obfuscate
+homepage: http://github.com/mavenlink/my_obfuscate
 licenses: []
 post_install_message: 
 rdoc_options: []
@@ -89,9 +114,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.19
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Standalone Ruby code for the selective rewriting of MySQL dumps in order
   to protect user privacy.
-test_files: []
+test_files:
+- spec/my_obfuscate/database_helper_shared_examples.rb
+- spec/my_obfuscate/mysql_spec.rb
+- spec/my_obfuscate/postgres_spec.rb
+- spec/my_obfuscate/sql_server_spec.rb
+- spec/my_obfuscate_spec.rb
+- spec/spec_helper.rb