my_obfuscate 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/.gitignore +6 -0
- data/LICENSE +19 -0
- data/README.rdoc +55 -0
- data/Rakefile +52 -0
- data/VERSION +1 -0
- data/lib/my_obfuscate.rb +172 -0
- data/my_obfuscate.gemspec +54 -0
- data/spec/my_obfuscate_spec.rb +196 -0
- data/spec/spec_helper.rb +10 -0
- metadata +78 -0
data/.document
ADDED
data/.gitignore
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
Copyright (c) 2009 Honk
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
4
|
+
of this software and associated documentation files (the "Software"), to deal
|
5
|
+
in the Software without restriction, including without limitation the rights
|
6
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
7
|
+
copies of the Software, and to permit persons to whom the Software is
|
8
|
+
furnished to do so, subject to the following conditions:
|
9
|
+
|
10
|
+
The above copyright notice and this permission notice shall be included in
|
11
|
+
all copies or substantial portions of the Software.
|
12
|
+
|
13
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
14
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
15
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
16
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
17
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
18
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
19
|
+
THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
= my_obfuscate
|
2
|
+
|
3
|
+
Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.
|
4
|
+
|
5
|
+
= Example Usage
|
6
|
+
|
7
|
+
Make an obfuscator.rb script:
|
8
|
+
|
9
|
+
#!/usr/bin/env ruby
|
10
|
+
require "rubygems"
|
11
|
+
require "my_obfuscate"
|
12
|
+
|
13
|
+
obfuscator = MyObfuscate.new({
|
14
|
+
:people => {
|
15
|
+
:email => { :type => :email, :skip_regexes => [/^[\w\.\_]+@my_company\.com$/i] },
|
16
|
+
:ethnicity => { :type => :null },
|
17
|
+
:crypted_password => { :type => :fixed, :string => "SOME_FIXED_PASSWORD_FOR_EASE_OF_DEBUGGING" },
|
18
|
+
:salt => { :type => :fixed, :string => "SOME_THING" },
|
19
|
+
:remember_token => { :type => :null },
|
20
|
+
:remember_token_expires_at => { :type => :null },
|
21
|
+
:photo_file_name => { :type => :null },
|
22
|
+
:photo_content_type => { :type => :null },
|
23
|
+
:photo_file_size => { :type => :null },
|
24
|
+
:photo_updated_at => { :type => :null },
|
25
|
+
:postal_code => { :type => :fixed, :string => "94109" },
|
26
|
+
:name => { :type => :fixed, :string => "Production User" },
|
27
|
+
:relationship_status => { :type => :fixed, :one_of => ["Single", "Divorced", "Married", "Engaged", "In a Relationship"] },
|
28
|
+
:has_children => { :type => :integer, :between => 0..1 },
|
29
|
+
},
|
30
|
+
|
31
|
+
:invites => :truncate,
|
32
|
+
:invite_requests => :truncate,
|
33
|
+
|
34
|
+
:relationships => {
|
35
|
+
:account_id => { :type => :string, :length => 8, :chars => DatabaseDumpObfuscator::NUMBER_CHARS },
|
36
|
+
:code => { :type => :string, :length => 8, :chars => DatabaseDumpObfuscator::USERNAME_CHARS }
|
37
|
+
}
|
38
|
+
})
|
39
|
+
obfuscator.obfuscate(STDIN, STDOUT)
|
40
|
+
|
41
|
+
And to get an obfuscated dump:
|
42
|
+
mysqldump -c --add-drop-table -u user -p database | ruby obfuscator.rb > obfuscated_dump.sql
|
43
|
+
Note that the -c option on mysqldump is required to use my_obfuscator.
|
44
|
+
|
45
|
+
== Note on Patches/Pull Requests
|
46
|
+
|
47
|
+
* Fork the project.
|
48
|
+
* Make your feature addition or bug fix.
|
49
|
+
* Add tests for it. This is important so I don't break it in a future version unintentionally.
|
50
|
+
* Commit, do not mess with rakefile, version, or history. (If you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
|
51
|
+
* Send me a pull request. Bonus points for topic branches.
|
52
|
+
|
53
|
+
== Copyright
|
54
|
+
|
55
|
+
Copyright (c) 2009 Honk. See LICENSE for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
|
4
|
+
begin
|
5
|
+
require 'jeweler'
|
6
|
+
Jeweler::Tasks.new do |gem|
|
7
|
+
gem.name = "my_obfuscate"
|
8
|
+
gem.summary = %Q{Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.}
|
9
|
+
gem.description = %Q{Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.}
|
10
|
+
gem.email = "andrew@pivotallabs.com"
|
11
|
+
gem.homepage = "http://github.com/honkster/myobfuscate"
|
12
|
+
gem.authors = ["Andrew Cantino", "Dave Willett", "Mike Grafton", "Mason Glaves"]
|
13
|
+
gem.add_development_dependency "rspec"
|
14
|
+
gem.rubyforge_project = 'my-obfuscate'
|
15
|
+
end
|
16
|
+
|
17
|
+
Jeweler::RubyforgeTasks.new do |rubyforge|
|
18
|
+
rubyforge.doc_task = "rdoc"
|
19
|
+
end
|
20
|
+
rescue LoadError
|
21
|
+
puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
|
22
|
+
end
|
23
|
+
|
24
|
+
require 'spec/rake/spectask'
|
25
|
+
Spec::Rake::SpecTask.new(:spec) do |spec|
|
26
|
+
spec.libs << 'lib' << 'spec'
|
27
|
+
spec.spec_files = FileList['spec/**/*_spec.rb']
|
28
|
+
end
|
29
|
+
|
30
|
+
Spec::Rake::SpecTask.new(:rcov) do |spec|
|
31
|
+
spec.libs << 'lib' << 'spec'
|
32
|
+
spec.pattern = 'spec/**/*_spec.rb'
|
33
|
+
spec.rcov = true
|
34
|
+
end
|
35
|
+
|
36
|
+
task :spec => :check_dependencies
|
37
|
+
|
38
|
+
task :default => :spec
|
39
|
+
|
40
|
+
require 'rake/rdoctask'
|
41
|
+
Rake::RDocTask.new do |rdoc|
|
42
|
+
if File.exist?('VERSION')
|
43
|
+
version = File.read('VERSION')
|
44
|
+
else
|
45
|
+
version = ""
|
46
|
+
end
|
47
|
+
|
48
|
+
rdoc.rdoc_dir = 'rdoc'
|
49
|
+
rdoc.title = "my_obfuscate #{version}"
|
50
|
+
rdoc.rdoc_files.include('README*')
|
51
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
52
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
data/lib/my_obfuscate.rb
ADDED
@@ -0,0 +1,172 @@
|
|
1
|
+
# This can parse mysqldump outputs when the dumps have option -c, which includes column names in the insert statements.
|
2
|
+
require 'jcode'
|
3
|
+
|
4
|
+
class MyObfuscate
|
5
|
+
attr_accessor :config
|
6
|
+
|
7
|
+
INSERT_REGEX = /^\s*INSERT INTO `(.*?)` \((.*?)\) VALUES\s*/i
|
8
|
+
NUMBER_CHARS = "1234567890"
|
9
|
+
USERNAME_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_" + NUMBER_CHARS
|
10
|
+
SENSIBLE_CHARS = USERNAME_CHARS + '+-=[{]}/?|!@#$%^&*()`~'
|
11
|
+
|
12
|
+
def initialize(configuration = {})
|
13
|
+
@config = configuration
|
14
|
+
end
|
15
|
+
|
16
|
+
# We assume that every INSERT INTO line occupies one line in the file, with no internal linebreaks.
|
17
|
+
def obfuscate(input_io, output_io)
|
18
|
+
input_io.each do |line|
|
19
|
+
if regex_result = INSERT_REGEX.match(line)
|
20
|
+
table_name = regex_result[1].to_sym
|
21
|
+
columns = regex_result[2].split(/`\s*,\s*`/).map { |col| col.gsub('`',"").to_sym }
|
22
|
+
if config[table_name]
|
23
|
+
output_io.puts obfuscate_line(line, table_name, columns)
|
24
|
+
else
|
25
|
+
output_io.write line
|
26
|
+
end
|
27
|
+
else
|
28
|
+
output_io.write line
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.reasembling_each_insert(line, table_name, columns)
|
34
|
+
line = line.gsub(INSERT_REGEX, '').gsub(/\s*;\s*$/, '')
|
35
|
+
output = context_aware_mysql_string_split(line).map do |sub_insert|
|
36
|
+
result = yield(sub_insert)
|
37
|
+
result = result.map do |i|
|
38
|
+
if i.nil?
|
39
|
+
"NULL"
|
40
|
+
else
|
41
|
+
"'" + i + "'"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
result = result.join(",")
|
45
|
+
"(" + result + ")"
|
46
|
+
end.join(",")
|
47
|
+
"INSERT INTO `#{table_name}` (`#{columns.join('`, `')}`) VALUES #{output};"
|
48
|
+
end
|
49
|
+
|
50
|
+
# Note: Strings must be quoted in single quotes!
|
51
|
+
def self.context_aware_mysql_string_split(string)
|
52
|
+
in_sub_insert = false
|
53
|
+
in_quoted_string = false
|
54
|
+
escaped = false
|
55
|
+
current_field = nil
|
56
|
+
length = string.length
|
57
|
+
index = 0
|
58
|
+
fields = []
|
59
|
+
output = []
|
60
|
+
string.each_char do |i|
|
61
|
+
if escaped
|
62
|
+
escaped = false
|
63
|
+
current_field ||= ""
|
64
|
+
current_field << i
|
65
|
+
else
|
66
|
+
if i == "\\"
|
67
|
+
escaped = true
|
68
|
+
current_field ||= ""
|
69
|
+
current_field << i
|
70
|
+
elsif i == "(" && !in_quoted_string && !in_sub_insert
|
71
|
+
in_sub_insert = true
|
72
|
+
elsif i == ")" && !in_quoted_string && in_sub_insert
|
73
|
+
fields << current_field unless current_field.nil?
|
74
|
+
output << fields unless fields.length == 0
|
75
|
+
in_sub_insert = false
|
76
|
+
fields = []
|
77
|
+
current_field = nil
|
78
|
+
elsif i == "'" && !in_quoted_string
|
79
|
+
fields << current_field unless current_field.nil?
|
80
|
+
current_field = ''
|
81
|
+
in_quoted_string = true
|
82
|
+
elsif i == "'" && in_quoted_string
|
83
|
+
fields << current_field unless current_field.nil?
|
84
|
+
current_field = nil
|
85
|
+
in_quoted_string = false
|
86
|
+
elsif i == "," && !in_quoted_string && in_sub_insert
|
87
|
+
fields << current_field unless current_field.nil?
|
88
|
+
current_field = nil
|
89
|
+
elsif i == "L" && !in_quoted_string && in_sub_insert && current_field == "NUL"
|
90
|
+
current_field = nil
|
91
|
+
fields << current_field
|
92
|
+
elsif (i == " " || i == "\t") && !in_quoted_string
|
93
|
+
# Don't add whitespace not in a string
|
94
|
+
elsif in_sub_insert
|
95
|
+
current_field ||= ""
|
96
|
+
current_field << i
|
97
|
+
end
|
98
|
+
end
|
99
|
+
index += 1
|
100
|
+
end
|
101
|
+
fields << current_field unless current_field.nil?
|
102
|
+
output << fields unless fields.length == 0
|
103
|
+
output
|
104
|
+
end
|
105
|
+
|
106
|
+
def self.apply_table_config(row, table_config, columns)
|
107
|
+
return row unless table_config.is_a?(Hash)
|
108
|
+
|
109
|
+
table_config.each do |column, definition|
|
110
|
+
index = columns.index(column)
|
111
|
+
|
112
|
+
if definition[:skip_regexes]
|
113
|
+
next if definition[:skip_regexes].any? {|regex| row[index] =~ regex}
|
114
|
+
end
|
115
|
+
|
116
|
+
row[index.to_i] = case definition[:type]
|
117
|
+
when :email
|
118
|
+
random_string(4..10, USERNAME_CHARS) + "@example.com"
|
119
|
+
when :string
|
120
|
+
random_string(definition[:length], definition[:chars] || SENSIBLE_CHARS)
|
121
|
+
when :integer
|
122
|
+
random_integer(definition[:between] || (0..1000)).to_s
|
123
|
+
when :fixed
|
124
|
+
if definition[:one_of]
|
125
|
+
definition[:one_of][(rand * definition[:one_of].length).to_i]
|
126
|
+
else
|
127
|
+
definition[:string]
|
128
|
+
end
|
129
|
+
when :null
|
130
|
+
nil
|
131
|
+
else
|
132
|
+
row[index]
|
133
|
+
end
|
134
|
+
end
|
135
|
+
row
|
136
|
+
end
|
137
|
+
|
138
|
+
def self.random_integer(between)
|
139
|
+
(between.min + (between.max - between.min) * rand).round
|
140
|
+
end
|
141
|
+
|
142
|
+
def self.random_string(length_or_range, chars)
|
143
|
+
length_or_range = (length_or_range..length_or_range) if length_or_range.is_a?(Fixnum)
|
144
|
+
times = random_integer(length_or_range)
|
145
|
+
out = ""
|
146
|
+
times.times { out << chars[rand * chars.length] }
|
147
|
+
out
|
148
|
+
end
|
149
|
+
|
150
|
+
def check_for_missing_columns(table_name, columns)
|
151
|
+
missing_columns = config[table_name].keys - columns
|
152
|
+
unless missing_columns.length == 0
|
153
|
+
error_message = missing_columns.map do |missing_column|
|
154
|
+
"Column '#{missing_column}' could not be found in table '#{table_name}', please fix your obfuscator config."
|
155
|
+
end.join("\n")
|
156
|
+
raise RuntimeError.new(error_message)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def obfuscate_line(line, table_name, columns)
|
161
|
+
table_config = config[table_name]
|
162
|
+
if table_config == :truncate
|
163
|
+
""
|
164
|
+
else
|
165
|
+
check_for_missing_columns(table_name, columns)
|
166
|
+
# Note: Remember to SQL escape strings in what you pass back.
|
167
|
+
MyObfuscate.reasembling_each_insert(line, table_name, columns) do |row|
|
168
|
+
MyObfuscate.apply_table_config(row, table_config, columns)
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{my_obfuscate}
|
8
|
+
s.version = "0.1.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Andrew Cantino", "Dave Willett", "Mike Grafton", "Mason Glaves"]
|
12
|
+
s.date = %q{2009-10-01}
|
13
|
+
s.description = %q{Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.}
|
14
|
+
s.email = %q{andrew@pivotallabs.com}
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"LICENSE",
|
17
|
+
"README.rdoc"
|
18
|
+
]
|
19
|
+
s.files = [
|
20
|
+
".document",
|
21
|
+
".gitignore",
|
22
|
+
"LICENSE",
|
23
|
+
"README.rdoc",
|
24
|
+
"Rakefile",
|
25
|
+
"VERSION",
|
26
|
+
"lib/my_obfuscate.rb",
|
27
|
+
"my_obfuscate.gemspec",
|
28
|
+
"spec/my_obfuscate_spec.rb",
|
29
|
+
"spec/spec_helper.rb"
|
30
|
+
]
|
31
|
+
s.homepage = %q{http://github.com/honkster/myobfuscate}
|
32
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
33
|
+
s.require_paths = ["lib"]
|
34
|
+
s.rubyforge_project = %q{my-obfuscate}
|
35
|
+
s.rubygems_version = %q{1.3.5}
|
36
|
+
s.summary = %q{Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.}
|
37
|
+
s.test_files = [
|
38
|
+
"spec/my_obfuscate_spec.rb",
|
39
|
+
"spec/spec_helper.rb"
|
40
|
+
]
|
41
|
+
|
42
|
+
if s.respond_to? :specification_version then
|
43
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
44
|
+
s.specification_version = 3
|
45
|
+
|
46
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
47
|
+
s.add_development_dependency(%q<rspec>, [">= 0"])
|
48
|
+
else
|
49
|
+
s.add_dependency(%q<rspec>, [">= 0"])
|
50
|
+
end
|
51
|
+
else
|
52
|
+
s.add_dependency(%q<rspec>, [">= 0"])
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,196 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
|
3
|
+
describe MyObfuscate do
|
4
|
+
describe "MyObfuscate.context_aware_mysql_string_split" do
|
5
|
+
it "should split a mysql string into fields" do
|
6
|
+
string = "('bob@bob.com','bob', 'somethingelse1', 25, '2', 10, 'hi')"
|
7
|
+
fields = [['bob@bob.com', 'bob', 'somethingelse1', '25', '2', '10', "hi"]]
|
8
|
+
MyObfuscate.context_aware_mysql_string_split(string).should == fields
|
9
|
+
end
|
10
|
+
|
11
|
+
it "should work ok with escaped characters" do
|
12
|
+
string = "('bob,@bob.c , om', 'bo\\', b', 'some\"thin\\gel\\\\\\'se1', 25, '2', 10, 'hi', 5)"
|
13
|
+
fields = [['bob,@bob.c , om', 'bo\\\', b', 'some"thin\\gel\\\\\\\'se1', '25', '2', '10', "hi", "5"]]
|
14
|
+
MyObfuscate.context_aware_mysql_string_split(string).should == fields
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should work with multiple subinserts" do
|
18
|
+
string = "(1,2,3, '((m))(oo()s,e'), ('bob,@bob.c , om', 'bo\\', b', 'some\"thin\\gel\\\\\\'se1', 25, '2', 10, 'hi', 5)"
|
19
|
+
fields = [["1", "2", "3", "((m))(oo()s,e"], ['bob,@bob.c , om', 'bo\\\', b', 'some"thin\\gel\\\\\\\'se1', '25', '2', '10', "hi", "5"]]
|
20
|
+
MyObfuscate.context_aware_mysql_string_split(string).should == fields
|
21
|
+
end
|
22
|
+
|
23
|
+
it "should work ok with NULL values" do
|
24
|
+
string = "(NULL , 'bob@bob.com','bob', NULL, 25, '2', NULL, 'hi', NULL )"
|
25
|
+
fields = [[nil, 'bob@bob.com', 'bob', nil, '25', '2', nil, "hi", nil]]
|
26
|
+
MyObfuscate.context_aware_mysql_string_split(string).should == fields
|
27
|
+
end
|
28
|
+
|
29
|
+
it "should work with empty strings" do
|
30
|
+
string = "(NULL , '', '' , '', 25, '2','', 'hi','')"
|
31
|
+
fields = [[nil, '', '', '', '25', '2', '', "hi", '']]
|
32
|
+
MyObfuscate.context_aware_mysql_string_split(string).should == fields
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
describe "MyObfuscate.reasembling_each_insert" do
|
37
|
+
before do
|
38
|
+
@column_names = [:a, :b, :c, :d]
|
39
|
+
@test_insert = "INSERT INTO `some_table` (`a`, `b`, `c`, `d`) VALUES ('(\\'bob@bob.com','b()ob','some(thingelse1','25)('),('joe@joe.com','joe','somethingelse2','54');"
|
40
|
+
@test_insert_passes = [
|
41
|
+
["(\\'bob@bob.com","b()ob", "some(thingelse1", "25)("],
|
42
|
+
["joe@joe.com","joe", "somethingelse2", "54"]
|
43
|
+
]
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should yield each subinsert and reassemble the result" do
|
47
|
+
count = 0
|
48
|
+
reassembled = MyObfuscate.reasembling_each_insert(@test_insert, "some_table", @column_names) do |sub_insert|
|
49
|
+
sub_insert.should == @test_insert_passes.shift
|
50
|
+
count += 1
|
51
|
+
sub_insert
|
52
|
+
end
|
53
|
+
count.should == 2
|
54
|
+
reassembled.should == @test_insert
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
describe "MyObfuscate.apply_table_config" do
|
59
|
+
it "should work on email addresses" do
|
60
|
+
new_row = MyObfuscate.apply_table_config(["blah", "something_else"], { :a => { :type => :email }}, [:a, :b])
|
61
|
+
new_row.length.should == 2
|
62
|
+
new_row.first.should =~ /^\w+\@\w+\.\w+$/
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should work on strings" do
|
66
|
+
new_row = MyObfuscate.apply_table_config(["blah", "something_else", "something crazy"], { :b => { :type => :string, :length => 7 }}, [:a, :b, :c])
|
67
|
+
new_row.length.should == 3
|
68
|
+
new_row[1].length.should == 7
|
69
|
+
new_row[1].should_not == "something_else"
|
70
|
+
end
|
71
|
+
|
72
|
+
it "should be able to generate random integers in ranges" do
|
73
|
+
new_row = MyObfuscate.apply_table_config(["blah", "something_else", "5"], { :c => { :type => :integer, :between => 10..100 }}, [:a, :b, :c])
|
74
|
+
new_row.length.should == 3
|
75
|
+
new_row[2].to_i.to_s.should == new_row[2] # It should be an integer.
|
76
|
+
new_row[2].should_not == "5"
|
77
|
+
end
|
78
|
+
|
79
|
+
it "should be able to substitute fixed strings" do
|
80
|
+
new_row = MyObfuscate.apply_table_config(["blah", "something_else", "5"], { :b => { :type => :fixed, :string => "hello" }}, [:a, :b, :c])
|
81
|
+
new_row.length.should == 3
|
82
|
+
new_row[1].should == "hello"
|
83
|
+
end
|
84
|
+
|
85
|
+
it "should be able to substitute fixed strings from a random set" do
|
86
|
+
looking_for = ["hello", "world"]
|
87
|
+
original_looking_for = looking_for.dup
|
88
|
+
guard = 0
|
89
|
+
while !looking_for.empty? && guard < 1000
|
90
|
+
new_row = MyObfuscate.apply_table_config(["blah", "something_else", "5"], { :a => { :type => :fixed, :one_of => ["hello", "world"] }}, [:a, :b, :c])
|
91
|
+
new_row.length.should == 3
|
92
|
+
original_looking_for.should include(new_row[0])
|
93
|
+
looking_for.delete new_row[0]
|
94
|
+
guard += 1
|
95
|
+
end
|
96
|
+
looking_for.should be_empty
|
97
|
+
end
|
98
|
+
|
99
|
+
it "should be able to set things NULL" do
|
100
|
+
new_row = MyObfuscate.apply_table_config(["blah", "something_else", "5"], { :b => { :type => :null }}, [:a, :b, :c])
|
101
|
+
new_row.length.should == 3
|
102
|
+
new_row[1].should == nil
|
103
|
+
end
|
104
|
+
|
105
|
+
end
|
106
|
+
|
107
|
+
describe "#obfuscate" do
|
108
|
+
context "when there is nothing to obfuscate" do
|
109
|
+
it "should accept an IO object for input and output, and copy the input to the output" do
|
110
|
+
ddo = MyObfuscate.new
|
111
|
+
string = "hello, world\nsup?"
|
112
|
+
input = StringIO.new(string)
|
113
|
+
output = StringIO.new
|
114
|
+
ddo.obfuscate(input, output)
|
115
|
+
input.rewind
|
116
|
+
output.rewind
|
117
|
+
output.read.should == string
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
context "when the dump to obfuscate is missing columns" do
|
122
|
+
before do
|
123
|
+
@database_dump = StringIO.new(<<-SQL)
|
124
|
+
INSERT INTO `some_table` (`email`, `name`, `something`, `age`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54);
|
125
|
+
SQL
|
126
|
+
@ddo = MyObfuscate.new({
|
127
|
+
:some_table => {
|
128
|
+
:email => { :type => :email, :honk_email_skip => true },
|
129
|
+
:name => { :type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS },
|
130
|
+
:gender => { :type => :fixed, :string => "m" }
|
131
|
+
}})
|
132
|
+
@output = StringIO.new
|
133
|
+
end
|
134
|
+
|
135
|
+
it "should raise an error if a column name can't be found" do
|
136
|
+
lambda {
|
137
|
+
@ddo.obfuscate(@database_dump, @output)
|
138
|
+
}.should raise_error
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
context "when there is something to obfuscate" do
|
143
|
+
before do
|
144
|
+
@database_dump = StringIO.new(<<-SQL)
|
145
|
+
INSERT INTO `some_table` (`email`, `name`, `something`, `age`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54),('dontmurderme@direwolf.com','direwolf', 'somethingelse3', 44);
|
146
|
+
INSERT INTO `another_table` (`a`, `b`, `c`, `d`) VALUES (1,2,3,4), (5,6,7,8);
|
147
|
+
INSERT INTO `one_more_table` (`a`, `password`, `c`, `d,d`) VALUES ('hello','kjhjd^&dkjh', 'aawefjkafe'), ('hello1','kjhj!', 892938), ('hello2','moose!!', NULL);
|
148
|
+
INSERT INTO `an_ignored_table` (`col`, `col2`) VALUES ('hello','kjhjd^&dkjh'), ('hello1','kjhj!'), ('hello2','moose!!');
|
149
|
+
SQL
|
150
|
+
|
151
|
+
@ddo = MyObfuscate.new({
|
152
|
+
:some_table => {
|
153
|
+
:email => { :type => :email, :skip_regexes => [/^[\w\.\_]+@honk\.com$/i, /^dontmurderme@direwolf.com$/] },
|
154
|
+
:name => { :type => :string, :length => 8, :chars => MyObfuscate::USERNAME_CHARS },
|
155
|
+
:age => { :type => :integer, :between => 10...80 }
|
156
|
+
},
|
157
|
+
:another_table => :truncate,
|
158
|
+
:one_more_table => {
|
159
|
+
# Note: fixed strings must be pre-SQL escaped!
|
160
|
+
:password => { :type => :fixed, :string => "monkey" },
|
161
|
+
:c => { :type => :null }
|
162
|
+
}
|
163
|
+
})
|
164
|
+
@output = StringIO.new
|
165
|
+
@ddo.obfuscate(@database_dump, @output)
|
166
|
+
@output.rewind
|
167
|
+
@output_string = @output.read
|
168
|
+
end
|
169
|
+
|
170
|
+
it "should be able to truncate tables" do
|
171
|
+
@output_string.should_not include("INSERT INTO `another_table`")
|
172
|
+
@output_string.should include("INSERT INTO `one_more_table`")
|
173
|
+
end
|
174
|
+
|
175
|
+
it "should ignore tables that it doesn't know about" do
|
176
|
+
@output_string.should include("INSERT INTO `an_ignored_table` (`col`, `col2`) VALUES ('hello','kjhjd^&dkjh'), ('hello1','kjhj!'), ('hello2','moose!!');")
|
177
|
+
end
|
178
|
+
|
179
|
+
it "should obfuscate the tables" do
|
180
|
+
@output_string.should include("INSERT INTO `some_table` (`email`, `name`, `something`, `age`) VALUES (")
|
181
|
+
@output_string.should include("INSERT INTO `one_more_table` (`a`, `password`, `c`, `d,d`) VALUES (")
|
182
|
+
@output_string.should include("'some\\'thin,ge())lse1'")
|
183
|
+
@output_string.should include("INSERT INTO `one_more_table` (`a`, `password`, `c`, `d,d`) VALUES ('hello','monkey',NULL),('hello1','monkey',NULL),('hello2','monkey',NULL);")
|
184
|
+
@output_string.should_not include("INSERT INTO `one_more_table` (`a`, `password`, `c`, `d,d`) VALUES ('hello','kjhjd^&dkjh', 'aawefjkafe'), ('hello1','kjhj!', 892938), ('hello2','moose!!', NULL);")
|
185
|
+
@output_string.should_not include("INSERT INTO `one_more_table` (`a`, `password`, `c`, `d,d`) VALUES ('hello','kjhjd^&dkjh','aawefjkafe'),('hello1','kjhj!',892938),('hello2','moose!!',NULL);")
|
186
|
+
@output_string.should_not include("INSERT INTO `some_table` (`email`, `name`, `something`, `age`) VALUES ('bob@honk.com','bob', 'some\\'thin,ge())lse1', 25),('joe@joe.com','joe', 'somethingelse2', 54);")
|
187
|
+
end
|
188
|
+
|
189
|
+
it "honors a special case: on the people table, rows with anything@honk.com in a slot marked with :honk_email_skip do not change this slot" do
|
190
|
+
@output_string.should include("('bob@honk.com',")
|
191
|
+
@output_string.should include("('dontmurderme@direwolf.com',")
|
192
|
+
@output_string.should_not include("joe@joe.com")
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: my_obfuscate
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Andrew Cantino
|
8
|
+
- Dave Willett
|
9
|
+
- Mike Grafton
|
10
|
+
- Mason Glaves
|
11
|
+
autorequire:
|
12
|
+
bindir: bin
|
13
|
+
cert_chain: []
|
14
|
+
|
15
|
+
date: 2009-10-01 00:00:00 -07:00
|
16
|
+
default_executable:
|
17
|
+
dependencies:
|
18
|
+
- !ruby/object:Gem::Dependency
|
19
|
+
name: rspec
|
20
|
+
type: :development
|
21
|
+
version_requirement:
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: "0"
|
27
|
+
version:
|
28
|
+
description: Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.
|
29
|
+
email: andrew@pivotallabs.com
|
30
|
+
executables: []
|
31
|
+
|
32
|
+
extensions: []
|
33
|
+
|
34
|
+
extra_rdoc_files:
|
35
|
+
- LICENSE
|
36
|
+
- README.rdoc
|
37
|
+
files:
|
38
|
+
- .document
|
39
|
+
- .gitignore
|
40
|
+
- LICENSE
|
41
|
+
- README.rdoc
|
42
|
+
- Rakefile
|
43
|
+
- VERSION
|
44
|
+
- lib/my_obfuscate.rb
|
45
|
+
- my_obfuscate.gemspec
|
46
|
+
- spec/my_obfuscate_spec.rb
|
47
|
+
- spec/spec_helper.rb
|
48
|
+
has_rdoc: true
|
49
|
+
homepage: http://github.com/honkster/myobfuscate
|
50
|
+
licenses: []
|
51
|
+
|
52
|
+
post_install_message:
|
53
|
+
rdoc_options:
|
54
|
+
- --charset=UTF-8
|
55
|
+
require_paths:
|
56
|
+
- lib
|
57
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: "0"
|
62
|
+
version:
|
63
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: "0"
|
68
|
+
version:
|
69
|
+
requirements: []
|
70
|
+
|
71
|
+
rubyforge_project: my-obfuscate
|
72
|
+
rubygems_version: 1.3.5
|
73
|
+
signing_key:
|
74
|
+
specification_version: 3
|
75
|
+
summary: Standalone Ruby code for the selective rewriting of MySQL dumps in order to protect user privacy.
|
76
|
+
test_files:
|
77
|
+
- spec/my_obfuscate_spec.rb
|
78
|
+
- spec/spec_helper.rb
|