sp-squealer 1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +3 -0
- data/.rvmrc +6 -0
- data/.watchr +27 -0
- data/LICENSE +20 -0
- data/README.md +83 -0
- data/Rakefile +44 -0
- data/VERSION +1 -0
- data/bin/skewer +161 -0
- data/lib/example_squeal.rb +59 -0
- data/lib/squealer.rb +6 -0
- data/lib/squealer/database.rb +87 -0
- data/lib/squealer/hash.rb +7 -0
- data/lib/squealer/object.rb +34 -0
- data/lib/squealer/progress_bar.rb +122 -0
- data/lib/squealer/target.rb +177 -0
- data/lib/tasks/jeweler.rake +15 -0
- data/spec/integration/export_a_record_spec.rb +136 -0
- data/spec/integration/imports_from_mongodb.rb +166 -0
- data/spec/integration/spec_helper_dbms.rb +111 -0
- data/spec/integration/spec_helper_dbms_mysql.rb +50 -0
- data/spec/integration/spec_helper_dbms_postgres.rb +49 -0
- data/spec/spec.opts +3 -0
- data/spec/spec_helper.rb +16 -0
- data/spec/squealer/database_spec.rb +4 -0
- data/spec/squealer/hash_spec.rb +28 -0
- data/spec/squealer/object_spec.rb +119 -0
- data/spec/squealer/progress_bar_spec.rb +254 -0
- data/spec/squealer/target_spec.rb +288 -0
- data/squealer.gemspec +99 -0
- metadata +201 -0
@@ -0,0 +1,34 @@
|
|
1
|
+
class Object
|
2
|
+
|
3
|
+
def target(table_name, &block)
|
4
|
+
Squealer::Target.new(Squealer::Database.instance.export, table_name, &block)
|
5
|
+
end
|
6
|
+
|
7
|
+
def assign(column_name, &block)
|
8
|
+
Squealer::Target.current.assign(column_name, &block)
|
9
|
+
end
|
10
|
+
|
11
|
+
def import(*args)
|
12
|
+
if args.length > 0
|
13
|
+
Squealer::Database.instance.import_from(*args)
|
14
|
+
else
|
15
|
+
Squealer::Database.instance.import
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def export(*args)
|
20
|
+
if args.length > 0
|
21
|
+
Squealer::Database.instance.export_to(*args)
|
22
|
+
else
|
23
|
+
Squealer::Database.instance.export
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
class NilClass
|
30
|
+
include Enumerable
|
31
|
+
def each
|
32
|
+
[]
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,122 @@
|
|
1
|
+
module Squealer
|
2
|
+
class ProgressBar
|
3
|
+
|
4
|
+
@@progress_bar = nil
|
5
|
+
|
6
|
+
def self.new(*args)
|
7
|
+
if @@progress_bar
|
8
|
+
nil
|
9
|
+
else
|
10
|
+
@@progress_bar = super
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(total)
|
15
|
+
@total = total
|
16
|
+
@ticks = 0
|
17
|
+
|
18
|
+
@progress_bar_width = 50
|
19
|
+
@count_width = total.to_s.size
|
20
|
+
end
|
21
|
+
|
22
|
+
def start
|
23
|
+
@start_time = clock
|
24
|
+
@emitter = start_emitter if total > 0
|
25
|
+
self
|
26
|
+
end
|
27
|
+
|
28
|
+
def finish
|
29
|
+
@end_time = clock
|
30
|
+
@emitter.wakeup.join if @emitter
|
31
|
+
@@progress_bar = nil
|
32
|
+
end
|
33
|
+
|
34
|
+
def tick
|
35
|
+
@ticks += 1
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def start_emitter
|
41
|
+
Thread.new do
|
42
|
+
emit
|
43
|
+
sleep(1) and emit until done?
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def emit
|
48
|
+
format = "\r[%-#{progress_bar_width}s] %#{count_width}i/%i (%i%%) [%i/s]"
|
49
|
+
console.print format % [progress_markers, ticks, total, percentage, tps]
|
50
|
+
emit_final if done?
|
51
|
+
end
|
52
|
+
|
53
|
+
def emit_final
|
54
|
+
console.puts
|
55
|
+
|
56
|
+
console.puts "Start: #{start_time}"
|
57
|
+
console.puts "End: #{end_time}"
|
58
|
+
console.puts "Duration: #{duration}"
|
59
|
+
end
|
60
|
+
|
61
|
+
def clock
|
62
|
+
Time.now
|
63
|
+
end
|
64
|
+
|
65
|
+
def done?
|
66
|
+
ticks >= total || end_time
|
67
|
+
end
|
68
|
+
|
69
|
+
def start_time
|
70
|
+
@start_time
|
71
|
+
end
|
72
|
+
|
73
|
+
def end_time
|
74
|
+
@end_time
|
75
|
+
end
|
76
|
+
|
77
|
+
def ticks
|
78
|
+
@ticks
|
79
|
+
end
|
80
|
+
|
81
|
+
def tps
|
82
|
+
elapsed_secs = (clock - start_time) / 60
|
83
|
+
(ticks / elapsed_secs).ceil
|
84
|
+
rescue
|
85
|
+
0
|
86
|
+
end
|
87
|
+
|
88
|
+
def total
|
89
|
+
@total
|
90
|
+
end
|
91
|
+
|
92
|
+
def percentage
|
93
|
+
((ticks.to_f / total) * 100).floor
|
94
|
+
end
|
95
|
+
|
96
|
+
def progress_markers
|
97
|
+
"=" * ((ticks.to_f / total) * progress_bar_width).floor
|
98
|
+
end
|
99
|
+
|
100
|
+
def console
|
101
|
+
$stderr
|
102
|
+
end
|
103
|
+
|
104
|
+
def progress_bar_width
|
105
|
+
@progress_bar_width
|
106
|
+
end
|
107
|
+
|
108
|
+
def count_width
|
109
|
+
@count_width
|
110
|
+
end
|
111
|
+
|
112
|
+
def total_time
|
113
|
+
@end_time - @start_time
|
114
|
+
end
|
115
|
+
|
116
|
+
def duration
|
117
|
+
duration = Time.at(total_time).utc
|
118
|
+
duration.strftime("%H:%M:%S.#{duration.usec}")
|
119
|
+
end
|
120
|
+
|
121
|
+
end
|
122
|
+
end
|
@@ -0,0 +1,177 @@
|
|
1
|
+
require 'delegate'
|
2
|
+
require 'singleton'
|
3
|
+
|
4
|
+
#TODO: Use logger and log throughout
|
5
|
+
|
6
|
+
module Squealer
|
7
|
+
class Target
|
8
|
+
|
9
|
+
def self.current
|
10
|
+
Queue.instance.current
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(database_connection, table_name, &block)
|
14
|
+
raise BlockRequired, "Block must be given to target (otherwise, there's no work to do)" unless block_given?
|
15
|
+
raise ArgumentError, "Table name must be supplied" if table_name.to_s.strip.empty?
|
16
|
+
|
17
|
+
@dbc = database_connection
|
18
|
+
@table_name = table_name.to_s
|
19
|
+
@binding = block.binding
|
20
|
+
|
21
|
+
verify_table_name_in_scope
|
22
|
+
@row_id = infer_row_id
|
23
|
+
@column_names = []
|
24
|
+
@column_values = []
|
25
|
+
@sql = ''
|
26
|
+
|
27
|
+
target(&block)
|
28
|
+
end
|
29
|
+
|
30
|
+
def sql
|
31
|
+
@sql
|
32
|
+
end
|
33
|
+
|
34
|
+
def assign(column_name, &block)
|
35
|
+
@column_names << column_name
|
36
|
+
if block_given?
|
37
|
+
@column_values << yield
|
38
|
+
else
|
39
|
+
@column_values << infer_value(column_name, @binding)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def infer_row_id
|
47
|
+
(
|
48
|
+
(eval "#{@table_name}[:_id]", @binding, __FILE__, __LINE__) ||
|
49
|
+
(eval "#{@table_name}['_id']", @binding, __FILE__, __LINE__)
|
50
|
+
).to_s
|
51
|
+
end
|
52
|
+
|
53
|
+
def verify_table_name_in_scope
|
54
|
+
table = eval "#{@table_name}", @binding, __FILE__, __LINE__
|
55
|
+
raise ArgumentError, "The variable '#{@table_name}' is not a hashmap" unless table.is_a? Hash
|
56
|
+
raise ArgumentError, "The hashmap '#{@table_name}' must have an '_id' key" unless table.has_key?('_id') || table.has_key?(:_id)
|
57
|
+
rescue NameError
|
58
|
+
raise NameError, "A variable named '#{@table_name}' must be in scope, and reference a hashmap with at least an '_id' key."
|
59
|
+
end
|
60
|
+
|
61
|
+
|
62
|
+
def infer_value(column_name, binding)
|
63
|
+
value = eval "#{@table_name}.#{column_name}", binding, __FILE__, __LINE__
|
64
|
+
unless value
|
65
|
+
name = column_name.to_s
|
66
|
+
if name =~ /_id$/
|
67
|
+
related = name[0..-4] #strip "_id"
|
68
|
+
value = eval "#{related}._id", binding, __FILE__, __LINE__
|
69
|
+
end
|
70
|
+
end
|
71
|
+
value
|
72
|
+
end
|
73
|
+
|
74
|
+
def target
|
75
|
+
Queue.instance.push(self)
|
76
|
+
|
77
|
+
yield self
|
78
|
+
|
79
|
+
insert_statement = %{INSERT INTO "#{@table_name}"}
|
80
|
+
insert_statement << %{ (#{pk_name}#{column_names}) VALUES ('#{@row_id}'#{column_value_markers})}
|
81
|
+
if Database.instance.upsertable?
|
82
|
+
insert_statement << %{ ON DUPLICATE KEY UPDATE #{column_markers}}
|
83
|
+
@sql = insert_statement
|
84
|
+
else
|
85
|
+
update_statement = %{UPDATE "#{@table_name}" SET #{column_markers} WHERE #{pk_name}='#{@row_id}'}
|
86
|
+
process_sql(update_statement)
|
87
|
+
@sql = update_statement + "; " + insert_statement
|
88
|
+
end
|
89
|
+
|
90
|
+
process_sql(insert_statement)
|
91
|
+
|
92
|
+
Queue.instance.pop
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.targets
|
96
|
+
@@targets
|
97
|
+
end
|
98
|
+
|
99
|
+
def targets
|
100
|
+
@@targets
|
101
|
+
end
|
102
|
+
|
103
|
+
def process_sql(sql)
|
104
|
+
values = Database.instance.upsertable? ? typecast_values * 2 : typecast_values
|
105
|
+
execute_sql(sql, values)
|
106
|
+
end
|
107
|
+
|
108
|
+
def execute_sql(sql, values)
|
109
|
+
@dbc.create_command(sql).execute_non_query(*values)
|
110
|
+
rescue DataObjects::IntegrityError
|
111
|
+
raise "Failed to execute statement: #{sql} with #{values.inspect}.\nOriginal Exception was: #{$!.to_s}" if Database.instance.upsertable?
|
112
|
+
rescue
|
113
|
+
raise "Failed to execute statement: #{sql} with #{values.inspect}.\nOriginal Exception was: #{$!.to_s}"
|
114
|
+
end
|
115
|
+
|
116
|
+
def pk_name
|
117
|
+
'id'
|
118
|
+
end
|
119
|
+
|
120
|
+
def column_names
|
121
|
+
return if @column_names.size == 0
|
122
|
+
",#{@column_names.map { |name| quote_identifier(name) }.join(',')}"
|
123
|
+
end
|
124
|
+
|
125
|
+
def column_values
|
126
|
+
@column_values
|
127
|
+
end
|
128
|
+
|
129
|
+
def column_value_markers
|
130
|
+
return if @column_names.size == 0
|
131
|
+
result = ""
|
132
|
+
@column_names.size.times { result << ',?'}
|
133
|
+
result
|
134
|
+
end
|
135
|
+
|
136
|
+
def column_markers
|
137
|
+
return if @column_names.size == 0
|
138
|
+
result = ""
|
139
|
+
@column_names.each {|k| result << "#{quote_identifier(k)}=?," }
|
140
|
+
result.chop
|
141
|
+
end
|
142
|
+
|
143
|
+
def typecast_values
|
144
|
+
column_values.map do |value|
|
145
|
+
case value
|
146
|
+
when Array
|
147
|
+
value.join(",")
|
148
|
+
when BSON::ObjectId
|
149
|
+
value.to_s
|
150
|
+
else
|
151
|
+
value
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def quote_identifier(name)
|
157
|
+
%{"#{name}"}
|
158
|
+
end
|
159
|
+
|
160
|
+
class Queue < DelegateClass(Array)
|
161
|
+
include Singleton
|
162
|
+
|
163
|
+
def current
|
164
|
+
last
|
165
|
+
end
|
166
|
+
|
167
|
+
protected
|
168
|
+
|
169
|
+
def initialize
|
170
|
+
super([])
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
class BlockRequired < ArgumentError; end
|
175
|
+
|
176
|
+
end
|
177
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
begin
|
2
|
+
require 'jeweler'
|
3
|
+
Jeweler::Tasks.new do |gemspec|
|
4
|
+
gemspec.name = "squealer"
|
5
|
+
gemspec.summary = "Document-oriented to Relational database exporter"
|
6
|
+
gemspec.description = "Exports mongodb to mysql. More later."
|
7
|
+
gemspec.email = "joshua.graham@grahamis.com"
|
8
|
+
gemspec.homepage = "http://github.com/deltiscere/squealer/"
|
9
|
+
gemspec.authors = ["Josh Graham", "Durran Jordan"]
|
10
|
+
gem.add_dependency('mysql', '>= 2.8.1')
|
11
|
+
gem.add_dependency('mongo', '>= 0.18.3')
|
12
|
+
end
|
13
|
+
rescue LoadError
|
14
|
+
puts "Jeweler not available. Install it with: gem install jeweler"
|
15
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/./spec_helper_dbms')
|
2
|
+
|
3
|
+
describe "Exporting" do
|
4
|
+
let(:databases) { Squealer::Database.instance }
|
5
|
+
let(:today) { Date.today }
|
6
|
+
|
7
|
+
def squeal_basic_users_document(user=users_document)
|
8
|
+
target(:user) do
|
9
|
+
assign(:name)
|
10
|
+
assign(:organization_id)
|
11
|
+
assign(:dob)
|
12
|
+
assign(:gender)
|
13
|
+
assign(:foreign)
|
14
|
+
assign(:dull)
|
15
|
+
assign(:symbolic)
|
16
|
+
assign(:interests)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def squeal_users_document_with_activities(user=users_document)
|
21
|
+
target(:user) do
|
22
|
+
assign(:name)
|
23
|
+
assign(:organization_id)
|
24
|
+
assign(:dob)
|
25
|
+
assign(:gender)
|
26
|
+
assign(:foreign)
|
27
|
+
assign(:dull)
|
28
|
+
assign(:symbolic)
|
29
|
+
assign(:interests)
|
30
|
+
|
31
|
+
user.activities.each do |activity|
|
32
|
+
target(:activity) do
|
33
|
+
assign(:name)
|
34
|
+
assign(:due_date)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
let :users_document do
|
41
|
+
{ :_id => 'ABCDEFGHIJKLMNOPQRSTUVWX',
|
42
|
+
'name' => 'Test User', 'dob' => as_time(Date.parse('04-Jul-1776')), 'gender' => 'M',
|
43
|
+
'foreign' => true,
|
44
|
+
'dull' => false,
|
45
|
+
'symbolic' => :of_course,
|
46
|
+
'interests' => ['health', 'education'],
|
47
|
+
'organization_id' => '123456789012345678901234',
|
48
|
+
'activities' => [
|
49
|
+
{ :_id => 'a1', 'name' => 'Be independent', 'due_date' => as_time(today + 1) },
|
50
|
+
{ :_id => 'a2', 'name' => 'Fight each other', 'due_date' => as_time(today + 7) }
|
51
|
+
]
|
52
|
+
}
|
53
|
+
end
|
54
|
+
|
55
|
+
let :first_users_record do
|
56
|
+
dbc = databases.export
|
57
|
+
reader = dbc.create_command(%{SELECT * FROM "user"}).execute_reader
|
58
|
+
result = reader.each { |x| break x }
|
59
|
+
reader.close
|
60
|
+
result
|
61
|
+
end
|
62
|
+
|
63
|
+
let :first_activity_record do
|
64
|
+
dbc = databases.export
|
65
|
+
reader = dbc.create_command(%{SELECT * FROM "activity"}).execute_reader
|
66
|
+
result = reader.each { |x| break x }
|
67
|
+
reader.close
|
68
|
+
result
|
69
|
+
end
|
70
|
+
|
71
|
+
context "a new record" do
|
72
|
+
it "saves the data correctly" do
|
73
|
+
squeal_basic_users_document
|
74
|
+
result = first_users_record
|
75
|
+
|
76
|
+
result['name'].should == 'Test User'
|
77
|
+
|
78
|
+
result['dob'].mday.should == 4
|
79
|
+
result['dob'].mon.should == 7
|
80
|
+
result['dob'].year.should == 1776
|
81
|
+
|
82
|
+
result['gender'].should == 'M'
|
83
|
+
|
84
|
+
result['foreign'].should be_true
|
85
|
+
result['dull'].should be_false
|
86
|
+
|
87
|
+
result['symbolic'].should == :of_course.to_s
|
88
|
+
|
89
|
+
result['interests'].should == 'health,education'
|
90
|
+
end
|
91
|
+
|
92
|
+
it "saves embedded documents correctly" do
|
93
|
+
squeal_users_document_with_activities
|
94
|
+
result = first_activity_record
|
95
|
+
|
96
|
+
result['name'].should == 'Be independent'
|
97
|
+
result['due_date'].mday.should == (today + 1).mday
|
98
|
+
result['due_date'].mon.should == (today + 1).mon
|
99
|
+
result['due_date'].year.should == (today + 1).year
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
context "an existing record" do
|
104
|
+
it "updates the data correctly" do
|
105
|
+
squeal_basic_users_document
|
106
|
+
squeal_basic_users_document(users_document.merge('foreign' => false, 'gender' => 'F'))
|
107
|
+
|
108
|
+
result = first_users_record
|
109
|
+
|
110
|
+
result['name'].should == 'Test User'
|
111
|
+
|
112
|
+
result['dob'].mday.should == 4
|
113
|
+
result['dob'].mon.should == 7
|
114
|
+
result['dob'].year.should == 1776
|
115
|
+
|
116
|
+
result['gender'].should == 'F'
|
117
|
+
|
118
|
+
result['foreign'].should be_false
|
119
|
+
result['dull'].should be_false
|
120
|
+
|
121
|
+
result['symbolic'].should == :of_course.to_s
|
122
|
+
|
123
|
+
result['interests'].should == 'health,education'
|
124
|
+
end
|
125
|
+
|
126
|
+
it "updates the child record correctly" do
|
127
|
+
squeal_users_document_with_activities(users_document.merge('activities' => [{ :_id => 'a1', 'name' => 'Be expansionist', 'due_date' => as_time(today + 1) }]))
|
128
|
+
result = first_activity_record
|
129
|
+
|
130
|
+
result['name'].should == 'Be expansionist'
|
131
|
+
result['due_date'].mday.should == (today + 1).mday
|
132
|
+
result['due_date'].mon.should == (today + 1).mon
|
133
|
+
result['due_date'].year.should == (today + 1).year
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|