wyrm 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.rvmrc +1 -1
- data/Gemfile +0 -3
- data/README.md +18 -11
- data/bin/wyrm +40 -24
- data/bin/wyrm-view +34 -0
- data/lib/wyrm.rb +3 -5
- data/lib/wyrm/cli.rb +9 -0
- data/lib/wyrm/core_extensions.rb +10 -0
- data/lib/wyrm/{dump_schema.rb → dump.rb} +22 -21
- data/lib/wyrm/hole.rb +164 -0
- data/lib/wyrm/logger.rb +11 -0
- data/lib/wyrm/module.rb +1 -0
- data/lib/wyrm/{db_pump.rb → pump.rb} +40 -34
- data/lib/wyrm/pump_maker.rb +10 -4
- data/lib/wyrm/{restore_schema.rb → restore.rb} +40 -33
- data/lib/wyrm/schema_tools.rb +91 -0
- data/lib/wyrm/version.rb +1 -1
- data/snippets/console.rb +5 -3
- data/spec/core_extensions_spec.rb +50 -0
- data/spec/hole_mouth_spec.rb +176 -0
- data/spec/pump_spec.rb +62 -0
- data/spec/schema_tools_spec.rb +201 -0
- data/wyrm.gemspec +12 -3
- metadata +135 -23
- data/lib/wyrm/other_schema.rb +0 -6
- data/lib/wyrm/transferer.rb +0 -32
data/wyrm.gemspec
CHANGED
|
@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
|
|
|
8
8
|
spec.version = Wyrm::VERSION
|
|
9
9
|
spec.authors = ["John Anderson"]
|
|
10
10
|
spec.email = ["panic@semiosix.com"]
|
|
11
|
-
spec.description = %q{
|
|
12
|
-
spec.summary = %q{Transfer from one SQL
|
|
11
|
+
spec.description = %q{Compressed cross-rdbms data transfer}
|
|
12
|
+
spec.summary = %q{Transfer from one SQL rdbms to another }
|
|
13
13
|
spec.homepage = "https://github.com/djellemah/wyrm"
|
|
14
14
|
spec.license = "MIT"
|
|
15
15
|
|
|
@@ -21,6 +21,15 @@ Gem::Specification.new do |spec|
|
|
|
21
21
|
spec.add_runtime_dependency 'sequel'
|
|
22
22
|
spec.add_runtime_dependency "fastandand"
|
|
23
23
|
|
|
24
|
-
spec.add_development_dependency "
|
|
24
|
+
spec.add_development_dependency "pry"
|
|
25
|
+
spec.add_development_dependency "pry-debugger"
|
|
26
|
+
spec.add_development_dependency "bundler", ">= 1.3"
|
|
25
27
|
spec.add_development_dependency "rake"
|
|
28
|
+
spec.add_development_dependency "rspec"
|
|
29
|
+
spec.add_development_dependency "pg"
|
|
30
|
+
spec.add_development_dependency "sequel_pg"
|
|
31
|
+
spec.add_development_dependency "sqlite3"
|
|
32
|
+
|
|
33
|
+
# version is for mysql streaming result sets
|
|
34
|
+
spec.add_development_dependency "mysql2", '>= 0.3.12'
|
|
26
35
|
end
|
metadata
CHANGED
|
@@ -1,95 +1,203 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: wyrm
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- John Anderson
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2014-03-08 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: sequel
|
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
|
16
16
|
requirements:
|
|
17
|
-
- -
|
|
17
|
+
- - ">="
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
19
|
version: '0'
|
|
20
20
|
type: :runtime
|
|
21
21
|
prerelease: false
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
|
-
- -
|
|
24
|
+
- - ">="
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
26
|
version: '0'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
28
|
name: fastandand
|
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
|
30
30
|
requirements:
|
|
31
|
-
- -
|
|
31
|
+
- - ">="
|
|
32
32
|
- !ruby/object:Gem::Version
|
|
33
33
|
version: '0'
|
|
34
34
|
type: :runtime
|
|
35
35
|
prerelease: false
|
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
37
|
requirements:
|
|
38
|
-
- -
|
|
38
|
+
- - ">="
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: pry
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - ">="
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '0'
|
|
48
|
+
type: :development
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - ">="
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '0'
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: pry-debugger
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - ">="
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: '0'
|
|
62
|
+
type: :development
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - ">="
|
|
39
67
|
- !ruby/object:Gem::Version
|
|
40
68
|
version: '0'
|
|
41
69
|
- !ruby/object:Gem::Dependency
|
|
42
70
|
name: bundler
|
|
43
71
|
requirement: !ruby/object:Gem::Requirement
|
|
44
72
|
requirements:
|
|
45
|
-
- -
|
|
73
|
+
- - ">="
|
|
46
74
|
- !ruby/object:Gem::Version
|
|
47
75
|
version: '1.3'
|
|
48
76
|
type: :development
|
|
49
77
|
prerelease: false
|
|
50
78
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
79
|
requirements:
|
|
52
|
-
- -
|
|
80
|
+
- - ">="
|
|
53
81
|
- !ruby/object:Gem::Version
|
|
54
82
|
version: '1.3'
|
|
55
83
|
- !ruby/object:Gem::Dependency
|
|
56
84
|
name: rake
|
|
57
85
|
requirement: !ruby/object:Gem::Requirement
|
|
58
86
|
requirements:
|
|
59
|
-
- -
|
|
87
|
+
- - ">="
|
|
60
88
|
- !ruby/object:Gem::Version
|
|
61
89
|
version: '0'
|
|
62
90
|
type: :development
|
|
63
91
|
prerelease: false
|
|
64
92
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
93
|
requirements:
|
|
66
|
-
- -
|
|
94
|
+
- - ">="
|
|
95
|
+
- !ruby/object:Gem::Version
|
|
96
|
+
version: '0'
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: rspec
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - ">="
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '0'
|
|
104
|
+
type: :development
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - ">="
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: '0'
|
|
111
|
+
- !ruby/object:Gem::Dependency
|
|
112
|
+
name: pg
|
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
|
114
|
+
requirements:
|
|
115
|
+
- - ">="
|
|
67
116
|
- !ruby/object:Gem::Version
|
|
68
117
|
version: '0'
|
|
69
|
-
|
|
118
|
+
type: :development
|
|
119
|
+
prerelease: false
|
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - ">="
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: '0'
|
|
125
|
+
- !ruby/object:Gem::Dependency
|
|
126
|
+
name: sequel_pg
|
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
|
128
|
+
requirements:
|
|
129
|
+
- - ">="
|
|
130
|
+
- !ruby/object:Gem::Version
|
|
131
|
+
version: '0'
|
|
132
|
+
type: :development
|
|
133
|
+
prerelease: false
|
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
135
|
+
requirements:
|
|
136
|
+
- - ">="
|
|
137
|
+
- !ruby/object:Gem::Version
|
|
138
|
+
version: '0'
|
|
139
|
+
- !ruby/object:Gem::Dependency
|
|
140
|
+
name: sqlite3
|
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
|
142
|
+
requirements:
|
|
143
|
+
- - ">="
|
|
144
|
+
- !ruby/object:Gem::Version
|
|
145
|
+
version: '0'
|
|
146
|
+
type: :development
|
|
147
|
+
prerelease: false
|
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
149
|
+
requirements:
|
|
150
|
+
- - ">="
|
|
151
|
+
- !ruby/object:Gem::Version
|
|
152
|
+
version: '0'
|
|
153
|
+
- !ruby/object:Gem::Dependency
|
|
154
|
+
name: mysql2
|
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
|
156
|
+
requirements:
|
|
157
|
+
- - ">="
|
|
158
|
+
- !ruby/object:Gem::Version
|
|
159
|
+
version: 0.3.12
|
|
160
|
+
type: :development
|
|
161
|
+
prerelease: false
|
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
163
|
+
requirements:
|
|
164
|
+
- - ">="
|
|
165
|
+
- !ruby/object:Gem::Version
|
|
166
|
+
version: 0.3.12
|
|
167
|
+
description: Compressed cross-rdbms data transfer
|
|
70
168
|
email:
|
|
71
169
|
- panic@semiosix.com
|
|
72
170
|
executables:
|
|
73
171
|
- wyrm
|
|
172
|
+
- wyrm-view
|
|
74
173
|
extensions: []
|
|
75
174
|
extra_rdoc_files: []
|
|
76
175
|
files:
|
|
77
|
-
- .gitignore
|
|
78
|
-
- .rvmrc
|
|
176
|
+
- ".gitignore"
|
|
177
|
+
- ".rvmrc"
|
|
79
178
|
- Gemfile
|
|
80
179
|
- LICENSE.txt
|
|
81
180
|
- README.md
|
|
82
181
|
- Rakefile
|
|
83
182
|
- bin/wyrm
|
|
183
|
+
- bin/wyrm-view
|
|
84
184
|
- lib/wyrm.rb
|
|
85
|
-
- lib/wyrm/
|
|
86
|
-
- lib/wyrm/
|
|
87
|
-
- lib/wyrm/
|
|
185
|
+
- lib/wyrm/cli.rb
|
|
186
|
+
- lib/wyrm/core_extensions.rb
|
|
187
|
+
- lib/wyrm/dump.rb
|
|
188
|
+
- lib/wyrm/hole.rb
|
|
189
|
+
- lib/wyrm/logger.rb
|
|
190
|
+
- lib/wyrm/module.rb
|
|
191
|
+
- lib/wyrm/pump.rb
|
|
88
192
|
- lib/wyrm/pump_maker.rb
|
|
89
|
-
- lib/wyrm/
|
|
90
|
-
- lib/wyrm/
|
|
193
|
+
- lib/wyrm/restore.rb
|
|
194
|
+
- lib/wyrm/schema_tools.rb
|
|
91
195
|
- lib/wyrm/version.rb
|
|
92
196
|
- snippets/console.rb
|
|
197
|
+
- spec/core_extensions_spec.rb
|
|
198
|
+
- spec/hole_mouth_spec.rb
|
|
199
|
+
- spec/pump_spec.rb
|
|
200
|
+
- spec/schema_tools_spec.rb
|
|
93
201
|
- wyrm.gemspec
|
|
94
202
|
homepage: https://github.com/djellemah/wyrm
|
|
95
203
|
licenses:
|
|
@@ -101,18 +209,22 @@ require_paths:
|
|
|
101
209
|
- lib
|
|
102
210
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
103
211
|
requirements:
|
|
104
|
-
- -
|
|
212
|
+
- - ">="
|
|
105
213
|
- !ruby/object:Gem::Version
|
|
106
214
|
version: '0'
|
|
107
215
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
108
216
|
requirements:
|
|
109
|
-
- -
|
|
217
|
+
- - ">="
|
|
110
218
|
- !ruby/object:Gem::Version
|
|
111
219
|
version: '0'
|
|
112
220
|
requirements: []
|
|
113
221
|
rubyforge_project:
|
|
114
|
-
rubygems_version: 2.
|
|
222
|
+
rubygems_version: 2.2.2
|
|
115
223
|
signing_key:
|
|
116
224
|
specification_version: 4
|
|
117
|
-
summary: Transfer from one SQL
|
|
118
|
-
test_files:
|
|
225
|
+
summary: Transfer from one SQL rdbms to another
|
|
226
|
+
test_files:
|
|
227
|
+
- spec/core_extensions_spec.rb
|
|
228
|
+
- spec/hole_mouth_spec.rb
|
|
229
|
+
- spec/pump_spec.rb
|
|
230
|
+
- spec/schema_tools_spec.rb
|
data/lib/wyrm/other_schema.rb
DELETED
data/lib/wyrm/transferer.rb
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
class Transferer
|
|
2
|
-
def transfer_table( table_name, options = {} )
|
|
3
|
-
options = OpenStruct.new( {page_size: 10000, dry_run: false}.merge( options ) )
|
|
4
|
-
total_records = @src_db[table_name].count
|
|
5
|
-
logger.info "transferring #{total_records}"
|
|
6
|
-
column_names = @src_db.schema(table_name.to_sym).map( &:first )
|
|
7
|
-
|
|
8
|
-
@src_db[table_name].each_page(options.page_size) do |page|
|
|
9
|
-
logger.info "#{page.sql} of #{total_records}"
|
|
10
|
-
unless options.dry_run
|
|
11
|
-
@dst_db.transaction do
|
|
12
|
-
rows_ary = []
|
|
13
|
-
page.each do |row_hash|
|
|
14
|
-
rows_ary << row_hash.values
|
|
15
|
-
end
|
|
16
|
-
@dst_db[table_name.to_sym].import column_names, rows_ary
|
|
17
|
-
end
|
|
18
|
-
end
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
# copy the data in the tables
|
|
23
|
-
def transfer
|
|
24
|
-
create
|
|
25
|
-
transfer_tables
|
|
26
|
-
index
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def self.transfer( src_db, dst_db )
|
|
30
|
-
new( src_db, dst_db ).transfer
|
|
31
|
-
end
|
|
32
|
-
end
|