tapsoob 0.3.12 → 0.3.18
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +17 -0
- data/lib/tapsoob/data_stream.rb +10 -0
- data/lib/tapsoob/schema.rb +11 -2
- data/lib/tapsoob/utils.rb +12 -6
- data/lib/tapsoob/version.rb +1 -1
- data/tapsoob.gemspec +2 -2
- metadata +10 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d2a5689451bbf6f165c3e6484e249c788c08a53e1534f9da72dadbe6fb09adae
|
4
|
+
data.tar.gz: 993b87e324fdea34d4c748d0463dd1bf16c6c041f786603ced8fb7d675b67f9e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dc71103e4461b0595f074bd6623576fd2f07269447a0e790b3147ec1fd9b48d2bf41aad62750b43c6ccd57fa7a124b62e1b2b7747eab99008fd4f4a216098b04
|
7
|
+
data.tar.gz: 9aa26eb1920ec927921235317fe43089fa5462e786a2f5548e38af2b42f5e31375e452d5c9e994548ac41645d2f4fd4814cc2846718299da1e52f2f37eaccba4
|
data/README.md
CHANGED
@@ -36,6 +36,23 @@ You can list all available options using the command:
|
|
36
36
|
tapsoob push -h
|
37
37
|
|
38
38
|
|
39
|
+
## NEW : Piping your schema/indexes/data
|
40
|
+
|
41
|
+
Due to some needs we added ways to pipe your schema/indexes/data directly from one database to another, here's an equivalent of the export/import process described above using this technique :
|
42
|
+
|
43
|
+
```
|
44
|
+
tapsoob schema dump <db_source_url> | tapsoob schema load <db_target_url> --drop=true
|
45
|
+
tapsoob schema indexes <db_source_url> | tapsoob schema load_indexes <db_target_url>
|
46
|
+
tapsoob data pull <db_source_url> -p false | tapsoob data push <db_target_url>
|
47
|
+
tapsoob schema reset_db_sequences <db_target_url>
|
48
|
+
```
|
49
|
+
|
50
|
+
A few notes here :
|
51
|
+
|
52
|
+
* the `--drop` option for the `schema load` command defaults to false, if you don't intend to drop your tables on your target databases you can ommit it (if a table already exists tapsoob will exit w/ an error) ;
|
53
|
+
* the `data pull` and `data push` commands are new and have a few options that you can display w/ `tapsoob data <pull|push> -h`, by defaults it prints the progress of your data extraction/import but if you want to pipe directly your export into another database you'll have to set the `--progress` (shorthand `-p`) option to `false` (or `--no-progress`) as above ;
|
54
|
+
* resetting database sequences is important as a `data push` command doesn't handle that directly.
|
55
|
+
|
39
56
|
## Integration with Rails
|
40
57
|
|
41
58
|
If you're using Rails, there's also two Rake tasks provided:
|
data/lib/tapsoob/data_stream.rb
CHANGED
@@ -218,6 +218,16 @@ module Tapsoob
|
|
218
218
|
end
|
219
219
|
|
220
220
|
def import_rows(rows)
|
221
|
+
# Decode blobs
|
222
|
+
if rows.has_key?(:types) && rows[:types].include?("blob")
|
223
|
+
blob_indices = rows[:types].each_index.select { |idx| rows[:types][idx] == "blob" }
|
224
|
+
rows[:data].each_index do |idx|
|
225
|
+
blob_indices.each do |bi|
|
226
|
+
rows[:data][idx][bi] = Sequel::SQL::Blob.new(Tapsoob::Utils.base64decode(rows[:data][idx][bi])) unless rows[:data][idx][bi].nil?
|
227
|
+
end
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
221
231
|
table.import(rows[:header], rows[:data], :commit_every => 100)
|
222
232
|
state[:offset] += rows[:data].size
|
223
233
|
rescue Exception => ex
|
data/lib/tapsoob/schema.rb
CHANGED
@@ -22,7 +22,7 @@ Class.new(Sequel::Migration) do
|
|
22
22
|
|
23
23
|
def down
|
24
24
|
<% db.tables.each do |table| %>
|
25
|
-
drop_table(<%= table
|
25
|
+
drop_table("<%= table %>", if_exists: true)
|
26
26
|
<% end %>
|
27
27
|
end
|
28
28
|
end
|
@@ -84,7 +84,16 @@ END_MIG
|
|
84
84
|
Sequel.connect(database_url) do |db|
|
85
85
|
db.extension :schema_dumper
|
86
86
|
klass = eval(schema)
|
87
|
-
|
87
|
+
if options[:drop]
|
88
|
+
# Start special hack for MySQL
|
89
|
+
db.run("SET foreign_key_checks = 0") if [:mysql, :mysql2].include?(db.adapter_scheme)
|
90
|
+
|
91
|
+
# Run down migration
|
92
|
+
klass.apply(db, :down)
|
93
|
+
|
94
|
+
# End special hack for MySQL
|
95
|
+
db.run("SET foreign_key_checks = 1") if [:mysql, :mysql2].include?(db.adapter_scheme)
|
96
|
+
end
|
88
97
|
klass.apply(db, :up)
|
89
98
|
end
|
90
99
|
end
|
data/lib/tapsoob/utils.rb
CHANGED
@@ -51,7 +51,7 @@ module Tapsoob
|
|
51
51
|
|
52
52
|
header = data[0].keys
|
53
53
|
only_data = data.collect do |row|
|
54
|
-
row =
|
54
|
+
row = encode_blobs(row, string_columns)
|
55
55
|
row.each do |column, data|
|
56
56
|
if data.to_s.length > (max_lengths[column] || data.to_s.length)
|
57
57
|
raise Tapsoob::InvalidData.new(<<-ERROR)
|
@@ -70,27 +70,33 @@ Data : #{data}
|
|
70
70
|
end
|
71
71
|
header.collect { |h| row[h] }
|
72
72
|
end
|
73
|
-
|
73
|
+
|
74
|
+
res = { table_name: table, header: header, data: only_data }
|
75
|
+
|
76
|
+
# Add types if schema isn't empty
|
77
|
+
res[:types] = schema.map { |c| c.last[:type] } unless schema.empty?
|
78
|
+
|
79
|
+
res
|
74
80
|
end
|
75
81
|
|
76
82
|
# mysql text and blobs fields are handled the same way internally
|
77
83
|
# this is not true for other databases so we must check if the field is
|
78
84
|
# actually text and manually convert it back to a string
|
79
85
|
def incorrect_blobs(db, table)
|
80
|
-
return [] if (db.url =~ /mysql:\/\//).nil?
|
86
|
+
return [] if (db.url =~ /(mysql|mysql2):\/\//).nil?
|
81
87
|
|
82
88
|
columns = []
|
83
89
|
db.schema(table).each do |data|
|
84
90
|
column, cdata = data
|
85
|
-
columns << column if cdata[:
|
91
|
+
columns << column if cdata[:type] == :blob
|
86
92
|
end
|
87
93
|
columns
|
88
94
|
end
|
89
95
|
|
90
|
-
def
|
96
|
+
def encode_blobs(row, columns)
|
91
97
|
return row if columns.size == 0
|
92
98
|
columns.each do |c|
|
93
|
-
row[c] = row[c]
|
99
|
+
row[c] = base64encode(row[c]) unless row[c].nil?
|
94
100
|
end
|
95
101
|
row
|
96
102
|
end
|
data/lib/tapsoob/version.rb
CHANGED
data/tapsoob.gemspec
CHANGED
@@ -21,8 +21,8 @@ Gem::Specification.new do |s|
|
|
21
21
|
|
22
22
|
# Dependencies
|
23
23
|
s.add_dependency "ripl", "~> 0.7.1"
|
24
|
-
s.add_dependency "sequel", "~> 5.
|
25
|
-
s.add_dependency "thor", "~> 0.
|
24
|
+
s.add_dependency "sequel", "~> 5.36.0"
|
25
|
+
s.add_dependency "thor", "~> 1.0.1"
|
26
26
|
|
27
27
|
if (RUBY_PLATFORM =~ /java/).nil?
|
28
28
|
s.add_development_dependency "mysql2", "~> 0.4.10"
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tapsoob
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.18
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Félix Bellanger
|
8
8
|
- Michael Chrisco
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2020-
|
12
|
+
date: 2020-09-28 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: ripl
|
@@ -31,28 +31,28 @@ dependencies:
|
|
31
31
|
requirements:
|
32
32
|
- - "~>"
|
33
33
|
- !ruby/object:Gem::Version
|
34
|
-
version: 5.
|
34
|
+
version: 5.36.0
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
39
|
- - "~>"
|
40
40
|
- !ruby/object:Gem::Version
|
41
|
-
version: 5.
|
41
|
+
version: 5.36.0
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
name: thor
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
45
45
|
requirements:
|
46
46
|
- - "~>"
|
47
47
|
- !ruby/object:Gem::Version
|
48
|
-
version: 0.
|
48
|
+
version: 1.0.1
|
49
49
|
type: :runtime
|
50
50
|
prerelease: false
|
51
51
|
version_requirements: !ruby/object:Gem::Requirement
|
52
52
|
requirements:
|
53
53
|
- - "~>"
|
54
54
|
- !ruby/object:Gem::Version
|
55
|
-
version: 0.
|
55
|
+
version: 1.0.1
|
56
56
|
- !ruby/object:Gem::Dependency
|
57
57
|
name: mysql2
|
58
58
|
requirement: !ruby/object:Gem::Requirement
|
@@ -134,7 +134,7 @@ homepage: https://github.com/Keeguon/tapsoob
|
|
134
134
|
licenses:
|
135
135
|
- MIT
|
136
136
|
metadata: {}
|
137
|
-
post_install_message:
|
137
|
+
post_install_message:
|
138
138
|
rdoc_options: []
|
139
139
|
require_paths:
|
140
140
|
- lib
|
@@ -149,9 +149,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
149
149
|
- !ruby/object:Gem::Version
|
150
150
|
version: '0'
|
151
151
|
requirements: []
|
152
|
-
|
153
|
-
|
154
|
-
signing_key:
|
152
|
+
rubygems_version: 3.1.4
|
153
|
+
signing_key:
|
155
154
|
specification_version: 4
|
156
155
|
summary: Simple tool to import/export databases.
|
157
156
|
test_files:
|