tapsoob 0.3.16 → 0.3.21
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/tapsoob/cli/root.rb +4 -0
- data/lib/tapsoob/data_stream.rb +10 -0
- data/lib/tapsoob/operation.rb +2 -1
- data/lib/tapsoob/utils.rb +12 -6
- data/lib/tapsoob/version.rb +1 -1
- data/tapsoob.gemspec +1 -1
- metadata +5 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7109b252c22de2c4c796068f6ac2fc4df267e4b1130ee875a40b08f9ee7a1224
|
4
|
+
data.tar.gz: 5530cb2317ae8a4829335b20e90bbb1788a546f898839a1f330c379ecd2973c1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cf1960e1ef01a40a0e22f66a042616d7a16b3030f8c7ad20227b7ab44805b175872ec3b030d4e1a3c08469ccf3a33f606b8bcdd14bfddb58ce2378ade773dfac
|
7
|
+
data.tar.gz: 9abb44b2ca81049159c78dc771938bb13e8334f008f51d9c899f5b8c22928621d003a429537fc639eef713d3005ce9804731b09db95d5bde24f97d2615dd7bef
|
data/lib/tapsoob/cli/root.rb
CHANGED
@@ -38,6 +38,7 @@ module Tapsoob
|
|
38
38
|
option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
|
39
39
|
option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
|
40
40
|
option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
|
41
|
+
option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
|
41
42
|
option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
|
42
43
|
def push(dump_path, database_url)
|
43
44
|
opts = parse_opts(options)
|
@@ -70,6 +71,9 @@ module Tapsoob
|
|
70
71
|
debug: options[:debug]
|
71
72
|
}
|
72
73
|
|
74
|
+
# Purge (push only)
|
75
|
+
opts[:purge] = options[:purge] if options.key?(:purge)
|
76
|
+
|
73
77
|
# Resume
|
74
78
|
if options[:resume]
|
75
79
|
if File.exists?(options[:resume])
|
data/lib/tapsoob/data_stream.rb
CHANGED
@@ -218,6 +218,16 @@ module Tapsoob
|
|
218
218
|
end
|
219
219
|
|
220
220
|
def import_rows(rows)
|
221
|
+
# Decode blobs
|
222
|
+
if rows.has_key?(:types) && rows[:types].include?("blob")
|
223
|
+
blob_indices = rows[:types].each_index.select { |idx| rows[:types][idx] == "blob" }
|
224
|
+
rows[:data].each_index do |idx|
|
225
|
+
blob_indices.each do |bi|
|
226
|
+
rows[:data][idx][bi] = Sequel::SQL::Blob.new(Tapsoob::Utils.base64decode(rows[:data][idx][bi])) unless rows[:data][idx][bi].nil?
|
227
|
+
end
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
221
231
|
table.import(rows[:header], rows[:data], :commit_every => 100)
|
222
232
|
state[:offset] += rows[:data].size
|
223
233
|
rescue Exception => ex
|
data/lib/tapsoob/operation.rb
CHANGED
@@ -393,6 +393,7 @@ module Tapsoob
|
|
393
393
|
|
394
394
|
tables.each do |table_name, count|
|
395
395
|
next unless File.exists?(File.join(dump_path, "data", "#{table_name}.json"))
|
396
|
+
db[table_name.to_sym].truncate if @opts[:purge]
|
396
397
|
stream = Tapsoob::DataStream.factory(db,
|
397
398
|
:table_name => table_name,
|
398
399
|
:chunksize => default_chunksize)
|
@@ -476,7 +477,7 @@ module Tapsoob
|
|
476
477
|
|
477
478
|
def fetch_local_tables_info
|
478
479
|
tables_with_counts = {}
|
479
|
-
tbls = Dir.glob(File.join(dump_path, "
|
480
|
+
tbls = Dir.glob(File.join(dump_path, "data", "*")).map { |path| File.basename(path, ".json") }
|
480
481
|
tbls.each do |table|
|
481
482
|
if File.exists?(File.join(dump_path, "data", "#{table}.json"))
|
482
483
|
data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
|
data/lib/tapsoob/utils.rb
CHANGED
@@ -51,7 +51,7 @@ module Tapsoob
|
|
51
51
|
|
52
52
|
header = data[0].keys
|
53
53
|
only_data = data.collect do |row|
|
54
|
-
row =
|
54
|
+
row = encode_blobs(row, string_columns)
|
55
55
|
row.each do |column, data|
|
56
56
|
if data.to_s.length > (max_lengths[column] || data.to_s.length)
|
57
57
|
raise Tapsoob::InvalidData.new(<<-ERROR)
|
@@ -70,27 +70,33 @@ Data : #{data}
|
|
70
70
|
end
|
71
71
|
header.collect { |h| row[h] }
|
72
72
|
end
|
73
|
-
|
73
|
+
|
74
|
+
res = { table_name: table, header: header, data: only_data }
|
75
|
+
|
76
|
+
# Add types if schema isn't empty
|
77
|
+
res[:types] = schema.map { |c| c.last[:type] } unless schema.empty?
|
78
|
+
|
79
|
+
res
|
74
80
|
end
|
75
81
|
|
76
82
|
# mysql text and blobs fields are handled the same way internally
|
77
83
|
# this is not true for other databases so we must check if the field is
|
78
84
|
# actually text and manually convert it back to a string
|
79
85
|
def incorrect_blobs(db, table)
|
80
|
-
return [] if (db.url =~ /mysql:\/\//).nil?
|
86
|
+
return [] if (db.url =~ /(mysql|mysql2):\/\//).nil?
|
81
87
|
|
82
88
|
columns = []
|
83
89
|
db.schema(table).each do |data|
|
84
90
|
column, cdata = data
|
85
|
-
columns << column if cdata[:
|
91
|
+
columns << column if cdata[:type] == :blob
|
86
92
|
end
|
87
93
|
columns
|
88
94
|
end
|
89
95
|
|
90
|
-
def
|
96
|
+
def encode_blobs(row, columns)
|
91
97
|
return row if columns.size == 0
|
92
98
|
columns.each do |c|
|
93
|
-
row[c] = row[c]
|
99
|
+
row[c] = base64encode(row[c]) unless row[c].nil?
|
94
100
|
end
|
95
101
|
row
|
96
102
|
end
|
data/lib/tapsoob/version.rb
CHANGED
data/tapsoob.gemspec
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tapsoob
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.21
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Félix Bellanger
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2021-06-11 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: ripl
|
@@ -31,14 +31,14 @@ dependencies:
|
|
31
31
|
requirements:
|
32
32
|
- - "~>"
|
33
33
|
- !ruby/object:Gem::Version
|
34
|
-
version: 5.
|
34
|
+
version: 5.37.0
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
39
|
- - "~>"
|
40
40
|
- !ruby/object:Gem::Version
|
41
|
-
version: 5.
|
41
|
+
version: 5.37.0
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
name: thor
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
@@ -149,8 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
149
149
|
- !ruby/object:Gem::Version
|
150
150
|
version: '0'
|
151
151
|
requirements: []
|
152
|
-
|
153
|
-
rubygems_version: 2.7.6.2
|
152
|
+
rubygems_version: 3.1.6
|
154
153
|
signing_key:
|
155
154
|
specification_version: 4
|
156
155
|
summary: Simple tool to import/export databases.
|