tapsoob 0.3.20 → 0.3.25
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/tapsoob/cli/root.rb +3 -1
- data/lib/tapsoob/data_stream.rb +19 -9
- data/lib/tapsoob/operation.rb +3 -3
- data/lib/tapsoob/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cfc8db7ed412920a9e55e43d5ee66c57b0dd09a86e322c0b6f37bd90762ef261
|
4
|
+
data.tar.gz: 46dba518c294c033bf7c12c06f79278700be456d00554ad30d9149decaf8b62e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c2ef008de5be1f5e169055026c65b04518163e7987e34f2e1be5c554d7114be96d99f4fe7ae3284798f4e11225c1554f461db8c9cc487d1befb09ef654e8d0ad
|
7
|
+
data.tar.gz: b532c43909abc6afda5314e55da082ddbceb3a3ed285b5ba133f024f969120ec0d17b3d6f75c0423588b3fb61c015763038219f7b6c025d224c340416f7abf6c
|
data/lib/tapsoob/cli/root.rb
CHANGED
@@ -39,6 +39,7 @@ module Tapsoob
|
|
39
39
|
option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
|
40
40
|
option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
|
41
41
|
option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
|
42
|
+
option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
|
42
43
|
option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
|
43
44
|
def push(dump_path, database_url)
|
44
45
|
opts = parse_opts(options)
|
@@ -71,8 +72,9 @@ module Tapsoob
|
|
71
72
|
debug: options[:debug]
|
72
73
|
}
|
73
74
|
|
74
|
-
#
|
75
|
+
# Push only options
|
75
76
|
opts[:purge] = options[:purge] if options.key?(:purge)
|
77
|
+
opts[:"discard-identity"] = options[:"discard-identity"] if options.key?(:"discard-identity")
|
76
78
|
|
77
79
|
# Resume
|
78
80
|
if options[:resume]
|
data/lib/tapsoob/data_stream.rb
CHANGED
@@ -6,9 +6,9 @@ module Tapsoob
|
|
6
6
|
class DataStream
|
7
7
|
DEFAULT_CHUNKSIZE = 1000
|
8
8
|
|
9
|
-
attr_reader :db, :state
|
9
|
+
attr_reader :db, :state, :options
|
10
10
|
|
11
|
-
def initialize(db, state)
|
11
|
+
def initialize(db, state, opts = {})
|
12
12
|
@db = db
|
13
13
|
@state = {
|
14
14
|
:offset => 0,
|
@@ -17,6 +17,7 @@ module Tapsoob
|
|
17
17
|
:total_chunksize => 0
|
18
18
|
}.merge(state)
|
19
19
|
@state[:chunksize] ||= DEFAULT_CHUNKSIZE
|
20
|
+
@options = opts
|
20
21
|
@complete = false
|
21
22
|
end
|
22
23
|
|
@@ -218,6 +219,9 @@ module Tapsoob
|
|
218
219
|
end
|
219
220
|
|
220
221
|
def import_rows(rows)
|
222
|
+
columns = rows[:header]
|
223
|
+
data = rows[:data]
|
224
|
+
|
221
225
|
# Decode blobs
|
222
226
|
if rows.has_key?(:types) && rows[:types].include?("blob")
|
223
227
|
blob_indices = rows[:types].each_index.select { |idx| rows[:types][idx] == "blob" }
|
@@ -227,8 +231,14 @@ module Tapsoob
|
|
227
231
|
end
|
228
232
|
end
|
229
233
|
end
|
234
|
+
|
235
|
+
# Remove id column
|
236
|
+
if @options[:"discard-identity"]
|
237
|
+
columns = rows[:header] - ["id"]
|
238
|
+
data = data.map { |d| d[1..-1] }
|
239
|
+
end
|
230
240
|
|
231
|
-
table.import(
|
241
|
+
table.import(columns, data, :commit_every => 100)
|
232
242
|
state[:offset] += rows[:data].size
|
233
243
|
rescue Exception => ex
|
234
244
|
case ex.message
|
@@ -246,19 +256,19 @@ module Tapsoob
|
|
246
256
|
state[:offset] = table.count
|
247
257
|
end
|
248
258
|
|
249
|
-
def self.factory(db, state)
|
259
|
+
def self.factory(db, state, opts)
|
250
260
|
if defined?(Sequel::MySQL) && Sequel::MySQL.respond_to?(:convert_invalid_date_time=)
|
251
261
|
Sequel::MySQL.convert_invalid_date_time = :nil
|
252
262
|
end
|
253
263
|
|
254
264
|
if state.has_key?(:klass)
|
255
|
-
return eval(state[:klass]).new(db, state)
|
265
|
+
return eval(state[:klass]).new(db, state, opts)
|
256
266
|
end
|
257
267
|
|
258
268
|
if Tapsoob::Utils.single_integer_primary_key(db, state[:table_name].to_sym)
|
259
|
-
DataStreamKeyed.new(db, state)
|
269
|
+
DataStreamKeyed.new(db, state, opts)
|
260
270
|
else
|
261
|
-
DataStream.new(db, state)
|
271
|
+
DataStream.new(db, state, opts)
|
262
272
|
end
|
263
273
|
end
|
264
274
|
end
|
@@ -266,8 +276,8 @@ module Tapsoob
|
|
266
276
|
class DataStreamKeyed < DataStream
|
267
277
|
attr_accessor :buffer
|
268
278
|
|
269
|
-
def initialize(db, state)
|
270
|
-
super(db, state)
|
279
|
+
def initialize(db, state, opts = {})
|
280
|
+
super(db, state, opts)
|
271
281
|
@state = { :primary_key => order_by(state[:table_name]).first, :filter => 0 }.merge(@state)
|
272
282
|
@state[:chunksize] ||= DEFAULT_CHUNKSIZE
|
273
283
|
@buffer = []
|
data/lib/tapsoob/operation.rb
CHANGED
@@ -394,9 +394,9 @@ module Tapsoob
|
|
394
394
|
tables.each do |table_name, count|
|
395
395
|
next unless File.exists?(File.join(dump_path, "data", "#{table_name}.json"))
|
396
396
|
db[table_name.to_sym].truncate if @opts[:purge]
|
397
|
-
stream = Tapsoob::DataStream.factory(db,
|
397
|
+
stream = Tapsoob::DataStream.factory(db, {
|
398
398
|
:table_name => table_name,
|
399
|
-
:chunksize => default_chunksize)
|
399
|
+
:chunksize => default_chunksize }, { :"discard-identity" => @opts[:"discard-identity"] || false })
|
400
400
|
progress = ProgressBar.new(table_name.to_s, count)
|
401
401
|
push_data_from_file(stream, progress)
|
402
402
|
end
|
@@ -477,7 +477,7 @@ module Tapsoob
|
|
477
477
|
|
478
478
|
def fetch_local_tables_info
|
479
479
|
tables_with_counts = {}
|
480
|
-
tbls = Dir.glob(File.join(dump_path, "
|
480
|
+
tbls = Dir.glob(File.join(dump_path, "data", "*")).map { |path| File.basename(path, ".json") }
|
481
481
|
tbls.each do |table|
|
482
482
|
if File.exists?(File.join(dump_path, "data", "#{table}.json"))
|
483
483
|
data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
|
data/lib/tapsoob/version.rb
CHANGED