tapsoob 0.3.20 → 0.3.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 52d4bfd61d315b90beb05943e4ed8d450ea6fef3f3d8cf8e685a36aace4a67d8
4
- data.tar.gz: 73a623f81cfd0b8b5238ecfa06e0933d0033128ea7bd573f29f801294049ed46
3
+ metadata.gz: cfc8db7ed412920a9e55e43d5ee66c57b0dd09a86e322c0b6f37bd90762ef261
4
+ data.tar.gz: 46dba518c294c033bf7c12c06f79278700be456d00554ad30d9149decaf8b62e
5
5
  SHA512:
6
- metadata.gz: df06895063ad4caf4fd7270a27771dc82f52d2b2d7e035c2a7a5651c2e6dc336e11741285e8a86a12b4af205391e472029c3941cc96412c7f7dbacd3ba6fca07
7
- data.tar.gz: 4ae7a6b34fce4d1f267f8fd95c2cf895608232d9dd687427c3e9ed33a7a35402f33c5db87a7bd9a6cb924cf8c961973fe73303a30d2d3b0f282c84127850c859
6
+ metadata.gz: c2ef008de5be1f5e169055026c65b04518163e7987e34f2e1be5c554d7114be96d99f4fe7ae3284798f4e11225c1554f461db8c9cc487d1befb09ef654e8d0ad
7
+ data.tar.gz: b532c43909abc6afda5314e55da082ddbceb3a3ed285b5ba133f024f969120ec0d17b3d6f75c0423588b3fb61c015763038219f7b6c025d224c340416f7abf6c
@@ -39,6 +39,7 @@ module Tapsoob
39
39
  option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
40
40
  option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
41
41
  option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
42
+ option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
42
43
  option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
43
44
  def push(dump_path, database_url)
44
45
  opts = parse_opts(options)
@@ -71,8 +72,9 @@ module Tapsoob
71
72
  debug: options[:debug]
72
73
  }
73
74
 
74
- # Purge (push only)
75
+ # Push only options
75
76
  opts[:purge] = options[:purge] if options.key?(:purge)
77
+ opts[:"discard-identity"] = options[:"discard-identity"] if options.key?(:"discard-identity")
76
78
 
77
79
  # Resume
78
80
  if options[:resume]
@@ -6,9 +6,9 @@ module Tapsoob
6
6
  class DataStream
7
7
  DEFAULT_CHUNKSIZE = 1000
8
8
 
9
- attr_reader :db, :state
9
+ attr_reader :db, :state, :options
10
10
 
11
- def initialize(db, state)
11
+ def initialize(db, state, opts = {})
12
12
  @db = db
13
13
  @state = {
14
14
  :offset => 0,
@@ -17,6 +17,7 @@ module Tapsoob
17
17
  :total_chunksize => 0
18
18
  }.merge(state)
19
19
  @state[:chunksize] ||= DEFAULT_CHUNKSIZE
20
+ @options = opts
20
21
  @complete = false
21
22
  end
22
23
 
@@ -218,6 +219,9 @@ module Tapsoob
218
219
  end
219
220
 
220
221
  def import_rows(rows)
222
+ columns = rows[:header]
223
+ data = rows[:data]
224
+
221
225
  # Decode blobs
222
226
  if rows.has_key?(:types) && rows[:types].include?("blob")
223
227
  blob_indices = rows[:types].each_index.select { |idx| rows[:types][idx] == "blob" }
@@ -227,8 +231,14 @@ module Tapsoob
227
231
  end
228
232
  end
229
233
  end
234
+
235
+ # Remove id column
236
+ if @options[:"discard-identity"]
237
+ columns = rows[:header] - ["id"]
238
+ data = data.map { |d| d[1..-1] }
239
+ end
230
240
 
231
- table.import(rows[:header], rows[:data], :commit_every => 100)
241
+ table.import(columns, data, :commit_every => 100)
232
242
  state[:offset] += rows[:data].size
233
243
  rescue Exception => ex
234
244
  case ex.message
@@ -246,19 +256,19 @@ module Tapsoob
246
256
  state[:offset] = table.count
247
257
  end
248
258
 
249
- def self.factory(db, state)
259
+ def self.factory(db, state, opts)
250
260
  if defined?(Sequel::MySQL) && Sequel::MySQL.respond_to?(:convert_invalid_date_time=)
251
261
  Sequel::MySQL.convert_invalid_date_time = :nil
252
262
  end
253
263
 
254
264
  if state.has_key?(:klass)
255
- return eval(state[:klass]).new(db, state)
265
+ return eval(state[:klass]).new(db, state, opts)
256
266
  end
257
267
 
258
268
  if Tapsoob::Utils.single_integer_primary_key(db, state[:table_name].to_sym)
259
- DataStreamKeyed.new(db, state)
269
+ DataStreamKeyed.new(db, state, opts)
260
270
  else
261
- DataStream.new(db, state)
271
+ DataStream.new(db, state, opts)
262
272
  end
263
273
  end
264
274
  end
@@ -266,8 +276,8 @@ module Tapsoob
266
276
  class DataStreamKeyed < DataStream
267
277
  attr_accessor :buffer
268
278
 
269
- def initialize(db, state)
270
- super(db, state)
279
+ def initialize(db, state, opts = {})
280
+ super(db, state, opts)
271
281
  @state = { :primary_key => order_by(state[:table_name]).first, :filter => 0 }.merge(@state)
272
282
  @state[:chunksize] ||= DEFAULT_CHUNKSIZE
273
283
  @buffer = []
@@ -394,9 +394,9 @@ module Tapsoob
394
394
  tables.each do |table_name, count|
395
395
  next unless File.exists?(File.join(dump_path, "data", "#{table_name}.json"))
396
396
  db[table_name.to_sym].truncate if @opts[:purge]
397
- stream = Tapsoob::DataStream.factory(db,
397
+ stream = Tapsoob::DataStream.factory(db, {
398
398
  :table_name => table_name,
399
- :chunksize => default_chunksize)
399
+ :chunksize => default_chunksize }, { :"discard-identity" => @opts[:"discard-identity"] || false })
400
400
  progress = ProgressBar.new(table_name.to_s, count)
401
401
  push_data_from_file(stream, progress)
402
402
  end
@@ -477,7 +477,7 @@ module Tapsoob
477
477
 
478
478
  def fetch_local_tables_info
479
479
  tables_with_counts = {}
480
- tbls = Dir.glob(File.join(dump_path, "schemas", "*")).map { |path| File.basename(path, ".rb") }
480
+ tbls = Dir.glob(File.join(dump_path, "data", "*")).map { |path| File.basename(path, ".json") }
481
481
  tbls.each do |table|
482
482
  if File.exists?(File.join(dump_path, "data", "#{table}.json"))
483
483
  data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
@@ -1,4 +1,4 @@
1
1
  # -*- encoding : utf-8 -*-
2
2
  module Tapsoob
3
- VERSION = "0.3.20".freeze
3
+ VERSION = "0.3.25".freeze
4
4
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tapsoob
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.20
4
+ version: 0.3.25
5
5
  platform: ruby
6
6
  authors:
7
7
  - Félix Bellanger