tapsoob 0.4.0-java → 0.4.5-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b7fe4d2055d9856830a8bb0ef1adf1970a99d2e924a13e973dfd4ee85c7a860d
-  data.tar.gz: 454a9fa96ab1dd465897b2324b5e04ecad65b81a640c5a677a24ff3111e3acc5
+  metadata.gz: a477e6b1731da4a418148d7516dad4f6f843b769838b42366f5b18f96afe817e
+  data.tar.gz: fce6cbd5aa1833ae1cd88847250fda83423d25511a18dc62adb600fab15171fc
 SHA512:
-  metadata.gz: fd9b4e500e99061c6d2b139e3a3b0cbedfc227a681c1cecde91df9a9203502eeede62d061594c7ccc6c24fb3e2d1ee42f492fb258a222a7f2f3f7e74fc70bd10
-  data.tar.gz: dd73a21cfc24b0bdba74a9cda436dda14d96ada20caa66af3ff4f65cc84d06799439bf2eb3c44f89f31fa24e93bb6e607cdb8a6b917f958d7cba28ae3b8cf6dc
+  metadata.gz: f08d796cb046da53d9e0663442160fd310401f06d3e3751ed2824009e8cc2a7edfb211e1179699aec1291d5153b53ff7d8c7eca078b33c707f3c0aece0bdd6f8
+  data.tar.gz: 6b96d63974a203090af5d01cc10bba53efeec0a9e8be19a53ddeb8b342f60ef6c6f796d2611a78eaa47ed5587921aa89612676f8ee87f0b033d46cf7e8c911cb

data/lib/tapsoob/cli/data_stream.rb

@@ -29,6 +29,7 @@ module Tapsoob
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
       option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
       option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
+      option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
       option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
       def push(database_url, dump_path = nil)
@@ -69,6 +70,7 @@ module Tapsoob
 
           # Push only options
           opts[:purge] = options[:purge] if options.key?(:purge)
+          opts[:"skip-duplicates"] = options[:"skip-duplicates"] if options.key?(:"skip-duplicates")
           opts[:"discard-identity"] = options[:"discard-identity"] if options.key?(:"discard-identity")
 
           # Default chunksize

data/lib/tapsoob/cli/root.rb

@@ -39,6 +39,7 @@ module Tapsoob
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
       option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
+      option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
       option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
       def push(dump_path, database_url)
@@ -74,6 +75,7 @@ module Tapsoob
 
           # Push only options
           opts[:purge] = options[:purge] if options.key?(:purge)
+          opts[:"skip-duplicates"] = options[:"skip-duplicates"] if options.key?(:"skip-duplicates")
           opts[:"discard-identity"] = options[:"discard-identity"] if options.key?(:"discard-identity")
 
           # Resume

data/lib/tapsoob/data_stream.rb

@@ -91,7 +91,7 @@ module Tapsoob
       rows = {
         :table_name => ds["table_name"],
         :header     => ds["header"],
-        :data       => (ds["data"][state[:offset], (state[:offset] + state[:chunksize])] || [ ]),
+        :data       => ((@options[:"skip-duplicates"] ? ds["data"].uniq : ds["data"])[state[:offset], (state[:offset] + state[:chunksize])] || [ ]),
         :types      => ds["types"]
       }
       update_chunksize_stats
@@ -152,7 +152,7 @@ module Tapsoob
 
       rows = parse_encoded_data(encoded_data, json[:checksum])
 
-      @complete = rows == { }
+      @complete = rows[:data] == [ ]
 
       # update local state
       state.merge!(json[:state].merge(:chunksize => state[:chunksize]))

data/lib/tapsoob/operation.rb

@@ -198,7 +198,7 @@ module Tapsoob
         stream   = Tapsoob::DataStream.factory(db, {
           :chunksize  => default_chunksize,
           :table_name => table_name
-        }, { :debug => opts[:debug] }))
+        }, { :debug => opts[:debug] })
         pull_data_from_table(stream, progress)
       end
     end
@@ -227,8 +227,6 @@ module Tapsoob
               Tapsoob::Utils.export_rows(dump_path, stream.table_name, rows)
             end
           end
-          break if stream.complete?
-          progress.inc(size) if progress && !exiting?
           stream.error = false
           self.stream_state = stream.to_hash
         rescue Tapsoob::CorruptedData => e
@@ -236,6 +234,9 @@ module Tapsoob
           stream.error = true
           next
         end
+
+        progress.inc(size) if progress && !exiting?
+        break if stream.complete?
       end
 
       progress.finish if progress
@@ -393,10 +394,16 @@ module Tapsoob
 
       tables.each do |table_name, count|
         next unless File.exists?(File.join(dump_path, "data", "#{table_name}.json"))
-        db[table.to_sym].truncate if @opts[:purge]
+        db[table_name.to_sym].truncate if @opts[:purge]
         stream = Tapsoob::DataStream.factory(db, {
           :table_name => table_name,
-          :chunksize => default_chunksize }, { :"discard-identity" => opts[:"discard-identity"] || false, :purge => opts[:purge] || false, :debug => opts[:debug] })
+          :chunksize => default_chunksize
+        }, {
+          :"skip-duplicates" => opts[:"skip-duplicates"] || false,
+          :"discard-identity" => opts[:"discard-identity"] || false,
+          :purge => opts[:purge] || false,
+          :debug => opts[:debug]
+        })
         progress = ProgressBar.new(table_name.to_s, count)
         push_data_from_file(stream, progress)
       end

data/lib/tapsoob/progress_bar.rb

@@ -22,7 +22,7 @@ class ProgressBar
     @current = 0
     @previous = 0
     @finished_p = false
-    @start_time = Time.now
+    @start_time = ::Time.now
     @previous_time = @start_time
     @title_width = 14
     @format = "%-#{@title_width}s %3d%% %s %s"
@@ -76,7 +76,7 @@ class ProgressBar
   end
 
   def transfer_rate
-    bytes_per_second = @current.to_f / (Time.now - @start_time)
+    bytes_per_second = @current.to_f / (::Time.now - @start_time)
     sprintf("%s/s", convert_bytes(bytes_per_second))
   end
 
@@ -97,14 +97,14 @@ class ProgressBar
     if @current == 0
       "ETA:  --:--:--"
     else
-      elapsed = Time.now - @start_time
+      elapsed = ::Time.now - @start_time
       eta = elapsed * @total / @current - elapsed;
       sprintf("ETA:  %s", format_time(eta))
     end
   end
 
   def elapsed
-    elapsed = Time.now - @start_time
+    elapsed = ::Time.now - @start_time
     sprintf("Time: %s", format_time(elapsed))
   end
 
@@ -155,7 +155,7 @@ class ProgressBar
       @terminal_width += width - line.length + 1
       show
     end
-    @previous_time = Time.now
+    @previous_time = ::Time.now
   end
 
   def show_if_needed
@@ -169,7 +169,7 @@ class ProgressBar
 
     # Use "!=" instead of ">" to support negative changes
     if cur_percentage != prev_percentage ||
-        Time.now - @previous_time >= 1 || @finished_p
+        ::Time.now - @previous_time >= 1 || @finished_p
       show
     end
   end

data/lib/tapsoob/utils.rb

@@ -1,5 +1,6 @@
 # -*- encoding : utf-8 -*-
 require 'zlib'
+require 'active_support/core_ext/file/atomic.rb'
 
 require 'tapsoob/errors'
 require 'tapsoob/chunksize'
@@ -143,10 +144,10 @@ Data : #{data}
       data = row_data
       if File.exists?(File.join(dump_path, "data", "#{table}.json"))
         previous_data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
-        data[:data] = previous_data["data"] + row_data[:data]
+        data[:data] = (previous_data["data"].nil?  ? rown_data[:data] : previous_data["data"] + row_data[:data])
       end
 
-      File.open(File.join(dump_path, "data", "#{table}.json"), 'w') do |file|
+      File.atomic_write(File.join(dump_path, "data", "#{table}.json")) do |file|
         file.write(JSON.generate(data))
       end
     end

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.4.0".freeze
+  VERSION = "0.4.5".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.5
 platform: java
 authors:
 - Félix Bellanger
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-21 00:00:00.000000000 Z
+date: 2021-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement