tapsoob 0.3.26 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59b4084ddb364d71b2409a23fdd8604b3ef46d5dac7a96518fdb664e8f743943
-  data.tar.gz: 4d72e5bae7bd8fe4acf116c81c1a139e5fdde4d7bbd5731bf25f896b71a00a90
+  metadata.gz: 7931ff4460451152355f05ac09c59013ca76b7afe46893620c1da7fdf30288fe
+  data.tar.gz: 0257a6e05c923148436d122f0c2592fd9551cbe23401989e18f720365469d11e
 SHA512:
-  metadata.gz: b9ca0470b23447547e1967588c1230ed3a2f6f141d5d9b3f14854bdeb155329d604082895dd344a27a69540cbe9ac6e3b630e46ae61bb3e23467cc5194011948
-  data.tar.gz: ab7f6656768aa0e078a921530e25f8da5498504d9c5e4abd90175acaeeba4540393b6905021b40187ab9070c479d9fc2db1f88c10a82aa283ac7488c25778e7d
+  metadata.gz: 1251f19e8812daec0ed94c7cbd55d3a22beb3112c91cab288159ccd59fd0144b99888ed9b082df3a14182469f20cbc482656108db75c36c4859d1a889f4165ad
+  data.tar.gz: c440040cf75ea377161394335a32bc9f4bc9cb635775d048ae076efeec69af9cb499601c7f63280cb1d76ecc14d6948f7d2a3b32f746094dfed2aec06bfe4747

data/lib/tapsoob/cli/data_stream.rb

@@ -28,6 +28,9 @@ module Tapsoob
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
       option :progress, desc: "Show progress", default: true, type: :boolean, aliases: "-p"
+      option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
+      option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
+      option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
       def push(database_url, dump_path = nil)
         # instantiate stuff
@@ -46,9 +49,8 @@ module Tapsoob
         data.each do |table|
           stream = Tapsoob::DataStream.factory(db(database_url, opts), {
             table_name: table[:table_name],
-            chunksize: opts[:default_chunksize],
-            debug: opts[:debug]
-          })
+            chunksize: opts[:default_chunksize]
+          }, { :"discard-identity" => opts[:"discard-identity"] || false, :purge => opts[:purge] || false, :debug => opts[:debug] })
 
           begin
             stream.import_rows(table)
@@ -66,6 +68,11 @@ module Tapsoob
             debug: options[:debug]
           }
 
+          # Push only options
+          opts[:purge] = options[:purge] if options.key?(:purge)
+          opts[:"skip-duplicates"] = options[:"skip-duplicates"] if options.key?(:"skip-duplicates")
+          opts[:"discard-identity"] = options[:"discard-identity"] if options.key?(:"discard-identity")
+
           # Default chunksize
           if options[:chunksize]
             opts[:default_chunksize] = (options[:chunksize] < 10 ? 10 : options[:chunksize])

data/lib/tapsoob/cli/root.rb

@@ -39,6 +39,7 @@ module Tapsoob
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
       option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
+      option :"skip-duplicates", desc: "Remove duplicates when loading data", default: false, type: :boolean
       option :"discard-identity", desc: "Remove identity when pushing data (may result in creating duplicates)", default: false, type: :boolean
       option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
       def push(dump_path, database_url)
@@ -74,6 +75,7 @@ module Tapsoob
 
           # Push only options
           opts[:purge] = options[:purge] if options.key?(:purge)
+          opts[:"skip-duplicates"] = options[:"skip-duplicates"] if options.key?(:"skip-duplicates")
           opts[:"discard-identity"] = options[:"discard-identity"] if options.key?(:"discard-identity")
 
           # Resume

data/lib/tapsoob/data_stream.rb

@@ -1,7 +1,4 @@
 # -*- encoding : utf-8 -*-
-require 'date'
-require 'time'
-
 require 'tapsoob/log'
 require 'tapsoob/utils'
 
@@ -94,7 +91,8 @@ module Tapsoob
       rows = {
         :table_name => ds["table_name"],
         :header     => ds["header"],
-        :data       => ds["data"][state[:offset], (state[:offset] + state[:chunksize])] || [ ]
+        :data       => ((@options[:"skip-duplicates"] ? ds["data"].uniq : ds["data"])[state[:offset], (state[:offset] + state[:chunksize])] || [ ]),
+        :types      => ds["types"]
       }
       update_chunksize_stats
       rows
@@ -225,12 +223,24 @@ module Tapsoob
       columns = rows[:header]
       data    = rows[:data]
 
+      # Only import existing columns
+      if table.columns.size != columns.size
+        existing_columns        = table.columns.map(&:to_s)
+        additional_columns      = columns - existing_columns
+        additional_columns_idxs = additional_columns.map { |c| columns.index(c) }
+        additional_columns_idxs.reverse.each do |idx|
+          columns.delete_at(idx)
+          rows[:types].delete_at(idx)
+        end
+        data.each_index { |didx| additional_columns_idxs.reverse.each { |idx| data[didx].delete_at(idx) } }
+      end
+
       # Decode blobs
       if rows.has_key?(:types) && rows[:types].include?("blob")
         blob_indices = rows[:types].each_index.select { |idx| rows[:types][idx] == "blob" }
-        rows[:data].each_index do |idx|
+        data.each_index do |idx|
           blob_indices.each do |bi|
-            rows[:data][idx][bi] = Sequel::SQL::Blob.new(Tapsoob::Utils.base64decode(rows[:data][idx][bi])) unless rows[:data][idx][bi].nil?
+            data[idx][bi] = Sequel::SQL::Blob.new(Tapsoob::Utils.base64decode(data[idx][bi])) unless data[idx][bi].nil?
           end
         end
       end
@@ -239,20 +249,10 @@ module Tapsoob
       if rows.has_key?(:types)
         %w(date datetime time).each do |type|
           if rows[:types].include?(type)
-            klass = case type
-            when "date"
-              Date
-            when "datetime"
-              DateTime
-            when "time"
-              Time
-            end
-
-
             type_indices = rows[:types].each_index.select { |idx| rows[:types][idx] == type }
-            rows[:data].each_index do |idx|
+            data.each_index do |idx|
               type_indices.each do |ti|
-                rows[:data][idx][ti] = klass.parse(rows[:data][idx][ti])
+                data[idx][ti] = Sequel.send("string_to_#{type}".to_sym, data[idx][ti]) unless data[idx][ti].nil?
               end
             end
           end
@@ -260,11 +260,11 @@ module Tapsoob
       end
 
       # Remove id column
-      if @options[:"discard-identity"]
+      if @options[:"discard-identity"] && rows[:header].include?("id")
         columns = rows[:header] - ["id"]
         data    = data.map { |d| d[1..-1] }
       end
-      
+
       table.import(columns, data, :commit_every => 100)
       state[:offset] += rows[:data].size
     rescue Exception => ex

data/lib/tapsoob/operation.rb

@@ -198,7 +198,7 @@ module Tapsoob
         stream   = Tapsoob::DataStream.factory(db, {
           :chunksize  => default_chunksize,
           :table_name => table_name
-        })
+        }, { :debug => opts[:debug] })
         pull_data_from_table(stream, progress)
       end
     end
@@ -227,8 +227,6 @@ module Tapsoob
               Tapsoob::Utils.export_rows(dump_path, stream.table_name, rows)
             end
           end
-          break if stream.complete?
-          progress.inc(size) if progress && !exiting?
           stream.error = false
           self.stream_state = stream.to_hash
         rescue Tapsoob::CorruptedData => e
@@ -236,6 +234,9 @@ module Tapsoob
           stream.error = true
           next
         end
+
+        progress.inc(size) if progress && !exiting?
+        break if stream.complete?
       end
 
       progress.finish if progress
@@ -396,7 +397,13 @@ module Tapsoob
         db[table_name.to_sym].truncate if @opts[:purge]
         stream = Tapsoob::DataStream.factory(db, {
           :table_name => table_name,
-          :chunksize => default_chunksize }, { :"discard-identity" => @opts[:"discard-identity"] || false })
+          :chunksize => default_chunksize
+        }, {
+          :"skip-duplicates" => opts[:"skip-duplicates"] || false,
+          :"discard-identity" => opts[:"discard-identity"] || false,
+          :purge => opts[:purge] || false,
+          :debug => opts[:debug]
+        })
         progress = ProgressBar.new(table_name.to_s, count)
         push_data_from_file(stream, progress)
       end
@@ -429,7 +436,8 @@ module Tapsoob
               }
             end
 
-            size = stream.fetch_data_in_database({ :encoded_data => encoded_data, :checksum => data[:checksum] })
+            row_size = stream.fetch_data_in_database({ :encoded_data => encoded_data, :checksum => data[:checksum] })
+            log.debug "row size: #{row_size}"
             self.stream_state = stream.to_hash
 
             c.idle_secs = (d1 + d2)
@@ -481,7 +489,7 @@ module Tapsoob
       tbls.each do |table|
         if File.exists?(File.join(dump_path, "data", "#{table}.json"))
           data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))
-          tables_with_counts[table] = data.size
+          tables_with_counts[table] = data["data"].size
         else
           tables_with_counts[table] = 0
         end

data/lib/tapsoob/progress_bar.rb

@@ -22,7 +22,7 @@ class ProgressBar
     @current = 0
     @previous = 0
     @finished_p = false
-    @start_time = Time.now
+    @start_time = ::Time.now
     @previous_time = @start_time
     @title_width = 14
     @format = "%-#{@title_width}s %3d%% %s %s"
@@ -76,7 +76,7 @@ class ProgressBar
   end
 
   def transfer_rate
-    bytes_per_second = @current.to_f / (Time.now - @start_time)
+    bytes_per_second = @current.to_f / (::Time.now - @start_time)
     sprintf("%s/s", convert_bytes(bytes_per_second))
   end
 
@@ -97,14 +97,14 @@ class ProgressBar
     if @current == 0
       "ETA:  --:--:--"
     else
-      elapsed = Time.now - @start_time
+      elapsed = ::Time.now - @start_time
       eta = elapsed * @total / @current - elapsed;
       sprintf("ETA:  %s", format_time(eta))
     end
   end
 
   def elapsed
-    elapsed = Time.now - @start_time
+    elapsed = ::Time.now - @start_time
     sprintf("Time: %s", format_time(elapsed))
   end
 
@@ -155,7 +155,7 @@ class ProgressBar
       @terminal_width += width - line.length + 1
       show
     end
-    @previous_time = Time.now
+    @previous_time = ::Time.now
   end
 
   def show_if_needed
@@ -169,7 +169,7 @@ class ProgressBar
 
     # Use "!=" instead of ">" to support negative changes
     if cur_percentage != prev_percentage ||
-        Time.now - @previous_time >= 1 || @finished_p
+        ::Time.now - @previous_time >= 1 || @finished_p
       show
     end
   end

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.3.26".freeze
+  VERSION = "0.4.3".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.3.26
+  version: 0.4.3
 platform: ruby
 authors:
 - Félix Bellanger
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-11 00:00:00.000000000 Z
+date: 2021-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ripl