tapsoob 0.3.16 → 0.3.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a55bc7c545448861ed66362939755b363326d7b2d87ccc048a670e47eee228be
-  data.tar.gz: c411d6a74337e37b7dd244066360320d81b1afda6ea1d0d070586011288f9f10
+  metadata.gz: 7109b252c22de2c4c796068f6ac2fc4df267e4b1130ee875a40b08f9ee7a1224
+  data.tar.gz: 5530cb2317ae8a4829335b20e90bbb1788a546f898839a1f330c379ecd2973c1
 SHA512:
-  metadata.gz: 0cdd9c82ff53ac8328e201c82729bb2128230ee2dcd83aea69ebcd4fa1df6f1371c330e1cb0e79235881a9e1ced4cd47400fac3e15e39bfce97d9d9d4f156ee7
-  data.tar.gz: 0efe17386c054d46541ce9a78464300b4d302df2ed62f1df3b7517096f6701561f70b906880c5a71b8526e8211c3b29d6856c65565ee14d274f2595d9dd63ee5
+  metadata.gz: cf1960e1ef01a40a0e22f66a042616d7a16b3030f8c7ad20227b7ab44805b175872ec3b030d4e1a3c08469ccf3a33f606b8bcdd14bfddb58ce2378ade773dfac
+  data.tar.gz: 9abb44b2ca81049159c78dc771938bb13e8334f008f51d9c899f5b8c22928621d003a429537fc639eef713d3005ce9804731b09db95d5bde24f97d2615dd7bef

data/lib/tapsoob/cli/root.rb

@@ -38,6 +38,7 @@ module Tapsoob
       option :filter, desc: "Regex Filter for tables", type: :string, aliases: "-f"
       option :tables, desc: "Shortcut to filter on a list of tables", type: :array, aliases: "-t"
       option :"exclude-tables", desc: "Shortcut to exclude a list of tables", type: :array, aliases: "-e"
+      option :purge, desc: "Purge data in tables prior to performing the import", default: false, type: :boolean, aliases: "-p"
       option :debug, desc: "Enable debug messages", default: false, type: :boolean, aliases: "-d"
       def push(dump_path, database_url)
         opts = parse_opts(options)
@@ -70,6 +71,9 @@ module Tapsoob
             debug: options[:debug]
           }
 
+          # Purge (push only)
+          opts[:purge] = options[:purge] if options.key?(:purge)
+
           # Resume
           if options[:resume]
             if File.exists?(options[:resume])

data/lib/tapsoob/data_stream.rb

@@ -218,6 +218,16 @@ module Tapsoob
     end
 
     def import_rows(rows)
+      # Decode blobs
+      if rows.has_key?(:types) && rows[:types].include?("blob")
+        blob_indices = rows[:types].each_index.select { |idx| rows[:types][idx] == "blob" }
+        rows[:data].each_index do |idx|
+          blob_indices.each do |bi|
+            rows[:data][idx][bi] = Sequel::SQL::Blob.new(Tapsoob::Utils.base64decode(rows[:data][idx][bi])) unless rows[:data][idx][bi].nil?
+          end
+        end
+      end
+      
       table.import(rows[:header], rows[:data], :commit_every => 100)
       state[:offset] += rows[:data].size
     rescue Exception => ex

data/lib/tapsoob/operation.rb

@@ -393,6 +393,7 @@ module Tapsoob
 
       tables.each do |table_name, count|
         next unless File.exists?(File.join(dump_path, "data", "#{table_name}.json"))
+        db[table_name.to_sym].truncate if @opts[:purge]
         stream = Tapsoob::DataStream.factory(db,
           :table_name => table_name,
           :chunksize => default_chunksize)
@@ -476,7 +477,7 @@ module Tapsoob
 
     def fetch_local_tables_info
       tables_with_counts = {}
-      tbls = Dir.glob(File.join(dump_path, "schemas", "*")).map { |path| File.basename(path, ".rb") }
+      tbls = Dir.glob(File.join(dump_path, "data", "*")).map { |path| File.basename(path, ".json") }
       tbls.each do |table|
         if File.exists?(File.join(dump_path, "data", "#{table}.json"))
           data = JSON.parse(File.read(File.join(dump_path, "data", "#{table}.json")))

data/lib/tapsoob/utils.rb

@@ -51,7 +51,7 @@ module Tapsoob
 
       header = data[0].keys
       only_data = data.collect do |row|
-        row = blobs_to_string(row, string_columns)
+        row = encode_blobs(row, string_columns)
         row.each do |column, data|
           if data.to_s.length > (max_lengths[column] || data.to_s.length)
             raise Tapsoob::InvalidData.new(<<-ERROR)
@@ -70,27 +70,33 @@ Data : #{data}
         end
         header.collect { |h| row[h] }
       end
-      { table_name: table, header: header, data: only_data }
+
+      res = { table_name: table, header: header, data: only_data }
+
+      # Add types if schema isn't empty
+      res[:types] = schema.map { |c| c.last[:type] } unless schema.empty?
+
+      res
     end
 
     # mysql text and blobs fields are handled the same way internally
     # this is not true for other databases so we must check if the field is
     # actually text and manually convert it back to a string
     def incorrect_blobs(db, table)
-      return [] if (db.url =~ /mysql:\/\//).nil?
+      return [] if (db.url =~ /(mysql|mysql2):\/\//).nil?
 
       columns = []
       db.schema(table).each do |data|
         column, cdata = data
-        columns << column if cdata[:db_type] =~ /text/
+        columns << column if cdata[:type] == :blob
       end
       columns
     end
 
-    def blobs_to_string(row, columns)
+    def encode_blobs(row, columns)
       return row if columns.size == 0
       columns.each do |c|
-        row[c] = row[c].to_s if row[c].kind_of?(Sequel::SQL::Blob)
+        row[c] = base64encode(row[c]) unless row[c].nil?
       end
       row
     end

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.3.16".freeze
+  VERSION = "0.3.21".freeze
 end

data/tapsoob.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
 
   # Dependencies
   s.add_dependency "ripl", "~> 0.7.1"
-  s.add_dependency "sequel", "~> 5.25.0"
+  s.add_dependency "sequel", "~> 5.37.0"
   s.add_dependency "thor", "~> 1.0.1"
 
   if (RUBY_PLATFORM =~ /java/).nil?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.3.16
+  version: 0.3.21
 platform: ruby
 authors:
 - Félix Bellanger
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-08 00:00:00.000000000 Z
+date: 2021-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ripl
@@ -31,14 +31,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.25.0
+        version: 5.37.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.25.0
+        version: 5.37.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -149,8 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.6.2
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Simple tool to import/export databases.