tapsoob 0.3.8-java → 0.3.19-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d3ab72e8b99be12c81678c0576411c9ecacdab0b1f40432710afad1de2eee38
-  data.tar.gz: 0d4f7624488356b7823b8f135a4e39627251e63497f208de6bd8848e367e3f40
+  metadata.gz: edc018c5a29b9591b4f6f8c5eaf01f2d275c430c9397b0c19c09d21da5fdb75c
+  data.tar.gz: 30bddf91e43eaef5bcacd9b7cb16a84dd4ad44f88e2ac678004317d9fdab8e60
 SHA512:
-  metadata.gz: b48f391f4a0ab695ebc73b0a76385b03580cbdf99888be4201b8375b31728f6e91af610e865ffd32c80e4a3ebb447d0a01d700136b391571859246936d87ee0a
-  data.tar.gz: 70b5a111087345c59f6c8b9c49f40913a4c07ad70b3647c1f875e3f671b19aa3e76f2c027531284827930969a3855919ea876a588e27440183974b7bf5676ab5
+  metadata.gz: 83033d2182ed00d868a5c380a58758ad7e93de9542f35db9e0819e123982dd75297ee716f202bf0fbe6e2ac5f7d05296a052f1669860f9e2a09f5d76b0edee27
+  data.tar.gz: 57a6a3355f4702753e1d028abec752bee789a6bd8e1daf8b959a40ae4fe75099db1891545c7a24851f518e1557d19b9843a0d3fe6c5fff1091a18d7556cd26ae

data/README.md

@@ -36,6 +36,23 @@ You can list all available options using the command:
     tapsoob push -h
 
 
+## NEW : Piping your schema/indexes/data
+
+Due to some needs we added ways to pipe your schema/indexes/data directly from one database to another, here's an equivalent of the export/import process described above using this technique :
+
+```
+tapsoob schema dump <db_source_url> | tapsoob schema load <db_target_url> --drop=true
+tapsoob schema indexes <db_source_url> | tapsoob schema load_indexes <db_target_url>
+tapsoob data pull <db_source_url> -p false | tapsoob data push <db_target_url>
+tapsoob schema reset_db_sequences <db_target_url>
+```
+
+A few notes here :
+
+* the `--drop` option for the `schema load` command defaults to false, if you don't intend to drop your tables on your target databases you can ommit it (if a table already exists tapsoob will exit w/ an error) ;
+* the `data pull` and `data push` commands are new and have a few options that you can display w/ `tapsoob data <pull|push> -h`, by defaults it prints the progress of your data extraction/import but if you want to pipe directly your export into another database you'll have to set the `--progress` (shorthand `-p`) option to `false` (or `--no-progress`) as above ;
+* resetting database sequences is important as a `data push` command doesn't handle that directly.
+
 ## Integration with Rails
 
 If you're using Rails, there's also two Rake tasks provided:

data/lib/tapsoob/data_stream.rb

@@ -218,6 +218,16 @@ module Tapsoob
     end
 
     def import_rows(rows)
+      # Decode blobs
+      if rows.has_key?(:types) && rows[:types].include?("blob")
+        blob_indices = rows[:types].each_index.select { |idx| rows[:types][idx] == "blob" }
+        rows[:data].each_index do |idx|
+          blob_indices.each do |bi|
+            rows[:data][idx][bi] = Sequel::SQL::Blob.new(Tapsoob::Utils.base64decode(rows[:data][idx][bi])) unless rows[:data][idx][bi].nil?
+          end
+        end
+      end
+      
       table.import(rows[:header], rows[:data], :commit_every => 100)
       state[:offset] += rows[:data].size
     rescue Exception => ex

data/lib/tapsoob/schema.rb

@@ -2,6 +2,7 @@
 require 'sequel'
 require 'sequel/extensions/schema_dumper'
 require 'sequel/extensions/migration'
+require 'erb'
 require 'json'
 
 module Tapsoob
@@ -11,7 +12,23 @@ module Tapsoob
     def dump(database_url)
       db = Sequel.connect(database_url)
       db.extension :schema_dumper
-      db.dump_schema_migration(:indexes => false)
+      template = ERB.new <<-END_MIG
+Class.new(Sequel::Migration) do
+  def up
+  <% db.tables.each do |table| %>
+    <%= db.dump_table_schema(table, indexes: false) %>
+  <% end %>
+  end
+
+  def down
+  <% db.tables.each do |table| %>
+    drop_table("<%= table %>", if_exists: true)
+  <% end %>
+  end
+end
+END_MIG
+
+      template.result(binding)
     end
 
     def dump_table(database_url, table)
@@ -21,7 +38,7 @@ module Tapsoob
         <<END_MIG
 Class.new(Sequel::Migration) do
   def up
-    #{db.dump_table_schema(table, :indexes => false)}
+    #{db.dump_table_schema(table, indexes: false)}
   end
 
   def down
@@ -67,7 +84,16 @@ END_MIG
       Sequel.connect(database_url) do |db|
         db.extension :schema_dumper
         klass = eval(schema)
-        klass.apply(db, :down) if options[:drop]
+        if options[:drop]
+          # Start special hack for MySQL
+          db.run("SET foreign_key_checks = 0") if [:mysql, :mysql2].include?(db.adapter_scheme)
+
+          # Run down migration
+          klass.apply(db, :down)
+
+          # End special hack for MySQL
+          db.run("SET foreign_key_checks = 1") if [:mysql, :mysql2].include?(db.adapter_scheme)
+        end
         klass.apply(db, :up)
       end
     end

data/lib/tapsoob/utils.rb

@@ -51,7 +51,7 @@ module Tapsoob
 
       header = data[0].keys
       only_data = data.collect do |row|
-        row = blobs_to_string(row, string_columns)
+        row = encode_blobs(row, string_columns)
         row.each do |column, data|
           if data.to_s.length > (max_lengths[column] || data.to_s.length)
             raise Tapsoob::InvalidData.new(<<-ERROR)
@@ -70,27 +70,33 @@ Data : #{data}
         end
         header.collect { |h| row[h] }
       end
-      { table_name: table, header: header, data: only_data }
+
+      res = { table_name: table, header: header, data: only_data }
+
+      # Add types if schema isn't empty
+      res[:types] = schema.map { |c| c.last[:type] } unless schema.empty?
+
+      res
     end
 
     # mysql text and blobs fields are handled the same way internally
     # this is not true for other databases so we must check if the field is
     # actually text and manually convert it back to a string
     def incorrect_blobs(db, table)
-      return [] if (db.url =~ /mysql:\/\//).nil?
+      return [] if (db.url =~ /(mysql|mysql2):\/\//).nil?
 
       columns = []
       db.schema(table).each do |data|
         column, cdata = data
-        columns << column if cdata[:db_type] =~ /text/
+        columns << column if cdata[:type] == :blob
       end
       columns
     end
 
-    def blobs_to_string(row, columns)
+    def encode_blobs(row, columns)
       return row if columns.size == 0
       columns.each do |c|
-        row[c] = row[c].to_s if row[c].kind_of?(Sequel::SQL::Blob)
+        row[c] = base64encode(row[c]) unless row[c].nil?
       end
       row
     end

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.3.8".freeze
+  VERSION = "0.3.19".freeze
 end

data/tapsoob.gemspec

@@ -21,8 +21,8 @@ Gem::Specification.new do |s|
 
   # Dependencies
   s.add_dependency "ripl", "~> 0.7.1"
-  s.add_dependency "sequel", "~> 5.25.0"
-  s.add_dependency "thor", "~> 0.20.3"
+  s.add_dependency "sequel", "~> 5.37.0"
+  s.add_dependency "thor", "~> 1.0.1"
 
   if (RUBY_PLATFORM =~ /java/).nil?
     s.add_development_dependency "mysql2",  "~> 0.4.10"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.3.19
 platform: java
 authors:
 - Félix Bellanger
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-27 00:00:00.000000000 Z
+date: 2020-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,7 +30,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.25.0
+        version: 5.37.0
   name: sequel
   prerelease: false
   type: :runtime
@@ -38,13 +38,13 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.25.0
+        version: 5.37.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.20.3
+        version: 1.0.1
   name: thor
   prerelease: false
   type: :runtime
@@ -52,7 +52,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.20.3
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -149,8 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.10
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: Simple tool to import/export databases.